diff --git "a/checkpoint-2500/trainer_state.json" "b/checkpoint-2500/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-2500/trainer_state.json" @@ -0,0 +1,35021 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.271716015049894, + "eval_steps": 500, + "global_step": 2500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 5.000000000000001e-07, + "logits/chosen": -2.586496591567993, + "logits/rejected": -2.58866286277771, + "logps/chosen": -140.71868896484375, + "logps/rejected": -141.9235382080078, + "loss": 0.6941, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.005378031637519598, + "rewards/margins": -0.0017773156287148595, + "rewards/rejected": -0.0036007165908813477, + "step": 1 + }, + { + "epoch": 0.0, + "learning_rate": 1.0000000000000002e-06, + "logits/chosen": -2.7250607013702393, + "logits/rejected": -2.737359046936035, + "logps/chosen": -239.8214111328125, + "logps/rejected": -245.04498291015625, + "loss": 0.69, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.0041117193177342415, + "rewards/margins": 0.006983352825045586, + "rewards/rejected": -0.002871633041650057, + "step": 2 + }, + { + "epoch": 0.0, + "learning_rate": 1.5e-06, + "logits/chosen": -2.407137393951416, + "logits/rejected": -2.3473005294799805, + "logps/chosen": -184.03599548339844, + "logps/rejected": -155.21461486816406, + "loss": 0.6886, + "rewards/accuracies": 0.4375, + "rewards/chosen": 0.003687595948576927, + "rewards/margins": 0.009552741423249245, + "rewards/rejected": -0.0058651454746723175, + "step": 3 + }, + { + "epoch": 0.01, + "learning_rate": 2.0000000000000003e-06, + "logits/chosen": -2.7127842903137207, + "logits/rejected": -2.777282953262329, + "logps/chosen": -171.6352081298828, + "logps/rejected": -191.40785217285156, + "loss": 0.7021, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.004523229319602251, + "rewards/margins": -0.01729598268866539, + "rewards/rejected": 0.012772750109434128, + "step": 4 + }, + { + "epoch": 0.01, + "learning_rate": 2.5e-06, + "logits/chosen": -2.4465034008026123, + "logits/rejected": -2.466231107711792, + "logps/chosen": -254.792724609375, + "logps/rejected": -245.39108276367188, + "loss": 0.6988, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.004072976298630238, + "rewards/margins": -0.010413647629320621, + "rewards/rejected": 0.006340669468045235, + "step": 5 + }, + { + "epoch": 0.01, + "learning_rate": 3e-06, + "logits/chosen": -2.2196829319000244, + "logits/rejected": -2.2641539573669434, + "logps/chosen": -179.39495849609375, + "logps/rejected": -220.39300537109375, + "loss": 0.6891, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.01630725897848606, + "rewards/margins": 0.008536052890121937, + "rewards/rejected": 0.00777120515704155, + "step": 6 + }, + { + "epoch": 0.01, + "learning_rate": 3.5000000000000004e-06, + "logits/chosen": -2.3294830322265625, + "logits/rejected": -2.380467653274536, + "logps/chosen": -194.7043914794922, + "logps/rejected": -199.31422424316406, + "loss": 0.6919, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.005030441097915173, + "rewards/margins": 0.0030234563164412975, + "rewards/rejected": 0.0020069843158125877, + "step": 7 + }, + { + "epoch": 0.01, + "learning_rate": 4.000000000000001e-06, + "logits/chosen": -2.590843677520752, + "logits/rejected": -2.5107312202453613, + "logps/chosen": -215.23556518554688, + "logps/rejected": -189.38424682617188, + "loss": 0.6995, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.0025288108736276627, + "rewards/margins": -0.012505888007581234, + "rewards/rejected": 0.009977078065276146, + "step": 8 + }, + { + "epoch": 0.01, + "learning_rate": 4.5e-06, + "logits/chosen": -2.659663200378418, + "logits/rejected": -2.6844048500061035, + "logps/chosen": -229.0231170654297, + "logps/rejected": -205.54054260253906, + "loss": 0.7032, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.014649391174316406, + "rewards/margins": -0.019562961533665657, + "rewards/rejected": 0.0049135684967041016, + "step": 9 + }, + { + "epoch": 0.01, + "learning_rate": 5e-06, + "logits/chosen": -2.6927154064178467, + "logits/rejected": -2.6597423553466797, + "logps/chosen": -207.30467224121094, + "logps/rejected": -197.54656982421875, + "loss": 0.692, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0019803522154688835, + "rewards/margins": 0.002554560313001275, + "rewards/rejected": -0.004534911829978228, + "step": 10 + }, + { + "epoch": 0.01, + "learning_rate": 5.500000000000001e-06, + "logits/chosen": -2.6474952697753906, + "logits/rejected": -2.6725170612335205, + "logps/chosen": -231.35108947753906, + "logps/rejected": -274.9834289550781, + "loss": 0.6924, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.0055945878848433495, + "rewards/margins": 0.0016048436518758535, + "rewards/rejected": 0.003989744000136852, + "step": 11 + }, + { + "epoch": 0.02, + "learning_rate": 6e-06, + "logits/chosen": -2.2198939323425293, + "logits/rejected": -2.2628731727600098, + "logps/chosen": -198.29022216796875, + "logps/rejected": -170.82257080078125, + "loss": 0.688, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0031775482930243015, + "rewards/margins": 0.010572671890258789, + "rewards/rejected": -0.013750219717621803, + "step": 12 + }, + { + "epoch": 0.02, + "learning_rate": 6.5000000000000004e-06, + "logits/chosen": -2.7613415718078613, + "logits/rejected": -2.8206562995910645, + "logps/chosen": -239.75253295898438, + "logps/rejected": -238.97320556640625, + "loss": 0.6999, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.0029205563478171825, + "rewards/margins": -0.013042164966464043, + "rewards/rejected": 0.010121609084308147, + "step": 13 + }, + { + "epoch": 0.02, + "learning_rate": 7.000000000000001e-06, + "logits/chosen": -2.0942578315734863, + "logits/rejected": -2.017871856689453, + "logps/chosen": -184.2721710205078, + "logps/rejected": -175.73641967773438, + "loss": 0.6872, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.004038786515593529, + "rewards/margins": 0.01193075068295002, + "rewards/rejected": -0.015969539061188698, + "step": 14 + }, + { + "epoch": 0.02, + "learning_rate": 7.5e-06, + "logits/chosen": -2.526791572570801, + "logits/rejected": -2.5704824924468994, + "logps/chosen": -200.25115966796875, + "logps/rejected": -223.4153289794922, + "loss": 0.6917, + "rewards/accuracies": 0.4375, + "rewards/chosen": 0.00700113782659173, + "rewards/margins": 0.003126000752672553, + "rewards/rejected": 0.003875136375427246, + "step": 15 + }, + { + "epoch": 0.02, + "learning_rate": 8.000000000000001e-06, + "logits/chosen": -2.5145041942596436, + "logits/rejected": -2.478262424468994, + "logps/chosen": -169.4720001220703, + "logps/rejected": -179.3622283935547, + "loss": 0.687, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.00018756365170702338, + "rewards/margins": 0.012972594238817692, + "rewards/rejected": -0.013160157017409801, + "step": 16 + }, + { + "epoch": 0.02, + "learning_rate": 8.500000000000002e-06, + "logits/chosen": -2.1660895347595215, + "logits/rejected": -2.1799678802490234, + "logps/chosen": -158.1959686279297, + "logps/rejected": -225.02365112304688, + "loss": 0.687, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.01399233564734459, + "rewards/margins": 0.01266777515411377, + "rewards/rejected": 0.001324558281339705, + "step": 17 + }, + { + "epoch": 0.02, + "learning_rate": 9e-06, + "logits/chosen": -2.417539596557617, + "logits/rejected": -2.4626646041870117, + "logps/chosen": -170.4958038330078, + "logps/rejected": -178.34300231933594, + "loss": 0.6877, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.0049651628360152245, + "rewards/margins": 0.011217641644179821, + "rewards/rejected": -0.006252479739487171, + "step": 18 + }, + { + "epoch": 0.02, + "learning_rate": 9.5e-06, + "logits/chosen": -2.6470260620117188, + "logits/rejected": -2.7213802337646484, + "logps/chosen": -179.53497314453125, + "logps/rejected": -200.94956970214844, + "loss": 0.69, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.0027004480361938477, + "rewards/margins": 0.006518864538520575, + "rewards/rejected": -0.00921931304037571, + "step": 19 + }, + { + "epoch": 0.03, + "learning_rate": 1e-05, + "logits/chosen": -2.4962029457092285, + "logits/rejected": -2.5313973426818848, + "logps/chosen": -208.752685546875, + "logps/rejected": -244.1136474609375, + "loss": 0.683, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.009996438398957253, + "rewards/margins": 0.020868420600891113, + "rewards/rejected": -0.01087198406457901, + "step": 20 + }, + { + "epoch": 0.03, + "learning_rate": 1.05e-05, + "logits/chosen": -2.531207799911499, + "logits/rejected": -2.5359435081481934, + "logps/chosen": -194.47694396972656, + "logps/rejected": -185.80142211914062, + "loss": 0.6895, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.002445125486701727, + "rewards/margins": 0.0077151767909526825, + "rewards/rejected": -0.005270051304250956, + "step": 21 + }, + { + "epoch": 0.03, + "learning_rate": 1.1000000000000001e-05, + "logits/chosen": -2.430760622024536, + "logits/rejected": -2.395613431930542, + "logps/chosen": -144.2230987548828, + "logps/rejected": -148.8902587890625, + "loss": 0.6942, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.00968785211443901, + "rewards/margins": -0.001990032149478793, + "rewards/rejected": -0.007697821129113436, + "step": 22 + }, + { + "epoch": 0.03, + "learning_rate": 1.1500000000000002e-05, + "logits/chosen": -2.4417223930358887, + "logits/rejected": -2.441425323486328, + "logps/chosen": -165.82875061035156, + "logps/rejected": -178.82815551757812, + "loss": 0.6886, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0060547590255737305, + "rewards/margins": 0.009464193135499954, + "rewards/rejected": -0.0034094336442649364, + "step": 23 + }, + { + "epoch": 0.03, + "learning_rate": 1.2e-05, + "logits/chosen": -2.3494763374328613, + "logits/rejected": -2.40555477142334, + "logps/chosen": -133.24130249023438, + "logps/rejected": -151.85446166992188, + "loss": 0.6986, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.011995697394013405, + "rewards/margins": -0.010638857260346413, + "rewards/rejected": -0.0013568403664976358, + "step": 24 + }, + { + "epoch": 0.03, + "learning_rate": 1.25e-05, + "logits/chosen": -2.3450560569763184, + "logits/rejected": -2.5124807357788086, + "logps/chosen": -242.19821166992188, + "logps/rejected": -259.1798095703125, + "loss": 0.6912, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.013580609112977982, + "rewards/margins": 0.004313135519623756, + "rewards/rejected": -0.017893744632601738, + "step": 25 + }, + { + "epoch": 0.03, + "learning_rate": 1.3000000000000001e-05, + "logits/chosen": -2.5465288162231445, + "logits/rejected": -2.5980730056762695, + "logps/chosen": -202.898193359375, + "logps/rejected": -248.03968811035156, + "loss": 0.6854, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.0018544672057032585, + "rewards/margins": 0.016238166019320488, + "rewards/rejected": -0.01438369695097208, + "step": 26 + }, + { + "epoch": 0.04, + "learning_rate": 1.3500000000000001e-05, + "logits/chosen": -2.733663320541382, + "logits/rejected": -2.712355375289917, + "logps/chosen": -205.80429077148438, + "logps/rejected": -174.95797729492188, + "loss": 0.6933, + "rewards/accuracies": 0.4375, + "rewards/chosen": -8.790497668087482e-05, + "rewards/margins": 2.1266518160700798e-05, + "rewards/rejected": -0.0001091718440875411, + "step": 27 + }, + { + "epoch": 0.04, + "learning_rate": 1.4000000000000001e-05, + "logits/chosen": -2.550997257232666, + "logits/rejected": -2.4390013217926025, + "logps/chosen": -171.59786987304688, + "logps/rejected": -176.4530029296875, + "loss": 0.6931, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.02987699769437313, + "rewards/margins": 0.0004779808223247528, + "rewards/rejected": -0.030354974791407585, + "step": 28 + }, + { + "epoch": 0.04, + "learning_rate": 1.45e-05, + "logits/chosen": -2.4879837036132812, + "logits/rejected": -2.480923652648926, + "logps/chosen": -149.10079956054688, + "logps/rejected": -162.999267578125, + "loss": 0.6834, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.005112767685204744, + "rewards/margins": 0.01993861421942711, + "rewards/rejected": -0.025051379576325417, + "step": 29 + }, + { + "epoch": 0.04, + "learning_rate": 1.5e-05, + "logits/chosen": -2.3427562713623047, + "logits/rejected": -2.5123889446258545, + "logps/chosen": -160.62950134277344, + "logps/rejected": -201.14747619628906, + "loss": 0.6952, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.015586448833346367, + "rewards/margins": -0.003298282390460372, + "rewards/rejected": -0.012288165278732777, + "step": 30 + }, + { + "epoch": 0.04, + "learning_rate": 1.55e-05, + "logits/chosen": -2.2548415660858154, + "logits/rejected": -2.3827996253967285, + "logps/chosen": -126.6205825805664, + "logps/rejected": -145.7274627685547, + "loss": 0.6883, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.02384941652417183, + "rewards/margins": 0.010512137785553932, + "rewards/rejected": -0.03436155617237091, + "step": 31 + }, + { + "epoch": 0.04, + "learning_rate": 1.6000000000000003e-05, + "logits/chosen": -2.5090818405151367, + "logits/rejected": -2.560624122619629, + "logps/chosen": -182.53944396972656, + "logps/rejected": -208.08763122558594, + "loss": 0.6877, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.0014843230601400137, + "rewards/margins": 0.011496281251311302, + "rewards/rejected": -0.01298060454428196, + "step": 32 + }, + { + "epoch": 0.04, + "learning_rate": 1.65e-05, + "logits/chosen": -2.800107479095459, + "logits/rejected": -2.7355594635009766, + "logps/chosen": -207.99392700195312, + "logps/rejected": -208.31610107421875, + "loss": 0.6927, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.014949416741728783, + "rewards/margins": 0.0012244456447660923, + "rewards/rejected": -0.016173863783478737, + "step": 33 + }, + { + "epoch": 0.04, + "learning_rate": 1.7000000000000003e-05, + "logits/chosen": -2.6325690746307373, + "logits/rejected": -2.630256175994873, + "logps/chosen": -196.8790283203125, + "logps/rejected": -202.53794860839844, + "loss": 0.7116, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.041500113904476166, + "rewards/margins": -0.03608906269073486, + "rewards/rejected": -0.005411052145063877, + "step": 34 + }, + { + "epoch": 0.05, + "learning_rate": 1.75e-05, + "logits/chosen": -2.476334571838379, + "logits/rejected": -2.473987579345703, + "logps/chosen": -256.1812438964844, + "logps/rejected": -250.76165771484375, + "loss": 0.6882, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.01104288175702095, + "rewards/margins": 0.010350894182920456, + "rewards/rejected": -0.021393775939941406, + "step": 35 + }, + { + "epoch": 0.05, + "learning_rate": 1.8e-05, + "logits/chosen": -2.342329502105713, + "logits/rejected": -2.3805429935455322, + "logps/chosen": -164.03692626953125, + "logps/rejected": -217.39849853515625, + "loss": 0.688, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.009070659056305885, + "rewards/margins": 0.010929775424301624, + "rewards/rejected": -0.020000435411930084, + "step": 36 + }, + { + "epoch": 0.05, + "learning_rate": 1.85e-05, + "logits/chosen": -2.4253318309783936, + "logits/rejected": -2.4550042152404785, + "logps/chosen": -146.70420837402344, + "logps/rejected": -129.80674743652344, + "loss": 0.6845, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.031095195561647415, + "rewards/margins": 0.017679547891020775, + "rewards/rejected": -0.04877474159002304, + "step": 37 + }, + { + "epoch": 0.05, + "learning_rate": 1.9e-05, + "logits/chosen": -2.4936635494232178, + "logits/rejected": -2.5476386547088623, + "logps/chosen": -164.6866455078125, + "logps/rejected": -198.52139282226562, + "loss": 0.6895, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.025130080059170723, + "rewards/margins": 0.007643342949450016, + "rewards/rejected": -0.03277342766523361, + "step": 38 + }, + { + "epoch": 0.05, + "learning_rate": 1.9500000000000003e-05, + "logits/chosen": -2.4614553451538086, + "logits/rejected": -2.4693663120269775, + "logps/chosen": -172.24966430664062, + "logps/rejected": -166.52056884765625, + "loss": 0.6975, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.027715325355529785, + "rewards/margins": -0.00841212272644043, + "rewards/rejected": -0.019303202629089355, + "step": 39 + }, + { + "epoch": 0.05, + "learning_rate": 2e-05, + "logits/chosen": -2.3024024963378906, + "logits/rejected": -2.2906301021575928, + "logps/chosen": -243.98683166503906, + "logps/rejected": -199.12408447265625, + "loss": 0.6979, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.028235863894224167, + "rewards/margins": -0.00847182422876358, + "rewards/rejected": -0.019764041528105736, + "step": 40 + }, + { + "epoch": 0.05, + "learning_rate": 2.05e-05, + "logits/chosen": -2.660562753677368, + "logits/rejected": -2.6754074096679688, + "logps/chosen": -156.67514038085938, + "logps/rejected": -147.5068359375, + "loss": 0.6887, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.03220677375793457, + "rewards/margins": 0.009226083755493164, + "rewards/rejected": -0.041432857513427734, + "step": 41 + }, + { + "epoch": 0.05, + "learning_rate": 2.1e-05, + "logits/chosen": -2.482787847518921, + "logits/rejected": -2.4663233757019043, + "logps/chosen": -207.6859588623047, + "logps/rejected": -182.35931396484375, + "loss": 0.6875, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.014715791679918766, + "rewards/margins": 0.012153576128184795, + "rewards/rejected": -0.02686937153339386, + "step": 42 + }, + { + "epoch": 0.06, + "learning_rate": 2.15e-05, + "logits/chosen": -2.6544148921966553, + "logits/rejected": -2.717411756515503, + "logps/chosen": -197.96400451660156, + "logps/rejected": -223.0219268798828, + "loss": 0.691, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.024796580895781517, + "rewards/margins": 0.005184031091630459, + "rewards/rejected": -0.0299806110560894, + "step": 43 + }, + { + "epoch": 0.06, + "learning_rate": 2.2000000000000003e-05, + "logits/chosen": -2.561187744140625, + "logits/rejected": -2.6224913597106934, + "logps/chosen": -194.60935974121094, + "logps/rejected": -194.01577758789062, + "loss": 0.6971, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.020246006548404694, + "rewards/margins": -0.0076716188341379166, + "rewards/rejected": -0.012574386782944202, + "step": 44 + }, + { + "epoch": 0.06, + "learning_rate": 2.25e-05, + "logits/chosen": -2.5130224227905273, + "logits/rejected": -2.672990083694458, + "logps/chosen": -154.0092010498047, + "logps/rejected": -190.70748901367188, + "loss": 0.6818, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.018519926816225052, + "rewards/margins": 0.023535681888461113, + "rewards/rejected": -0.042055610567331314, + "step": 45 + }, + { + "epoch": 0.06, + "learning_rate": 2.3000000000000003e-05, + "logits/chosen": -2.738673686981201, + "logits/rejected": -2.82279896736145, + "logps/chosen": -174.6641387939453, + "logps/rejected": -163.0157928466797, + "loss": 0.6967, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.014652942307293415, + "rewards/margins": -0.006221937946975231, + "rewards/rejected": -0.008431006222963333, + "step": 46 + }, + { + "epoch": 0.06, + "learning_rate": 2.35e-05, + "logits/chosen": -2.581393241882324, + "logits/rejected": -2.5734691619873047, + "logps/chosen": -137.8702392578125, + "logps/rejected": -154.74896240234375, + "loss": 0.6887, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.007426738273352385, + "rewards/margins": 0.009324884042143822, + "rewards/rejected": -0.016751624643802643, + "step": 47 + }, + { + "epoch": 0.06, + "learning_rate": 2.4e-05, + "logits/chosen": -2.544344425201416, + "logits/rejected": -2.690079689025879, + "logps/chosen": -185.8907012939453, + "logps/rejected": -195.9602508544922, + "loss": 0.6898, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.02690129354596138, + "rewards/margins": 0.007246016524732113, + "rewards/rejected": -0.03414731100201607, + "step": 48 + }, + { + "epoch": 0.06, + "learning_rate": 2.45e-05, + "logits/chosen": -2.583885669708252, + "logits/rejected": -2.609614849090576, + "logps/chosen": -215.93264770507812, + "logps/rejected": -234.49636840820312, + "loss": 0.6858, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.022774625569581985, + "rewards/margins": 0.015723586082458496, + "rewards/rejected": -0.03849821165204048, + "step": 49 + }, + { + "epoch": 0.07, + "learning_rate": 2.5e-05, + "logits/chosen": -2.6080195903778076, + "logits/rejected": -2.655003070831299, + "logps/chosen": -229.5977020263672, + "logps/rejected": -211.47779846191406, + "loss": 0.6925, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.026349734514951706, + "rewards/margins": 0.0026414887979626656, + "rewards/rejected": -0.028991222381591797, + "step": 50 + }, + { + "epoch": 0.07, + "learning_rate": 2.5500000000000003e-05, + "logits/chosen": -2.180418014526367, + "logits/rejected": -2.210296869277954, + "logps/chosen": -221.1846466064453, + "logps/rejected": -185.85655212402344, + "loss": 0.6891, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.013596129603683949, + "rewards/margins": 0.009130668826401234, + "rewards/rejected": -0.022726796567440033, + "step": 51 + }, + { + "epoch": 0.07, + "learning_rate": 2.6000000000000002e-05, + "logits/chosen": -2.647798538208008, + "logits/rejected": -2.715623617172241, + "logps/chosen": -186.01138305664062, + "logps/rejected": -180.89259338378906, + "loss": 0.6785, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.019299650564789772, + "rewards/margins": 0.030588869005441666, + "rewards/rejected": -0.011289214715361595, + "step": 52 + }, + { + "epoch": 0.07, + "learning_rate": 2.6500000000000004e-05, + "logits/chosen": -2.570209503173828, + "logits/rejected": -2.5728917121887207, + "logps/chosen": -209.2962188720703, + "logps/rejected": -216.60145568847656, + "loss": 0.6824, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.005204535089433193, + "rewards/margins": 0.021862652152776718, + "rewards/rejected": -0.027067184448242188, + "step": 53 + }, + { + "epoch": 0.07, + "learning_rate": 2.7000000000000002e-05, + "logits/chosen": -2.672147512435913, + "logits/rejected": -2.678018808364868, + "logps/chosen": -167.4713134765625, + "logps/rejected": -170.08120727539062, + "loss": 0.691, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.013661455363035202, + "rewards/margins": 0.004836535546928644, + "rewards/rejected": -0.018497992306947708, + "step": 54 + }, + { + "epoch": 0.07, + "learning_rate": 2.7500000000000004e-05, + "logits/chosen": -2.4589505195617676, + "logits/rejected": -2.462191343307495, + "logps/chosen": -200.73976135253906, + "logps/rejected": -195.30276489257812, + "loss": 0.704, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.03748317062854767, + "rewards/margins": -0.020142268389463425, + "rewards/rejected": -0.017340898513793945, + "step": 55 + }, + { + "epoch": 0.07, + "learning_rate": 2.8000000000000003e-05, + "logits/chosen": -2.230989933013916, + "logits/rejected": -2.2140636444091797, + "logps/chosen": -143.33631896972656, + "logps/rejected": -136.0796661376953, + "loss": 0.6989, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.020144915208220482, + "rewards/margins": -0.01022801361978054, + "rewards/rejected": -0.009916901588439941, + "step": 56 + }, + { + "epoch": 0.07, + "learning_rate": 2.8499999999999998e-05, + "logits/chosen": -2.5017194747924805, + "logits/rejected": -2.58662486076355, + "logps/chosen": -167.50990295410156, + "logps/rejected": -208.83607482910156, + "loss": 0.6808, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0027861359994858503, + "rewards/margins": 0.02604994922876358, + "rewards/rejected": -0.02883608266711235, + "step": 57 + }, + { + "epoch": 0.08, + "learning_rate": 2.9e-05, + "logits/chosen": -2.53066349029541, + "logits/rejected": -2.608471393585205, + "logps/chosen": -241.85113525390625, + "logps/rejected": -260.0408630371094, + "loss": 0.6939, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.024016117677092552, + "rewards/margins": -0.0005100721027702093, + "rewards/rejected": -0.023506049066781998, + "step": 58 + }, + { + "epoch": 0.08, + "learning_rate": 2.95e-05, + "logits/chosen": -2.2705140113830566, + "logits/rejected": -2.4387779235839844, + "logps/chosen": -179.22540283203125, + "logps/rejected": -182.5544891357422, + "loss": 0.6775, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.00418963422998786, + "rewards/margins": 0.033549048006534576, + "rewards/rejected": -0.02935941144824028, + "step": 59 + }, + { + "epoch": 0.08, + "learning_rate": 3e-05, + "logits/chosen": -2.46372652053833, + "logits/rejected": -2.513279438018799, + "logps/chosen": -230.43133544921875, + "logps/rejected": -227.0118865966797, + "loss": 0.6761, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.010919665917754173, + "rewards/margins": 0.03570995107293129, + "rewards/rejected": -0.04662961885333061, + "step": 60 + }, + { + "epoch": 0.08, + "learning_rate": 3.05e-05, + "logits/chosen": -2.530576467514038, + "logits/rejected": -2.5415897369384766, + "logps/chosen": -200.46568298339844, + "logps/rejected": -220.82989501953125, + "loss": 0.6794, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.0384901762008667, + "rewards/margins": 0.03156058490276337, + "rewards/rejected": -0.07005076855421066, + "step": 61 + }, + { + "epoch": 0.08, + "learning_rate": 3.1e-05, + "logits/chosen": -2.5888161659240723, + "logits/rejected": -2.639291286468506, + "logps/chosen": -132.55116271972656, + "logps/rejected": -167.082275390625, + "loss": 0.7042, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.03156990930438042, + "rewards/margins": -0.018804360181093216, + "rewards/rejected": -0.0127655528485775, + "step": 62 + }, + { + "epoch": 0.08, + "learning_rate": 3.15e-05, + "logits/chosen": -2.46744704246521, + "logits/rejected": -2.494837522506714, + "logps/chosen": -210.9451141357422, + "logps/rejected": -211.72433471679688, + "loss": 0.6799, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.0017502065747976303, + "rewards/margins": 0.027423406019806862, + "rewards/rejected": -0.02567320130765438, + "step": 63 + }, + { + "epoch": 0.08, + "learning_rate": 3.2000000000000005e-05, + "logits/chosen": -2.4783239364624023, + "logits/rejected": -2.490576982498169, + "logps/chosen": -197.71629333496094, + "logps/rejected": -207.23048400878906, + "loss": 0.6864, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.01412363164126873, + "rewards/margins": 0.01561451144516468, + "rewards/rejected": -0.029738139361143112, + "step": 64 + }, + { + "epoch": 0.09, + "learning_rate": 3.2500000000000004e-05, + "logits/chosen": -2.3918569087982178, + "logits/rejected": -2.4426636695861816, + "logps/chosen": -164.30029296875, + "logps/rejected": -201.85992431640625, + "loss": 0.6908, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.025302361696958542, + "rewards/margins": 0.009253643453121185, + "rewards/rejected": -0.034556008875370026, + "step": 65 + }, + { + "epoch": 0.09, + "learning_rate": 3.3e-05, + "logits/chosen": -2.4934239387512207, + "logits/rejected": -2.5188210010528564, + "logps/chosen": -178.12030029296875, + "logps/rejected": -211.23947143554688, + "loss": 0.6908, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.05651288107037544, + "rewards/margins": 0.0065690516494214535, + "rewards/rejected": -0.06308193504810333, + "step": 66 + }, + { + "epoch": 0.09, + "learning_rate": 3.35e-05, + "logits/chosen": -2.415985345840454, + "logits/rejected": -2.3687984943389893, + "logps/chosen": -211.91453552246094, + "logps/rejected": -216.65591430664062, + "loss": 0.6808, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.045519113540649414, + "rewards/margins": 0.028419354930520058, + "rewards/rejected": -0.07393846660852432, + "step": 67 + }, + { + "epoch": 0.09, + "learning_rate": 3.4000000000000007e-05, + "logits/chosen": -2.37481427192688, + "logits/rejected": -2.4693892002105713, + "logps/chosen": -172.92459106445312, + "logps/rejected": -209.3857421875, + "loss": 0.6801, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.03707296401262283, + "rewards/margins": 0.027949143201112747, + "rewards/rejected": -0.06502211093902588, + "step": 68 + }, + { + "epoch": 0.09, + "learning_rate": 3.45e-05, + "logits/chosen": -2.3615283966064453, + "logits/rejected": -2.3259239196777344, + "logps/chosen": -154.25360107421875, + "logps/rejected": -161.130126953125, + "loss": 0.7022, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.04533124342560768, + "rewards/margins": -0.016323519870638847, + "rewards/rejected": -0.029007721692323685, + "step": 69 + }, + { + "epoch": 0.09, + "learning_rate": 3.5e-05, + "logits/chosen": -2.64551043510437, + "logits/rejected": -2.7289645671844482, + "logps/chosen": -200.1268768310547, + "logps/rejected": -225.6541748046875, + "loss": 0.6993, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.0578455924987793, + "rewards/margins": -0.008899472653865814, + "rewards/rejected": -0.04894612357020378, + "step": 70 + }, + { + "epoch": 0.09, + "learning_rate": 3.55e-05, + "logits/chosen": -2.703721046447754, + "logits/rejected": -2.669755220413208, + "logps/chosen": -208.99668884277344, + "logps/rejected": -164.8292694091797, + "loss": 0.6756, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.022734597325325012, + "rewards/margins": 0.03882308304309845, + "rewards/rejected": -0.06155767664313316, + "step": 71 + }, + { + "epoch": 0.09, + "learning_rate": 3.6e-05, + "logits/chosen": -2.6480765342712402, + "logits/rejected": -2.696770191192627, + "logps/chosen": -172.99888610839844, + "logps/rejected": -201.09979248046875, + "loss": 0.7063, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.053855180740356445, + "rewards/margins": -0.024987507611513138, + "rewards/rejected": -0.028867674991488457, + "step": 72 + }, + { + "epoch": 0.1, + "learning_rate": 3.65e-05, + "logits/chosen": -2.6311445236206055, + "logits/rejected": -2.6575632095336914, + "logps/chosen": -251.72789001464844, + "logps/rejected": -271.0470886230469, + "loss": 0.6867, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.03289387375116348, + "rewards/margins": 0.014702942222356796, + "rewards/rejected": -0.04759681224822998, + "step": 73 + }, + { + "epoch": 0.1, + "learning_rate": 3.7e-05, + "logits/chosen": -2.3990683555603027, + "logits/rejected": -2.502671480178833, + "logps/chosen": -197.92665100097656, + "logps/rejected": -211.00967407226562, + "loss": 0.6841, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.014272330328822136, + "rewards/margins": 0.020740672945976257, + "rewards/rejected": -0.03501300886273384, + "step": 74 + }, + { + "epoch": 0.1, + "learning_rate": 3.7500000000000003e-05, + "logits/chosen": -2.575502872467041, + "logits/rejected": -2.527902364730835, + "logps/chosen": -214.5215301513672, + "logps/rejected": -201.05194091796875, + "loss": 0.6651, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.006246686447411776, + "rewards/margins": 0.059338975697755814, + "rewards/rejected": -0.065585657954216, + "step": 75 + }, + { + "epoch": 0.1, + "learning_rate": 3.8e-05, + "logits/chosen": -2.217294692993164, + "logits/rejected": -2.223574638366699, + "logps/chosen": -174.7819366455078, + "logps/rejected": -178.09852600097656, + "loss": 0.7013, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.09115451574325562, + "rewards/margins": -0.012327454052865505, + "rewards/rejected": -0.07882705330848694, + "step": 76 + }, + { + "epoch": 0.1, + "learning_rate": 3.85e-05, + "logits/chosen": -2.518965721130371, + "logits/rejected": -2.443446159362793, + "logps/chosen": -201.1251220703125, + "logps/rejected": -170.15533447265625, + "loss": 0.7029, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.04162323474884033, + "rewards/margins": -0.015459035523235798, + "rewards/rejected": -0.02616419643163681, + "step": 77 + }, + { + "epoch": 0.1, + "learning_rate": 3.9000000000000006e-05, + "logits/chosen": -2.521008253097534, + "logits/rejected": -2.6866934299468994, + "logps/chosen": -269.85284423828125, + "logps/rejected": -240.82626342773438, + "loss": 0.6928, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.05473289266228676, + "rewards/margins": 0.00423135980963707, + "rewards/rejected": -0.05896425247192383, + "step": 78 + }, + { + "epoch": 0.1, + "learning_rate": 3.9500000000000005e-05, + "logits/chosen": -2.3015501499176025, + "logits/rejected": -2.351773500442505, + "logps/chosen": -225.17691040039062, + "logps/rejected": -249.8846435546875, + "loss": 0.6873, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.06839534640312195, + "rewards/margins": 0.014748764224350452, + "rewards/rejected": -0.08314411342144012, + "step": 79 + }, + { + "epoch": 0.1, + "learning_rate": 4e-05, + "logits/chosen": -2.6201720237731934, + "logits/rejected": -2.680964708328247, + "logps/chosen": -225.08645629882812, + "logps/rejected": -232.00096130371094, + "loss": 0.6739, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.06486918777227402, + "rewards/margins": 0.04111311584711075, + "rewards/rejected": -0.10598230361938477, + "step": 80 + }, + { + "epoch": 0.11, + "learning_rate": 4.05e-05, + "logits/chosen": -2.5782110691070557, + "logits/rejected": -2.5646634101867676, + "logps/chosen": -249.90745544433594, + "logps/rejected": -257.51275634765625, + "loss": 0.6976, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.08720846474170685, + "rewards/margins": -0.0017095585353672504, + "rewards/rejected": -0.08549890667200089, + "step": 81 + }, + { + "epoch": 0.11, + "learning_rate": 4.1e-05, + "logits/chosen": -2.5777688026428223, + "logits/rejected": -2.665797472000122, + "logps/chosen": -171.82347106933594, + "logps/rejected": -211.63311767578125, + "loss": 0.6883, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.08461908996105194, + "rewards/margins": 0.013853237964212894, + "rewards/rejected": -0.09847234189510345, + "step": 82 + }, + { + "epoch": 0.11, + "learning_rate": 4.15e-05, + "logits/chosen": -2.3364946842193604, + "logits/rejected": -2.3084681034088135, + "logps/chosen": -193.4939727783203, + "logps/rejected": -184.15200805664062, + "loss": 0.717, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.0981694683432579, + "rewards/margins": -0.04480106756091118, + "rewards/rejected": -0.053368404507637024, + "step": 83 + }, + { + "epoch": 0.11, + "learning_rate": 4.2e-05, + "logits/chosen": -2.445204257965088, + "logits/rejected": -2.5341625213623047, + "logps/chosen": -174.59225463867188, + "logps/rejected": -210.94137573242188, + "loss": 0.699, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.09675168246030807, + "rewards/margins": -0.00985276885330677, + "rewards/rejected": -0.08689891546964645, + "step": 84 + }, + { + "epoch": 0.11, + "learning_rate": 4.25e-05, + "logits/chosen": -2.4798362255096436, + "logits/rejected": -2.4760632514953613, + "logps/chosen": -148.0768585205078, + "logps/rejected": -176.31700134277344, + "loss": 0.6945, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.07728591561317444, + "rewards/margins": 0.0020556673407554626, + "rewards/rejected": -0.0793415829539299, + "step": 85 + }, + { + "epoch": 0.11, + "learning_rate": 4.3e-05, + "logits/chosen": -2.5911381244659424, + "logits/rejected": -2.630659341812134, + "logps/chosen": -203.1630859375, + "logps/rejected": -183.4739227294922, + "loss": 0.653, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.04599933326244354, + "rewards/margins": 0.08558819442987442, + "rewards/rejected": -0.13158753514289856, + "step": 86 + }, + { + "epoch": 0.11, + "learning_rate": 4.35e-05, + "logits/chosen": -2.600797414779663, + "logits/rejected": -2.6318492889404297, + "logps/chosen": -156.8225860595703, + "logps/rejected": -161.6145477294922, + "loss": 0.6705, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.07066044956445694, + "rewards/margins": 0.05500438064336777, + "rewards/rejected": -0.1256648302078247, + "step": 87 + }, + { + "epoch": 0.12, + "learning_rate": 4.4000000000000006e-05, + "logits/chosen": -2.2124781608581543, + "logits/rejected": -2.259019374847412, + "logps/chosen": -142.0679473876953, + "logps/rejected": -177.9369659423828, + "loss": 0.6949, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.13633377850055695, + "rewards/margins": 0.000554969534277916, + "rewards/rejected": -0.1368887573480606, + "step": 88 + }, + { + "epoch": 0.12, + "learning_rate": 4.4500000000000004e-05, + "logits/chosen": -2.396183729171753, + "logits/rejected": -2.533904552459717, + "logps/chosen": -171.38790893554688, + "logps/rejected": -202.30979919433594, + "loss": 0.6554, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.09015192836523056, + "rewards/margins": 0.08280421048402786, + "rewards/rejected": -0.17295613884925842, + "step": 89 + }, + { + "epoch": 0.12, + "learning_rate": 4.5e-05, + "logits/chosen": -2.640425443649292, + "logits/rejected": -2.674539804458618, + "logps/chosen": -175.3508758544922, + "logps/rejected": -180.97915649414062, + "loss": 0.6768, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1077599823474884, + "rewards/margins": 0.03759467601776123, + "rewards/rejected": -0.14535464346408844, + "step": 90 + }, + { + "epoch": 0.12, + "learning_rate": 4.55e-05, + "logits/chosen": -2.5857696533203125, + "logits/rejected": -2.5822975635528564, + "logps/chosen": -176.55982971191406, + "logps/rejected": -196.5354461669922, + "loss": 0.7, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.09103889763355255, + "rewards/margins": -0.009319041855633259, + "rewards/rejected": -0.08171986043453217, + "step": 91 + }, + { + "epoch": 0.12, + "learning_rate": 4.600000000000001e-05, + "logits/chosen": -2.660163402557373, + "logits/rejected": -2.5935940742492676, + "logps/chosen": -233.04049682617188, + "logps/rejected": -212.1655731201172, + "loss": 0.6863, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.118477463722229, + "rewards/margins": 0.017601586878299713, + "rewards/rejected": -0.13607905805110931, + "step": 92 + }, + { + "epoch": 0.12, + "learning_rate": 4.6500000000000005e-05, + "logits/chosen": -2.5211241245269775, + "logits/rejected": -2.6064364910125732, + "logps/chosen": -176.89476013183594, + "logps/rejected": -187.39271545410156, + "loss": 0.6736, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.07700353115797043, + "rewards/margins": 0.04530329257249832, + "rewards/rejected": -0.12230683863162994, + "step": 93 + }, + { + "epoch": 0.12, + "learning_rate": 4.7e-05, + "logits/chosen": -2.6894078254699707, + "logits/rejected": -2.6806745529174805, + "logps/chosen": -244.7545623779297, + "logps/rejected": -212.83453369140625, + "loss": 0.6446, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10891007632017136, + "rewards/margins": 0.10942523181438446, + "rewards/rejected": -0.2183353304862976, + "step": 94 + }, + { + "epoch": 0.12, + "learning_rate": 4.75e-05, + "logits/chosen": -2.4758429527282715, + "logits/rejected": -2.5564448833465576, + "logps/chosen": -161.29554748535156, + "logps/rejected": -185.48199462890625, + "loss": 0.6991, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.152861088514328, + "rewards/margins": -0.0035788798704743385, + "rewards/rejected": -0.14928221702575684, + "step": 95 + }, + { + "epoch": 0.13, + "learning_rate": 4.8e-05, + "logits/chosen": -2.47318959236145, + "logits/rejected": -2.439746856689453, + "logps/chosen": -241.09283447265625, + "logps/rejected": -218.68580627441406, + "loss": 0.7017, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1952984780073166, + "rewards/margins": -0.01051153801381588, + "rewards/rejected": -0.18478693068027496, + "step": 96 + }, + { + "epoch": 0.13, + "learning_rate": 4.85e-05, + "logits/chosen": -2.541466474533081, + "logits/rejected": -2.56956148147583, + "logps/chosen": -180.84725952148438, + "logps/rejected": -211.02352905273438, + "loss": 0.6627, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15894030034542084, + "rewards/margins": 0.06507530808448792, + "rewards/rejected": -0.22401559352874756, + "step": 97 + }, + { + "epoch": 0.13, + "learning_rate": 4.9e-05, + "logits/chosen": -2.226611852645874, + "logits/rejected": -2.359558343887329, + "logps/chosen": -145.781494140625, + "logps/rejected": -181.21531677246094, + "loss": 0.6689, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.177232027053833, + "rewards/margins": 0.058251187205314636, + "rewards/rejected": -0.23548321425914764, + "step": 98 + }, + { + "epoch": 0.13, + "learning_rate": 4.9500000000000004e-05, + "logits/chosen": -2.5605039596557617, + "logits/rejected": -2.641690492630005, + "logps/chosen": -156.43446350097656, + "logps/rejected": -171.6468505859375, + "loss": 0.6801, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.23164159059524536, + "rewards/margins": 0.03432049974799156, + "rewards/rejected": -0.2659620940685272, + "step": 99 + }, + { + "epoch": 0.13, + "learning_rate": 5e-05, + "logits/chosen": -2.7840309143066406, + "logits/rejected": -2.8028616905212402, + "logps/chosen": -221.6836395263672, + "logps/rejected": -247.433349609375, + "loss": 0.647, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.14767608046531677, + "rewards/margins": 0.1068597286939621, + "rewards/rejected": -0.25453582406044006, + "step": 100 + }, + { + "epoch": 0.13, + "learning_rate": 4.99999978299634e-05, + "logits/chosen": -2.1325531005859375, + "logits/rejected": -2.1438934803009033, + "logps/chosen": -171.369140625, + "logps/rejected": -184.96827697753906, + "loss": 0.6417, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.18850459158420563, + "rewards/margins": 0.12018898129463196, + "rewards/rejected": -0.3086935877799988, + "step": 101 + }, + { + "epoch": 0.13, + "learning_rate": 4.999999131985394e-05, + "logits/chosen": -2.6781508922576904, + "logits/rejected": -2.5359549522399902, + "logps/chosen": -232.26742553710938, + "logps/rejected": -214.5188446044922, + "loss": 0.7372, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.32363808155059814, + "rewards/margins": -0.07766950130462646, + "rewards/rejected": -0.2459685504436493, + "step": 102 + }, + { + "epoch": 0.13, + "learning_rate": 4.999998046967279e-05, + "logits/chosen": -2.54486346244812, + "logits/rejected": -2.6447086334228516, + "logps/chosen": -179.6219482421875, + "logps/rejected": -222.37010192871094, + "loss": 0.6585, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.23128211498260498, + "rewards/margins": 0.08162947744131088, + "rewards/rejected": -0.31291159987449646, + "step": 103 + }, + { + "epoch": 0.14, + "learning_rate": 4.9999965279421804e-05, + "logits/chosen": -2.4281625747680664, + "logits/rejected": -2.5068893432617188, + "logps/chosen": -163.3339385986328, + "logps/rejected": -180.06222534179688, + "loss": 0.6739, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.22076301276683807, + "rewards/margins": 0.05238480493426323, + "rewards/rejected": -0.2731478214263916, + "step": 104 + }, + { + "epoch": 0.14, + "learning_rate": 4.999994574910364e-05, + "logits/chosen": -2.5497522354125977, + "logits/rejected": -2.558523178100586, + "logps/chosen": -214.64537048339844, + "logps/rejected": -210.29678344726562, + "loss": 0.6691, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.19515390694141388, + "rewards/margins": 0.06168302148580551, + "rewards/rejected": -0.2568369209766388, + "step": 105 + }, + { + "epoch": 0.14, + "learning_rate": 4.999992187872167e-05, + "logits/chosen": -2.6358773708343506, + "logits/rejected": -2.672025680541992, + "logps/chosen": -235.18109130859375, + "logps/rejected": -233.78045654296875, + "loss": 0.7203, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.25453630089759827, + "rewards/margins": -0.03941688686609268, + "rewards/rejected": -0.215119406580925, + "step": 106 + }, + { + "epoch": 0.14, + "learning_rate": 4.9999893668280043e-05, + "logits/chosen": -2.305128812789917, + "logits/rejected": -2.384552478790283, + "logps/chosen": -168.1253662109375, + "logps/rejected": -210.5357666015625, + "loss": 0.6558, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.30615508556365967, + "rewards/margins": 0.10809167474508286, + "rewards/rejected": -0.4142467677593231, + "step": 107 + }, + { + "epoch": 0.14, + "learning_rate": 4.999986111778367e-05, + "logits/chosen": -2.4207632541656494, + "logits/rejected": -2.4426565170288086, + "logps/chosen": -187.8740234375, + "logps/rejected": -188.6339569091797, + "loss": 0.6383, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.2796909809112549, + "rewards/margins": 0.13547499477863312, + "rewards/rejected": -0.4151660203933716, + "step": 108 + }, + { + "epoch": 0.14, + "learning_rate": 4.999982422723818e-05, + "logits/chosen": -2.505187749862671, + "logits/rejected": -2.4738471508026123, + "logps/chosen": -187.51710510253906, + "logps/rejected": -227.13916015625, + "loss": 0.7216, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.5267550945281982, + "rewards/margins": -0.03421187028288841, + "rewards/rejected": -0.4925432801246643, + "step": 109 + }, + { + "epoch": 0.14, + "learning_rate": 4.9999782996649994e-05, + "logits/chosen": -2.4367451667785645, + "logits/rejected": -2.4629135131835938, + "logps/chosen": -221.5149688720703, + "logps/rejected": -255.66635131835938, + "loss": 0.6923, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4186938405036926, + "rewards/margins": 0.03650692105293274, + "rewards/rejected": -0.45520079135894775, + "step": 110 + }, + { + "epoch": 0.15, + "learning_rate": 4.999973742602626e-05, + "logits/chosen": -2.6166908740997314, + "logits/rejected": -2.638493061065674, + "logps/chosen": -212.32894897460938, + "logps/rejected": -216.49720764160156, + "loss": 0.6789, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.47340863943099976, + "rewards/margins": 0.059217117726802826, + "rewards/rejected": -0.5326257944107056, + "step": 111 + }, + { + "epoch": 0.15, + "learning_rate": 4.999968751537489e-05, + "logits/chosen": -2.549008846282959, + "logits/rejected": -2.6095943450927734, + "logps/chosen": -205.5695343017578, + "logps/rejected": -196.99383544921875, + "loss": 0.7338, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.47608208656311035, + "rewards/margins": -0.05748309940099716, + "rewards/rejected": -0.4185989201068878, + "step": 112 + }, + { + "epoch": 0.15, + "learning_rate": 4.9999633264704564e-05, + "logits/chosen": -2.2947440147399902, + "logits/rejected": -2.240459680557251, + "logps/chosen": -203.31182861328125, + "logps/rejected": -175.49354553222656, + "loss": 0.7139, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.39874717593193054, + "rewards/margins": -0.014322709292173386, + "rewards/rejected": -0.38442447781562805, + "step": 113 + }, + { + "epoch": 0.15, + "learning_rate": 4.999957467402468e-05, + "logits/chosen": -2.118520736694336, + "logits/rejected": -2.0125582218170166, + "logps/chosen": -159.09979248046875, + "logps/rejected": -151.80679321289062, + "loss": 0.6981, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.43373599648475647, + "rewards/margins": 0.0064653148874640465, + "rewards/rejected": -0.4402012825012207, + "step": 114 + }, + { + "epoch": 0.15, + "learning_rate": 4.9999511743345426e-05, + "logits/chosen": -2.251662254333496, + "logits/rejected": -2.343283176422119, + "logps/chosen": -166.46377563476562, + "logps/rejected": -213.02813720703125, + "loss": 0.6281, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.44466906785964966, + "rewards/margins": 0.18120086193084717, + "rewards/rejected": -0.625869870185852, + "step": 115 + }, + { + "epoch": 0.15, + "learning_rate": 4.999944447267771e-05, + "logits/chosen": -2.347130298614502, + "logits/rejected": -2.342409610748291, + "logps/chosen": -216.30650329589844, + "logps/rejected": -210.64273071289062, + "loss": 0.7966, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6098219752311707, + "rewards/margins": -0.12615808844566345, + "rewards/rejected": -0.4836638867855072, + "step": 116 + }, + { + "epoch": 0.15, + "learning_rate": 4.999937286203322e-05, + "logits/chosen": -2.3265388011932373, + "logits/rejected": -2.3534939289093018, + "logps/chosen": -204.47706604003906, + "logps/rejected": -186.42286682128906, + "loss": 0.6674, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.291274756193161, + "rewards/margins": 0.0668107196688652, + "rewards/rejected": -0.3580854535102844, + "step": 117 + }, + { + "epoch": 0.15, + "learning_rate": 4.999929691142439e-05, + "logits/chosen": -2.062469720840454, + "logits/rejected": -2.2004528045654297, + "logps/chosen": -144.79664611816406, + "logps/rejected": -149.7763671875, + "loss": 0.6225, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.46431243419647217, + "rewards/margins": 0.1997339278459549, + "rewards/rejected": -0.6640464067459106, + "step": 118 + }, + { + "epoch": 0.16, + "learning_rate": 4.99992166208644e-05, + "logits/chosen": -2.2899487018585205, + "logits/rejected": -2.2257750034332275, + "logps/chosen": -208.61517333984375, + "logps/rejected": -200.343505859375, + "loss": 0.7233, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5118069052696228, + "rewards/margins": -0.018164699897170067, + "rewards/rejected": -0.4936422109603882, + "step": 119 + }, + { + "epoch": 0.16, + "learning_rate": 4.999913199036719e-05, + "logits/chosen": -2.520472288131714, + "logits/rejected": -2.612544298171997, + "logps/chosen": -160.17779541015625, + "logps/rejected": -191.498291015625, + "loss": 0.6421, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4993933439254761, + "rewards/margins": 0.15375012159347534, + "rewards/rejected": -0.6531434059143066, + "step": 120 + }, + { + "epoch": 0.16, + "learning_rate": 4.9999043019947454e-05, + "logits/chosen": -2.5590789318084717, + "logits/rejected": -2.594666004180908, + "logps/chosen": -219.09494018554688, + "logps/rejected": -271.0511474609375, + "loss": 0.6746, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.37969574332237244, + "rewards/margins": 0.06747917830944061, + "rewards/rejected": -0.44717487692832947, + "step": 121 + }, + { + "epoch": 0.16, + "learning_rate": 4.9998949709620636e-05, + "logits/chosen": -2.292649269104004, + "logits/rejected": -2.4197192192077637, + "logps/chosen": -156.77658081054688, + "logps/rejected": -181.56752014160156, + "loss": 0.6224, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.24466285109519958, + "rewards/margins": 0.17808005213737488, + "rewards/rejected": -0.4227428734302521, + "step": 122 + }, + { + "epoch": 0.16, + "learning_rate": 4.999885205940293e-05, + "logits/chosen": -2.526571750640869, + "logits/rejected": -2.6078643798828125, + "logps/chosen": -188.48431396484375, + "logps/rejected": -209.88192749023438, + "loss": 0.6138, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.38001808524131775, + "rewards/margins": 0.21444422006607056, + "rewards/rejected": -0.5944622755050659, + "step": 123 + }, + { + "epoch": 0.16, + "learning_rate": 4.9998750069311306e-05, + "logits/chosen": -2.477735757827759, + "logits/rejected": -2.524329900741577, + "logps/chosen": -170.30780029296875, + "logps/rejected": -167.05538940429688, + "loss": 0.7105, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.45474332571029663, + "rewards/margins": -0.0021874159574508667, + "rewards/rejected": -0.45255595445632935, + "step": 124 + }, + { + "epoch": 0.16, + "learning_rate": 4.999864373936345e-05, + "logits/chosen": -2.486284017562866, + "logits/rejected": -2.511571168899536, + "logps/chosen": -213.28150939941406, + "logps/rejected": -220.744140625, + "loss": 0.7154, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5180980563163757, + "rewards/margins": -0.013586281798779964, + "rewards/rejected": -0.5045117139816284, + "step": 125 + }, + { + "epoch": 0.16, + "learning_rate": 4.999853306957783e-05, + "logits/chosen": -2.149216651916504, + "logits/rejected": -2.331148862838745, + "logps/chosen": -212.50167846679688, + "logps/rejected": -229.76211547851562, + "loss": 0.6783, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.45535096526145935, + "rewards/margins": 0.13961246609687805, + "rewards/rejected": -0.5949634909629822, + "step": 126 + }, + { + "epoch": 0.17, + "learning_rate": 4.9998418059973654e-05, + "logits/chosen": -2.3633456230163574, + "logits/rejected": -2.3958113193511963, + "logps/chosen": -260.77191162109375, + "logps/rejected": -269.7747497558594, + "loss": 0.624, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.3385522663593292, + "rewards/margins": 0.154096320271492, + "rewards/rejected": -0.4926486015319824, + "step": 127 + }, + { + "epoch": 0.17, + "learning_rate": 4.99982987105709e-05, + "logits/chosen": -2.4027421474456787, + "logits/rejected": -2.47273588180542, + "logps/chosen": -190.19549560546875, + "logps/rejected": -199.79869079589844, + "loss": 0.647, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.45771288871765137, + "rewards/margins": 0.16400352120399475, + "rewards/rejected": -0.6217163801193237, + "step": 128 + }, + { + "epoch": 0.17, + "learning_rate": 4.999817502139027e-05, + "logits/chosen": -2.459967851638794, + "logits/rejected": -2.5030205249786377, + "logps/chosen": -152.8588104248047, + "logps/rejected": -151.28167724609375, + "loss": 0.682, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4902128279209137, + "rewards/margins": 0.07620880752801895, + "rewards/rejected": -0.566421627998352, + "step": 129 + }, + { + "epoch": 0.17, + "learning_rate": 4.999804699245325e-05, + "logits/chosen": -2.613358497619629, + "logits/rejected": -2.6325273513793945, + "logps/chosen": -240.51898193359375, + "logps/rejected": -209.2674560546875, + "loss": 0.6862, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5984245538711548, + "rewards/margins": 0.08296503126621246, + "rewards/rejected": -0.6813895106315613, + "step": 130 + }, + { + "epoch": 0.17, + "learning_rate": 4.999791462378206e-05, + "logits/chosen": -2.523002862930298, + "logits/rejected": -2.544940948486328, + "logps/chosen": -209.53164672851562, + "logps/rejected": -211.6482391357422, + "loss": 0.6617, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4731237292289734, + "rewards/margins": 0.139340341091156, + "rewards/rejected": -0.6124641299247742, + "step": 131 + }, + { + "epoch": 0.17, + "learning_rate": 4.999777791539968e-05, + "logits/chosen": -2.3467066287994385, + "logits/rejected": -2.4002304077148438, + "logps/chosen": -229.88079833984375, + "logps/rejected": -236.3416290283203, + "loss": 0.6545, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5287673473358154, + "rewards/margins": 0.11849495768547058, + "rewards/rejected": -0.6472623348236084, + "step": 132 + }, + { + "epoch": 0.17, + "learning_rate": 4.9997636867329844e-05, + "logits/chosen": -2.3951451778411865, + "logits/rejected": -2.3899013996124268, + "logps/chosen": -230.42471313476562, + "logps/rejected": -204.70712280273438, + "loss": 0.6886, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7095022797584534, + "rewards/margins": 0.06169421225786209, + "rewards/rejected": -0.7711963653564453, + "step": 133 + }, + { + "epoch": 0.18, + "learning_rate": 4.999749147959703e-05, + "logits/chosen": -2.4759864807128906, + "logits/rejected": -2.461402177810669, + "logps/chosen": -208.62255859375, + "logps/rejected": -215.98049926757812, + "loss": 0.6288, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.47989076375961304, + "rewards/margins": 0.18254442512989044, + "rewards/rejected": -0.6624351143836975, + "step": 134 + }, + { + "epoch": 0.18, + "learning_rate": 4.99973417522265e-05, + "logits/chosen": -2.6637802124023438, + "logits/rejected": -2.6086812019348145, + "logps/chosen": -201.55992126464844, + "logps/rejected": -185.3202667236328, + "loss": 0.7158, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5636375546455383, + "rewards/margins": 0.09092222899198532, + "rewards/rejected": -0.6545597910881042, + "step": 135 + }, + { + "epoch": 0.18, + "learning_rate": 4.9997187685244234e-05, + "logits/chosen": -2.386868953704834, + "logits/rejected": -2.438833475112915, + "logps/chosen": -149.93809509277344, + "logps/rejected": -164.56199645996094, + "loss": 0.7271, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.43254008889198303, + "rewards/margins": -0.0416962131857872, + "rewards/rejected": -0.39084386825561523, + "step": 136 + }, + { + "epoch": 0.18, + "learning_rate": 4.999702927867698e-05, + "logits/chosen": -2.382352113723755, + "logits/rejected": -2.447472095489502, + "logps/chosen": -195.46499633789062, + "logps/rejected": -191.3668212890625, + "loss": 0.7562, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5873669385910034, + "rewards/margins": -0.07318583130836487, + "rewards/rejected": -0.5141811370849609, + "step": 137 + }, + { + "epoch": 0.18, + "learning_rate": 4.999686653255222e-05, + "logits/chosen": -2.6838254928588867, + "logits/rejected": -2.735358476638794, + "logps/chosen": -232.84722900390625, + "logps/rejected": -253.7335205078125, + "loss": 0.5769, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5806477069854736, + "rewards/margins": 0.33124226331710815, + "rewards/rejected": -0.9118900299072266, + "step": 138 + }, + { + "epoch": 0.18, + "learning_rate": 4.9996699446898235e-05, + "logits/chosen": -2.3523752689361572, + "logits/rejected": -2.455883026123047, + "logps/chosen": -158.80831909179688, + "logps/rejected": -215.85203552246094, + "loss": 0.7447, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5600742697715759, + "rewards/margins": 0.03419441357254982, + "rewards/rejected": -0.5942687392234802, + "step": 139 + }, + { + "epoch": 0.18, + "learning_rate": 4.999652802174402e-05, + "logits/chosen": -2.5024056434631348, + "logits/rejected": -2.4992480278015137, + "logps/chosen": -233.17120361328125, + "logps/rejected": -249.13568115234375, + "loss": 0.6822, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7229613065719604, + "rewards/margins": 0.06977183371782303, + "rewards/rejected": -0.7927330732345581, + "step": 140 + }, + { + "epoch": 0.18, + "learning_rate": 4.999635225711933e-05, + "logits/chosen": -2.3752949237823486, + "logits/rejected": -2.499027729034424, + "logps/chosen": -165.4285888671875, + "logps/rejected": -213.1377716064453, + "loss": 0.6571, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4794481098651886, + "rewards/margins": 0.1226089596748352, + "rewards/rejected": -0.602057158946991, + "step": 141 + }, + { + "epoch": 0.19, + "learning_rate": 4.999617215305468e-05, + "logits/chosen": -2.578481674194336, + "logits/rejected": -2.5905745029449463, + "logps/chosen": -160.28704833984375, + "logps/rejected": -186.60206604003906, + "loss": 0.5771, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.379621684551239, + "rewards/margins": 0.28185874223709106, + "rewards/rejected": -0.6614804863929749, + "step": 142 + }, + { + "epoch": 0.19, + "learning_rate": 4.999598770958134e-05, + "logits/chosen": -2.1814939975738525, + "logits/rejected": -2.273460865020752, + "logps/chosen": -178.5402069091797, + "logps/rejected": -246.2064208984375, + "loss": 0.6741, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.47223299741744995, + "rewards/margins": 0.11296035349369049, + "rewards/rejected": -0.5851933360099792, + "step": 143 + }, + { + "epoch": 0.19, + "learning_rate": 4.999579892673133e-05, + "logits/chosen": -2.6444742679595947, + "logits/rejected": -2.651982545852661, + "logps/chosen": -191.74351501464844, + "logps/rejected": -191.7657928466797, + "loss": 0.6952, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6643162965774536, + "rewards/margins": 0.027530232444405556, + "rewards/rejected": -0.691846489906311, + "step": 144 + }, + { + "epoch": 0.19, + "learning_rate": 4.9995605804537426e-05, + "logits/chosen": -2.4908974170684814, + "logits/rejected": -2.550518274307251, + "logps/chosen": -188.7380828857422, + "logps/rejected": -211.574951171875, + "loss": 0.5798, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.3170633912086487, + "rewards/margins": 0.3337726294994354, + "rewards/rejected": -0.6508359313011169, + "step": 145 + }, + { + "epoch": 0.19, + "learning_rate": 4.999540834303315e-05, + "logits/chosen": -2.403130054473877, + "logits/rejected": -2.5044126510620117, + "logps/chosen": -216.05221557617188, + "logps/rejected": -224.658203125, + "loss": 0.5719, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5969533920288086, + "rewards/margins": 0.4197196662425995, + "rewards/rejected": -1.0166730880737305, + "step": 146 + }, + { + "epoch": 0.19, + "learning_rate": 4.999520654225278e-05, + "logits/chosen": -2.5251312255859375, + "logits/rejected": -2.6076760292053223, + "logps/chosen": -171.32049560546875, + "logps/rejected": -189.77108764648438, + "loss": 0.5929, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5219372510910034, + "rewards/margins": 0.3483983874320984, + "rewards/rejected": -0.8703356385231018, + "step": 147 + }, + { + "epoch": 0.19, + "learning_rate": 4.9995000402231354e-05, + "logits/chosen": -2.564474105834961, + "logits/rejected": -2.510350227355957, + "logps/chosen": -219.81732177734375, + "logps/rejected": -202.83493041992188, + "loss": 0.7793, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.7294387817382812, + "rewards/margins": -0.11600115895271301, + "rewards/rejected": -0.6134375333786011, + "step": 148 + }, + { + "epoch": 0.19, + "learning_rate": 4.999478992300466e-05, + "logits/chosen": -2.5275182723999023, + "logits/rejected": -2.6380527019500732, + "logps/chosen": -165.94358825683594, + "logps/rejected": -212.28086853027344, + "loss": 0.565, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4685615599155426, + "rewards/margins": 0.44902655482292175, + "rewards/rejected": -0.9175881147384644, + "step": 149 + }, + { + "epoch": 0.2, + "learning_rate": 4.999457510460923e-05, + "logits/chosen": -2.407663106918335, + "logits/rejected": -2.5699005126953125, + "logps/chosen": -167.9168701171875, + "logps/rejected": -214.4752197265625, + "loss": 0.5801, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.48059117794036865, + "rewards/margins": 0.3546959161758423, + "rewards/rejected": -0.8352870345115662, + "step": 150 + }, + { + "epoch": 0.2, + "learning_rate": 4.999435594708236e-05, + "logits/chosen": -2.3764097690582275, + "logits/rejected": -2.3675248622894287, + "logps/chosen": -201.25814819335938, + "logps/rejected": -229.6868133544922, + "loss": 0.6714, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6616516709327698, + "rewards/margins": 0.10846880823373795, + "rewards/rejected": -0.7701205611228943, + "step": 151 + }, + { + "epoch": 0.2, + "learning_rate": 4.999413245046211e-05, + "logits/chosen": -2.5667271614074707, + "logits/rejected": -2.5309338569641113, + "logps/chosen": -210.30267333984375, + "logps/rejected": -206.71014404296875, + "loss": 0.5848, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.572422444820404, + "rewards/margins": 0.30542662739753723, + "rewards/rejected": -0.8778490424156189, + "step": 152 + }, + { + "epoch": 0.2, + "learning_rate": 4.9993904614787254e-05, + "logits/chosen": -2.463524341583252, + "logits/rejected": -2.530627965927124, + "logps/chosen": -179.8057861328125, + "logps/rejected": -200.56394958496094, + "loss": 0.6231, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6925724148750305, + "rewards/margins": 0.21750670671463013, + "rewards/rejected": -0.9100791215896606, + "step": 153 + }, + { + "epoch": 0.2, + "learning_rate": 4.999367244009736e-05, + "logits/chosen": -2.4720635414123535, + "logits/rejected": -2.5370302200317383, + "logps/chosen": -194.29440307617188, + "logps/rejected": -196.02944946289062, + "loss": 0.8265, + "rewards/accuracies": 0.4375, + "rewards/chosen": -1.0657517910003662, + "rewards/margins": -0.16590073704719543, + "rewards/rejected": -0.8998512029647827, + "step": 154 + }, + { + "epoch": 0.2, + "learning_rate": 4.999343592643274e-05, + "logits/chosen": -2.3978195190429688, + "logits/rejected": -2.397984266281128, + "logps/chosen": -229.4518585205078, + "logps/rejected": -217.56869506835938, + "loss": 0.6598, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6397801637649536, + "rewards/margins": 0.10495337843894958, + "rewards/rejected": -0.7447335720062256, + "step": 155 + }, + { + "epoch": 0.2, + "learning_rate": 4.999319507383444e-05, + "logits/chosen": -2.3354411125183105, + "logits/rejected": -2.4454195499420166, + "logps/chosen": -149.3455810546875, + "logps/rejected": -187.3743438720703, + "loss": 0.5981, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7036041617393494, + "rewards/margins": 0.3835078775882721, + "rewards/rejected": -1.0871120691299438, + "step": 156 + }, + { + "epoch": 0.21, + "learning_rate": 4.999294988234428e-05, + "logits/chosen": -2.491671323776245, + "logits/rejected": -2.550612449645996, + "logps/chosen": -180.36758422851562, + "logps/rejected": -251.5354461669922, + "loss": 0.6038, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7723766565322876, + "rewards/margins": 0.2752906382083893, + "rewards/rejected": -1.047667384147644, + "step": 157 + }, + { + "epoch": 0.21, + "learning_rate": 4.999270035200483e-05, + "logits/chosen": -2.492790937423706, + "logits/rejected": -2.365570306777954, + "logps/chosen": -195.1597900390625, + "logps/rejected": -185.96673583984375, + "loss": 0.7169, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2230509519577026, + "rewards/margins": 0.14210942387580872, + "rewards/rejected": -1.3651604652404785, + "step": 158 + }, + { + "epoch": 0.21, + "learning_rate": 4.99924464828594e-05, + "logits/chosen": -2.412907838821411, + "logits/rejected": -2.508164882659912, + "logps/chosen": -196.51846313476562, + "logps/rejected": -218.0734405517578, + "loss": 0.5942, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.813502848148346, + "rewards/margins": 0.3059760332107544, + "rewards/rejected": -1.1194789409637451, + "step": 159 + }, + { + "epoch": 0.21, + "learning_rate": 4.9992188274952064e-05, + "logits/chosen": -2.4737277030944824, + "logits/rejected": -2.5720887184143066, + "logps/chosen": -155.16171264648438, + "logps/rejected": -195.86997985839844, + "loss": 0.7379, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.8073078989982605, + "rewards/margins": 0.11351939290761948, + "rewards/rejected": -0.920827329158783, + "step": 160 + }, + { + "epoch": 0.21, + "learning_rate": 4.999192572832765e-05, + "logits/chosen": -2.5303239822387695, + "logits/rejected": -2.5628015995025635, + "logps/chosen": -281.7694396972656, + "logps/rejected": -303.6089782714844, + "loss": 0.6735, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6562893986701965, + "rewards/margins": 0.12363787740468979, + "rewards/rejected": -0.7799273133277893, + "step": 161 + }, + { + "epoch": 0.21, + "learning_rate": 4.999165884303174e-05, + "logits/chosen": -2.481058359146118, + "logits/rejected": -2.4526753425598145, + "logps/chosen": -182.2716064453125, + "logps/rejected": -242.58828735351562, + "loss": 0.7317, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7687569260597229, + "rewards/margins": -0.007903970777988434, + "rewards/rejected": -0.7608529329299927, + "step": 162 + }, + { + "epoch": 0.21, + "learning_rate": 4.999138761911066e-05, + "logits/chosen": -2.459987163543701, + "logits/rejected": -2.5180883407592773, + "logps/chosen": -209.83102416992188, + "logps/rejected": -216.1147918701172, + "loss": 0.7265, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.0292317867279053, + "rewards/margins": 0.04909829795360565, + "rewards/rejected": -1.0783300399780273, + "step": 163 + }, + { + "epoch": 0.21, + "learning_rate": 4.99911120566115e-05, + "logits/chosen": -2.5340957641601562, + "logits/rejected": -2.654463052749634, + "logps/chosen": -210.88929748535156, + "logps/rejected": -198.64614868164062, + "loss": 0.7475, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2450902462005615, + "rewards/margins": 0.012418277561664581, + "rewards/rejected": -1.2575085163116455, + "step": 164 + }, + { + "epoch": 0.22, + "learning_rate": 4.99908321555821e-05, + "logits/chosen": -2.4791364669799805, + "logits/rejected": -2.5890302658081055, + "logps/chosen": -192.5817108154297, + "logps/rejected": -224.9603271484375, + "loss": 0.6089, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.9010199308395386, + "rewards/margins": 0.4015497863292694, + "rewards/rejected": -1.3025696277618408, + "step": 165 + }, + { + "epoch": 0.22, + "learning_rate": 4.999054791607105e-05, + "logits/chosen": -2.428842067718506, + "logits/rejected": -2.372598171234131, + "logps/chosen": -221.89833068847656, + "logps/rejected": -212.29595947265625, + "loss": 0.7305, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.0515369176864624, + "rewards/margins": 0.017772257328033447, + "rewards/rejected": -1.0693092346191406, + "step": 166 + }, + { + "epoch": 0.22, + "learning_rate": 4.999025933812769e-05, + "logits/chosen": -2.50471830368042, + "logits/rejected": -2.5308945178985596, + "logps/chosen": -198.7537384033203, + "logps/rejected": -236.5478515625, + "loss": 0.6487, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.8205583095550537, + "rewards/margins": 0.18831993639469147, + "rewards/rejected": -1.008878231048584, + "step": 167 + }, + { + "epoch": 0.22, + "learning_rate": 4.9989966421802114e-05, + "logits/chosen": -2.410905122756958, + "logits/rejected": -2.5401611328125, + "logps/chosen": -202.42991638183594, + "logps/rejected": -219.6848602294922, + "loss": 0.6257, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.9180124402046204, + "rewards/margins": 0.24881067872047424, + "rewards/rejected": -1.166823148727417, + "step": 168 + }, + { + "epoch": 0.22, + "learning_rate": 4.998966916714519e-05, + "logits/chosen": -2.4587323665618896, + "logits/rejected": -2.5724194049835205, + "logps/chosen": -154.88470458984375, + "logps/rejected": -172.47268676757812, + "loss": 0.618, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7627564072608948, + "rewards/margins": 0.294120192527771, + "rewards/rejected": -1.0568766593933105, + "step": 169 + }, + { + "epoch": 0.22, + "learning_rate": 4.998936757420851e-05, + "logits/chosen": -2.284227132797241, + "logits/rejected": -2.380892515182495, + "logps/chosen": -138.282958984375, + "logps/rejected": -180.36441040039062, + "loss": 0.6089, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7448742389678955, + "rewards/margins": 0.4253446161746979, + "rewards/rejected": -1.1702189445495605, + "step": 170 + }, + { + "epoch": 0.22, + "learning_rate": 4.9989061643044434e-05, + "logits/chosen": -2.5783276557922363, + "logits/rejected": -2.5471560955047607, + "logps/chosen": -186.851318359375, + "logps/rejected": -193.73837280273438, + "loss": 0.7108, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.0091472864151, + "rewards/margins": 0.033400412648916245, + "rewards/rejected": -1.0425477027893066, + "step": 171 + }, + { + "epoch": 0.23, + "learning_rate": 4.9988751373706075e-05, + "logits/chosen": -2.398940086364746, + "logits/rejected": -2.3498027324676514, + "logps/chosen": -171.69107055664062, + "logps/rejected": -209.19424438476562, + "loss": 0.7045, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.9662235975265503, + "rewards/margins": 0.137020543217659, + "rewards/rejected": -1.1032441854476929, + "step": 172 + }, + { + "epoch": 0.23, + "learning_rate": 4.9988436766247284e-05, + "logits/chosen": -2.415847063064575, + "logits/rejected": -2.3855319023132324, + "logps/chosen": -228.1761932373047, + "logps/rejected": -195.01991271972656, + "loss": 0.6696, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.9869641661643982, + "rewards/margins": 0.11401443928480148, + "rewards/rejected": -1.1009787321090698, + "step": 173 + }, + { + "epoch": 0.23, + "learning_rate": 4.9988117820722704e-05, + "logits/chosen": -2.638780117034912, + "logits/rejected": -2.607104539871216, + "logps/chosen": -234.91183471679688, + "logps/rejected": -203.67996215820312, + "loss": 0.9279, + "rewards/accuracies": 0.375, + "rewards/chosen": -1.2375922203063965, + "rewards/margins": -0.281325101852417, + "rewards/rejected": -0.956267237663269, + "step": 174 + }, + { + "epoch": 0.23, + "learning_rate": 4.998779453718768e-05, + "logits/chosen": -2.254124879837036, + "logits/rejected": -2.429471969604492, + "logps/chosen": -235.76698303222656, + "logps/rejected": -262.7959899902344, + "loss": 0.6238, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.0454847812652588, + "rewards/margins": 0.3030804395675659, + "rewards/rejected": -1.3485652208328247, + "step": 175 + }, + { + "epoch": 0.23, + "learning_rate": 4.9987466915698346e-05, + "logits/chosen": -2.276986598968506, + "logits/rejected": -2.337780475616455, + "logps/chosen": -138.3777313232422, + "logps/rejected": -166.30181884765625, + "loss": 0.5682, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5320202708244324, + "rewards/margins": 0.315082311630249, + "rewards/rejected": -0.8471025228500366, + "step": 176 + }, + { + "epoch": 0.23, + "learning_rate": 4.998713495631156e-05, + "logits/chosen": -2.420923948287964, + "logits/rejected": -2.565662384033203, + "logps/chosen": -180.71446228027344, + "logps/rejected": -192.2035369873047, + "loss": 0.647, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.8985862731933594, + "rewards/margins": 0.2061445415019989, + "rewards/rejected": -1.1047309637069702, + "step": 177 + }, + { + "epoch": 0.23, + "learning_rate": 4.998679865908499e-05, + "logits/chosen": -2.5881500244140625, + "logits/rejected": -2.6503522396087646, + "logps/chosen": -183.07540893554688, + "logps/rejected": -233.943603515625, + "loss": 0.6889, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.1374621391296387, + "rewards/margins": 0.19141921401023865, + "rewards/rejected": -1.3288813829421997, + "step": 178 + }, + { + "epoch": 0.23, + "learning_rate": 4.9986458024076984e-05, + "logits/chosen": -2.4952712059020996, + "logits/rejected": -2.50299072265625, + "logps/chosen": -192.31314086914062, + "logps/rejected": -197.241455078125, + "loss": 0.7729, + "rewards/accuracies": 0.4375, + "rewards/chosen": -1.13700532913208, + "rewards/margins": -0.10435198992490768, + "rewards/rejected": -1.0326533317565918, + "step": 179 + }, + { + "epoch": 0.24, + "learning_rate": 4.998611305134669e-05, + "logits/chosen": -2.5952248573303223, + "logits/rejected": -2.600719928741455, + "logps/chosen": -191.4530487060547, + "logps/rejected": -212.82955932617188, + "loss": 0.674, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.9059873819351196, + "rewards/margins": 0.1599172055721283, + "rewards/rejected": -1.0659046173095703, + "step": 180 + }, + { + "epoch": 0.24, + "learning_rate": 4.9985763740954e-05, + "logits/chosen": -2.4821391105651855, + "logits/rejected": -2.4574878215789795, + "logps/chosen": -264.071533203125, + "logps/rejected": -256.51611328125, + "loss": 0.7333, + "rewards/accuracies": 0.375, + "rewards/chosen": -1.0796548128128052, + "rewards/margins": 0.06612614542245865, + "rewards/rejected": -1.1457810401916504, + "step": 181 + }, + { + "epoch": 0.24, + "learning_rate": 4.9985410092959553e-05, + "logits/chosen": -2.4601786136627197, + "logits/rejected": -2.519011974334717, + "logps/chosen": -158.57687377929688, + "logps/rejected": -173.29666137695312, + "loss": 0.7693, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.7817510366439819, + "rewards/margins": -0.05254516005516052, + "rewards/rejected": -0.729205846786499, + "step": 182 + }, + { + "epoch": 0.24, + "learning_rate": 4.998505210742472e-05, + "logits/chosen": -2.6099700927734375, + "logits/rejected": -2.58197021484375, + "logps/chosen": -195.2617950439453, + "logps/rejected": -188.54165649414062, + "loss": 0.9197, + "rewards/accuracies": 0.25, + "rewards/chosen": -1.1366631984710693, + "rewards/margins": -0.32979804277420044, + "rewards/rejected": -0.8068650960922241, + "step": 183 + }, + { + "epoch": 0.24, + "learning_rate": 4.9984689784411686e-05, + "logits/chosen": -2.679287910461426, + "logits/rejected": -2.5679209232330322, + "logps/chosen": -211.75323486328125, + "logps/rejected": -245.02603149414062, + "loss": 0.6702, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.9691303372383118, + "rewards/margins": 0.17891569435596466, + "rewards/rejected": -1.1480460166931152, + "step": 184 + }, + { + "epoch": 0.24, + "learning_rate": 4.9984323123983334e-05, + "logits/chosen": -2.537733316421509, + "logits/rejected": -2.6055424213409424, + "logps/chosen": -151.74288940429688, + "logps/rejected": -250.8179168701172, + "loss": 0.6366, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.928146243095398, + "rewards/margins": 0.24633805453777313, + "rewards/rejected": -1.174484372138977, + "step": 185 + }, + { + "epoch": 0.24, + "learning_rate": 4.998395212620332e-05, + "logits/chosen": -2.4368271827697754, + "logits/rejected": -2.529536247253418, + "logps/chosen": -207.87677001953125, + "logps/rejected": -229.0767822265625, + "loss": 0.6199, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7506311535835266, + "rewards/margins": 0.24917976558208466, + "rewards/rejected": -0.9998109340667725, + "step": 186 + }, + { + "epoch": 0.24, + "learning_rate": 4.998357679113603e-05, + "logits/chosen": -2.5217807292938232, + "logits/rejected": -2.6954245567321777, + "logps/chosen": -192.20407104492188, + "logps/rejected": -200.1387176513672, + "loss": 0.6013, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6829215288162231, + "rewards/margins": 0.2588457763195038, + "rewards/rejected": -0.9417673349380493, + "step": 187 + }, + { + "epoch": 0.25, + "learning_rate": 4.9983197118846655e-05, + "logits/chosen": -2.4711546897888184, + "logits/rejected": -2.4839439392089844, + "logps/chosen": -160.9691925048828, + "logps/rejected": -167.21469116210938, + "loss": 0.7301, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7170782685279846, + "rewards/margins": 0.04331670328974724, + "rewards/rejected": -0.7603949904441833, + "step": 188 + }, + { + "epoch": 0.25, + "learning_rate": 4.9982813109401096e-05, + "logits/chosen": -2.3429412841796875, + "logits/rejected": -2.3843770027160645, + "logps/chosen": -221.22274780273438, + "logps/rejected": -246.99005126953125, + "loss": 0.7711, + "rewards/accuracies": 0.375, + "rewards/chosen": -1.114179015159607, + "rewards/margins": -0.07230883836746216, + "rewards/rejected": -1.0418701171875, + "step": 189 + }, + { + "epoch": 0.25, + "learning_rate": 4.998242476286601e-05, + "logits/chosen": -2.7864720821380615, + "logits/rejected": -2.7745487689971924, + "logps/chosen": -194.08201599121094, + "logps/rejected": -226.28591918945312, + "loss": 0.71, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.8893502354621887, + "rewards/margins": 0.06053268164396286, + "rewards/rejected": -0.949882984161377, + "step": 190 + }, + { + "epoch": 0.25, + "learning_rate": 4.998203207930882e-05, + "logits/chosen": -2.6679065227508545, + "logits/rejected": -2.711076498031616, + "logps/chosen": -190.53013610839844, + "logps/rejected": -200.187744140625, + "loss": 0.8201, + "rewards/accuracies": 0.4375, + "rewards/chosen": -1.0141409635543823, + "rewards/margins": -0.17862781882286072, + "rewards/rejected": -0.8355131149291992, + "step": 191 + }, + { + "epoch": 0.25, + "learning_rate": 4.998163505879769e-05, + "logits/chosen": -2.7404122352600098, + "logits/rejected": -2.732025146484375, + "logps/chosen": -178.48135375976562, + "logps/rejected": -183.08074951171875, + "loss": 0.7078, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.8886659145355225, + "rewards/margins": 0.043551601469516754, + "rewards/rejected": -0.9322174787521362, + "step": 192 + }, + { + "epoch": 0.25, + "learning_rate": 4.998123370140156e-05, + "logits/chosen": -2.6989643573760986, + "logits/rejected": -2.7124342918395996, + "logps/chosen": -223.03314208984375, + "logps/rejected": -221.12120056152344, + "loss": 0.6559, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.764815628528595, + "rewards/margins": 0.1835068166255951, + "rewards/rejected": -0.9483224153518677, + "step": 193 + }, + { + "epoch": 0.25, + "learning_rate": 4.99808280071901e-05, + "logits/chosen": -2.618727684020996, + "logits/rejected": -2.6078970432281494, + "logps/chosen": -176.1873779296875, + "logps/rejected": -199.52859497070312, + "loss": 0.7406, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.8613646626472473, + "rewards/margins": -0.05548467859625816, + "rewards/rejected": -0.805880069732666, + "step": 194 + }, + { + "epoch": 0.26, + "learning_rate": 4.9980417976233735e-05, + "logits/chosen": -2.6192498207092285, + "logits/rejected": -2.732285261154175, + "logps/chosen": -211.3184814453125, + "logps/rejected": -195.73056030273438, + "loss": 0.7921, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.9030998945236206, + "rewards/margins": -0.1349114179611206, + "rewards/rejected": -0.7681884765625, + "step": 195 + }, + { + "epoch": 0.26, + "learning_rate": 4.9980003608603656e-05, + "logits/chosen": -2.5657694339752197, + "logits/rejected": -2.5230712890625, + "logps/chosen": -203.0819091796875, + "logps/rejected": -226.5617218017578, + "loss": 0.6618, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.609666109085083, + "rewards/margins": 0.08627453446388245, + "rewards/rejected": -0.6959406137466431, + "step": 196 + }, + { + "epoch": 0.26, + "learning_rate": 4.997958490437178e-05, + "logits/chosen": -2.641559362411499, + "logits/rejected": -2.76076602935791, + "logps/chosen": -169.1241455078125, + "logps/rejected": -183.4690399169922, + "loss": 0.7612, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7050959467887878, + "rewards/margins": -0.03217136114835739, + "rewards/rejected": -0.6729245781898499, + "step": 197 + }, + { + "epoch": 0.26, + "learning_rate": 4.9979161863610816e-05, + "logits/chosen": -2.441647529602051, + "logits/rejected": -2.467510223388672, + "logps/chosen": -143.13748168945312, + "logps/rejected": -172.0884246826172, + "loss": 0.6647, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5210601091384888, + "rewards/margins": 0.14004816114902496, + "rewards/rejected": -0.6611082553863525, + "step": 198 + }, + { + "epoch": 0.26, + "learning_rate": 4.99787344863942e-05, + "logits/chosen": -2.627023458480835, + "logits/rejected": -2.5984585285186768, + "logps/chosen": -203.31980895996094, + "logps/rejected": -218.84060668945312, + "loss": 0.6172, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5382375717163086, + "rewards/margins": 0.20881405472755432, + "rewards/rejected": -0.7470515966415405, + "step": 199 + }, + { + "epoch": 0.26, + "learning_rate": 4.997830277279612e-05, + "logits/chosen": -2.595003604888916, + "logits/rejected": -2.722808361053467, + "logps/chosen": -223.01495361328125, + "logps/rejected": -240.90130615234375, + "loss": 0.7333, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7480457425117493, + "rewards/margins": -0.029575012624263763, + "rewards/rejected": -0.7184706926345825, + "step": 200 + }, + { + "epoch": 0.26, + "learning_rate": 4.997786672289152e-05, + "logits/chosen": -2.6220602989196777, + "logits/rejected": -2.6606202125549316, + "logps/chosen": -204.98797607421875, + "logps/rejected": -206.25839233398438, + "loss": 0.7368, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.6979295611381531, + "rewards/margins": -0.07002773880958557, + "rewards/rejected": -0.6279018521308899, + "step": 201 + }, + { + "epoch": 0.26, + "learning_rate": 4.997742633675612e-05, + "logits/chosen": -2.547811508178711, + "logits/rejected": -2.551816463470459, + "logps/chosen": -196.38748168945312, + "logps/rejected": -222.4247589111328, + "loss": 0.6402, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5409201383590698, + "rewards/margins": 0.19667872786521912, + "rewards/rejected": -0.7375988364219666, + "step": 202 + }, + { + "epoch": 0.27, + "learning_rate": 4.9976981614466344e-05, + "logits/chosen": -2.2461767196655273, + "logits/rejected": -2.2968175411224365, + "logps/chosen": -185.43954467773438, + "logps/rejected": -196.11392211914062, + "loss": 0.7743, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.5852402448654175, + "rewards/margins": -0.08380623161792755, + "rewards/rejected": -0.5014340281486511, + "step": 203 + }, + { + "epoch": 0.27, + "learning_rate": 4.997653255609942e-05, + "logits/chosen": -2.4160966873168945, + "logits/rejected": -2.389375925064087, + "logps/chosen": -189.16510009765625, + "logps/rejected": -202.54649353027344, + "loss": 0.7306, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.729293167591095, + "rewards/margins": -0.006243497133255005, + "rewards/rejected": -0.7230496406555176, + "step": 204 + }, + { + "epoch": 0.27, + "learning_rate": 4.997607916173329e-05, + "logits/chosen": -2.6787378787994385, + "logits/rejected": -2.6836445331573486, + "logps/chosen": -177.60980224609375, + "logps/rejected": -177.10487365722656, + "loss": 0.765, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5183064341545105, + "rewards/margins": -0.11038734763860703, + "rewards/rejected": -0.4079190790653229, + "step": 205 + }, + { + "epoch": 0.27, + "learning_rate": 4.997562143144668e-05, + "logits/chosen": -2.559563398361206, + "logits/rejected": -2.5205612182617188, + "logps/chosen": -208.58892822265625, + "logps/rejected": -248.25648498535156, + "loss": 0.6524, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5725008845329285, + "rewards/margins": 0.11152593046426773, + "rewards/rejected": -0.6840267777442932, + "step": 206 + }, + { + "epoch": 0.27, + "learning_rate": 4.997515936531903e-05, + "logits/chosen": -2.6034438610076904, + "logits/rejected": -2.528724193572998, + "logps/chosen": -159.13522338867188, + "logps/rejected": -135.77056884765625, + "loss": 0.7413, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5185827016830444, + "rewards/margins": -0.04235066473484039, + "rewards/rejected": -0.47623202204704285, + "step": 207 + }, + { + "epoch": 0.27, + "learning_rate": 4.9974692963430595e-05, + "logits/chosen": -2.3184618949890137, + "logits/rejected": -2.3163304328918457, + "logps/chosen": -130.70643615722656, + "logps/rejected": -142.0390625, + "loss": 0.7362, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6057229042053223, + "rewards/margins": -0.014669769443571568, + "rewards/rejected": -0.5910531282424927, + "step": 208 + }, + { + "epoch": 0.27, + "learning_rate": 4.99742222258623e-05, + "logits/chosen": -2.660839796066284, + "logits/rejected": -2.6470634937286377, + "logps/chosen": -221.08424377441406, + "logps/rejected": -213.25003051757812, + "loss": 0.6922, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.5536531209945679, + "rewards/margins": 0.022671688348054886, + "rewards/rejected": -0.5763247609138489, + "step": 209 + }, + { + "epoch": 0.27, + "learning_rate": 4.997374715269589e-05, + "logits/chosen": -2.2223334312438965, + "logits/rejected": -2.2852203845977783, + "logps/chosen": -172.71421813964844, + "logps/rejected": -206.3236541748047, + "loss": 0.6722, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5476804375648499, + "rewards/margins": 0.10758630931377411, + "rewards/rejected": -0.6552667617797852, + "step": 210 + }, + { + "epoch": 0.28, + "learning_rate": 4.997326774401383e-05, + "logits/chosen": -2.262892961502075, + "logits/rejected": -2.343332052230835, + "logps/chosen": -152.0110626220703, + "logps/rejected": -178.68345642089844, + "loss": 0.694, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.48357081413269043, + "rewards/margins": 0.07582694292068481, + "rewards/rejected": -0.55939781665802, + "step": 211 + }, + { + "epoch": 0.28, + "learning_rate": 4.9972783999899366e-05, + "logits/chosen": -2.2378625869750977, + "logits/rejected": -2.2833354473114014, + "logps/chosen": -154.71405029296875, + "logps/rejected": -177.74502563476562, + "loss": 0.709, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.5034060478210449, + "rewards/margins": -0.0023919548839330673, + "rewards/rejected": -0.5010141134262085, + "step": 212 + }, + { + "epoch": 0.28, + "learning_rate": 4.997229592043645e-05, + "logits/chosen": -2.4182140827178955, + "logits/rejected": -2.477410078048706, + "logps/chosen": -217.6938018798828, + "logps/rejected": -236.60140991210938, + "loss": 0.7474, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.42065930366516113, + "rewards/margins": -0.08166539669036865, + "rewards/rejected": -0.33899393677711487, + "step": 213 + }, + { + "epoch": 0.28, + "learning_rate": 4.997180350570984e-05, + "logits/chosen": -2.6971006393432617, + "logits/rejected": -2.709580659866333, + "logps/chosen": -231.80714416503906, + "logps/rejected": -234.41290283203125, + "loss": 0.6593, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4388115406036377, + "rewards/margins": 0.1023733913898468, + "rewards/rejected": -0.5411849617958069, + "step": 214 + }, + { + "epoch": 0.28, + "learning_rate": 4.9971306755804995e-05, + "logits/chosen": -2.600799083709717, + "logits/rejected": -2.5639290809631348, + "logps/chosen": -211.75051879882812, + "logps/rejected": -185.4350128173828, + "loss": 0.6654, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.3958936929702759, + "rewards/margins": 0.08459535241127014, + "rewards/rejected": -0.4804890751838684, + "step": 215 + }, + { + "epoch": 0.28, + "learning_rate": 4.997080567080817e-05, + "logits/chosen": -2.2965052127838135, + "logits/rejected": -2.3450510501861572, + "logps/chosen": -170.9476318359375, + "logps/rejected": -174.3068389892578, + "loss": 0.6604, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5262787938117981, + "rewards/margins": 0.09660908579826355, + "rewards/rejected": -0.6228878498077393, + "step": 216 + }, + { + "epoch": 0.28, + "learning_rate": 4.9970300250806346e-05, + "logits/chosen": -2.5405097007751465, + "logits/rejected": -2.5496726036071777, + "logps/chosen": -266.0992736816406, + "logps/rejected": -260.38275146484375, + "loss": 0.7048, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5492247939109802, + "rewards/margins": 0.042401984333992004, + "rewards/rejected": -0.591626763343811, + "step": 217 + }, + { + "epoch": 0.29, + "learning_rate": 4.996979049588727e-05, + "logits/chosen": -2.502474308013916, + "logits/rejected": -2.439131736755371, + "logps/chosen": -227.49476623535156, + "logps/rejected": -206.38299560546875, + "loss": 0.711, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5280985236167908, + "rewards/margins": 0.010388553142547607, + "rewards/rejected": -0.5384870767593384, + "step": 218 + }, + { + "epoch": 0.29, + "learning_rate": 4.996927640613944e-05, + "logits/chosen": -2.3517448902130127, + "logits/rejected": -2.5404696464538574, + "logps/chosen": -162.60267639160156, + "logps/rejected": -216.92562866210938, + "loss": 0.6095, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.2759331464767456, + "rewards/margins": 0.19823111593723297, + "rewards/rejected": -0.474164217710495, + "step": 219 + }, + { + "epoch": 0.29, + "learning_rate": 4.99687579816521e-05, + "logits/chosen": -2.411978006362915, + "logits/rejected": -2.5019097328186035, + "logps/chosen": -201.91445922851562, + "logps/rejected": -211.31448364257812, + "loss": 0.6797, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4115257263183594, + "rewards/margins": 0.07280793786048889, + "rewards/rejected": -0.48433366417884827, + "step": 220 + }, + { + "epoch": 0.29, + "learning_rate": 4.9968235222515246e-05, + "logits/chosen": -2.4315428733825684, + "logits/rejected": -2.4550862312316895, + "logps/chosen": -162.66555786132812, + "logps/rejected": -156.19947814941406, + "loss": 0.6482, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.36304140090942383, + "rewards/margins": 0.12375527620315552, + "rewards/rejected": -0.48679661750793457, + "step": 221 + }, + { + "epoch": 0.29, + "learning_rate": 4.996770812881964e-05, + "logits/chosen": -2.5614113807678223, + "logits/rejected": -2.5391292572021484, + "logps/chosen": -214.3634033203125, + "logps/rejected": -209.9477996826172, + "loss": 0.7018, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.319223552942276, + "rewards/margins": 0.018056731671094894, + "rewards/rejected": -0.3372803032398224, + "step": 222 + }, + { + "epoch": 0.29, + "learning_rate": 4.9967176700656776e-05, + "logits/chosen": -2.438467264175415, + "logits/rejected": -2.4638776779174805, + "logps/chosen": -227.1153564453125, + "logps/rejected": -219.0038299560547, + "loss": 0.6209, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.45765751600265503, + "rewards/margins": 0.19506625831127167, + "rewards/rejected": -0.6527237892150879, + "step": 223 + }, + { + "epoch": 0.29, + "learning_rate": 4.996664093811892e-05, + "logits/chosen": -2.5968563556671143, + "logits/rejected": -2.5226142406463623, + "logps/chosen": -184.9003143310547, + "logps/rejected": -202.8119354248047, + "loss": 0.5973, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.36458614468574524, + "rewards/margins": 0.22884023189544678, + "rewards/rejected": -0.5934264063835144, + "step": 224 + }, + { + "epoch": 0.29, + "learning_rate": 4.996610084129908e-05, + "logits/chosen": -2.356076955795288, + "logits/rejected": -2.464617967605591, + "logps/chosen": -233.56741333007812, + "logps/rejected": -252.82614135742188, + "loss": 0.7174, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.475138783454895, + "rewards/margins": 0.015569500625133514, + "rewards/rejected": -0.49070829153060913, + "step": 225 + }, + { + "epoch": 0.3, + "learning_rate": 4.996555641029101e-05, + "logits/chosen": -2.4807751178741455, + "logits/rejected": -2.5052688121795654, + "logps/chosen": -218.75250244140625, + "logps/rejected": -202.75149536132812, + "loss": 0.7541, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5363876819610596, + "rewards/margins": -0.06761842221021652, + "rewards/rejected": -0.4687691926956177, + "step": 226 + }, + { + "epoch": 0.3, + "learning_rate": 4.996500764518923e-05, + "logits/chosen": -2.5997631549835205, + "logits/rejected": -2.6119892597198486, + "logps/chosen": -212.090087890625, + "logps/rejected": -220.1112823486328, + "loss": 0.7185, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.5663864016532898, + "rewards/margins": -0.016858436167240143, + "rewards/rejected": -0.5495280027389526, + "step": 227 + }, + { + "epoch": 0.3, + "learning_rate": 4.9964454546089026e-05, + "logits/chosen": -2.25441837310791, + "logits/rejected": -2.251708507537842, + "logps/chosen": -216.14996337890625, + "logps/rejected": -218.1949005126953, + "loss": 0.7617, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.46758705377578735, + "rewards/margins": -0.09628176689147949, + "rewards/rejected": -0.37130531668663025, + "step": 228 + }, + { + "epoch": 0.3, + "learning_rate": 4.996389711308639e-05, + "logits/chosen": -2.475829601287842, + "logits/rejected": -2.4804282188415527, + "logps/chosen": -197.95965576171875, + "logps/rejected": -227.29605102539062, + "loss": 0.6275, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.2889898717403412, + "rewards/margins": 0.18446122109889984, + "rewards/rejected": -0.47345107793807983, + "step": 229 + }, + { + "epoch": 0.3, + "learning_rate": 4.996333534627809e-05, + "logits/chosen": -2.4503121376037598, + "logits/rejected": -2.40040922164917, + "logps/chosen": -190.1997833251953, + "logps/rejected": -170.68679809570312, + "loss": 0.7168, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4417100250720978, + "rewards/margins": -0.0062090009450912476, + "rewards/rejected": -0.4355010688304901, + "step": 230 + }, + { + "epoch": 0.3, + "learning_rate": 4.996276924576169e-05, + "logits/chosen": -2.5104312896728516, + "logits/rejected": -2.5510809421539307, + "logps/chosen": -134.1800079345703, + "logps/rejected": -153.13365173339844, + "loss": 0.7468, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.42858752608299255, + "rewards/margins": -0.08554935455322266, + "rewards/rejected": -0.3430381715297699, + "step": 231 + }, + { + "epoch": 0.3, + "learning_rate": 4.996219881163543e-05, + "logits/chosen": -2.456216335296631, + "logits/rejected": -2.4285402297973633, + "logps/chosen": -209.37985229492188, + "logps/rejected": -195.06568908691406, + "loss": 0.7159, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.46927523612976074, + "rewards/margins": -0.0166518222540617, + "rewards/rejected": -0.4526233971118927, + "step": 232 + }, + { + "epoch": 0.3, + "learning_rate": 4.996162404399835e-05, + "logits/chosen": -2.450845956802368, + "logits/rejected": -2.4209864139556885, + "logps/chosen": -198.9377899169922, + "logps/rejected": -203.7089385986328, + "loss": 0.6708, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4800047278404236, + "rewards/margins": 0.09784451127052307, + "rewards/rejected": -0.5778492093086243, + "step": 233 + }, + { + "epoch": 0.31, + "learning_rate": 4.996104494295024e-05, + "logits/chosen": -2.5744130611419678, + "logits/rejected": -2.641494035720825, + "logps/chosen": -182.59878540039062, + "logps/rejected": -207.31797790527344, + "loss": 0.6749, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4756154417991638, + "rewards/margins": 0.08592241257429123, + "rewards/rejected": -0.5615378618240356, + "step": 234 + }, + { + "epoch": 0.31, + "learning_rate": 4.996046150859161e-05, + "logits/chosen": -2.0956687927246094, + "logits/rejected": -2.1018872261047363, + "logps/chosen": -200.86215209960938, + "logps/rejected": -221.92138671875, + "loss": 0.6445, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.488430380821228, + "rewards/margins": 0.1436658799648285, + "rewards/rejected": -0.6320962905883789, + "step": 235 + }, + { + "epoch": 0.31, + "learning_rate": 4.9959873741023774e-05, + "logits/chosen": -2.3680927753448486, + "logits/rejected": -2.415205240249634, + "logps/chosen": -163.829833984375, + "logps/rejected": -182.74481201171875, + "loss": 0.6839, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.35495901107788086, + "rewards/margins": 0.03828255832195282, + "rewards/rejected": -0.3932415544986725, + "step": 236 + }, + { + "epoch": 0.31, + "learning_rate": 4.995928164034876e-05, + "logits/chosen": -2.5983476638793945, + "logits/rejected": -2.7171671390533447, + "logps/chosen": -189.3108673095703, + "logps/rejected": -195.94888305664062, + "loss": 0.6611, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4241011142730713, + "rewards/margins": 0.0877021998167038, + "rewards/rejected": -0.5118032693862915, + "step": 237 + }, + { + "epoch": 0.31, + "learning_rate": 4.995868520666936e-05, + "logits/chosen": -2.510660171508789, + "logits/rejected": -2.5224432945251465, + "logps/chosen": -227.51687622070312, + "logps/rejected": -231.05897521972656, + "loss": 0.7269, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.40695255994796753, + "rewards/margins": -0.046068765223026276, + "rewards/rejected": -0.36088380217552185, + "step": 238 + }, + { + "epoch": 0.31, + "learning_rate": 4.9958084440089095e-05, + "logits/chosen": -2.256051540374756, + "logits/rejected": -2.2608485221862793, + "logps/chosen": -209.19252014160156, + "logps/rejected": -206.2628631591797, + "loss": 0.6338, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3002999424934387, + "rewards/margins": 0.18500341475009918, + "rewards/rejected": -0.4853033423423767, + "step": 239 + }, + { + "epoch": 0.31, + "learning_rate": 4.995747934071229e-05, + "logits/chosen": -2.8373260498046875, + "logits/rejected": -2.8713366985321045, + "logps/chosen": -186.55723571777344, + "logps/rejected": -221.16917419433594, + "loss": 0.6489, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3408931493759155, + "rewards/margins": 0.18887601792812347, + "rewards/rejected": -0.5297691822052002, + "step": 240 + }, + { + "epoch": 0.32, + "learning_rate": 4.995686990864398e-05, + "logits/chosen": -2.328822135925293, + "logits/rejected": -2.4501209259033203, + "logps/chosen": -164.64404296875, + "logps/rejected": -164.53396606445312, + "loss": 0.6872, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5059252977371216, + "rewards/margins": 0.046310462057590485, + "rewards/rejected": -0.5522357225418091, + "step": 241 + }, + { + "epoch": 0.32, + "learning_rate": 4.995625614398996e-05, + "logits/chosen": -2.254148006439209, + "logits/rejected": -2.45564603805542, + "logps/chosen": -215.61080932617188, + "logps/rejected": -236.83526611328125, + "loss": 0.618, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3230094313621521, + "rewards/margins": 0.2154739797115326, + "rewards/rejected": -0.5384833812713623, + "step": 242 + }, + { + "epoch": 0.32, + "learning_rate": 4.995563804685678e-05, + "logits/chosen": -2.4298973083496094, + "logits/rejected": -2.6152379512786865, + "logps/chosen": -171.45925903320312, + "logps/rejected": -215.27342224121094, + "loss": 0.6461, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5956869721412659, + "rewards/margins": 0.1696903109550476, + "rewards/rejected": -0.7653772830963135, + "step": 243 + }, + { + "epoch": 0.32, + "learning_rate": 4.995501561735176e-05, + "logits/chosen": -2.5064899921417236, + "logits/rejected": -2.667478561401367, + "logps/chosen": -201.28167724609375, + "logps/rejected": -200.6697998046875, + "loss": 0.7252, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.50804603099823, + "rewards/margins": -0.021207518875598907, + "rewards/rejected": -0.48683851957321167, + "step": 244 + }, + { + "epoch": 0.32, + "learning_rate": 4.995438885558294e-05, + "logits/chosen": -2.575934886932373, + "logits/rejected": -2.5915310382843018, + "logps/chosen": -241.2596893310547, + "logps/rejected": -236.16864013671875, + "loss": 0.6113, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.41618824005126953, + "rewards/margins": 0.21841469407081604, + "rewards/rejected": -0.6346028447151184, + "step": 245 + }, + { + "epoch": 0.32, + "learning_rate": 4.995375776165913e-05, + "logits/chosen": -2.6145882606506348, + "logits/rejected": -2.6391665935516357, + "logps/chosen": -230.53524780273438, + "logps/rejected": -228.5028076171875, + "loss": 0.7431, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5892909169197083, + "rewards/margins": -0.044729601591825485, + "rewards/rejected": -0.5445613265037537, + "step": 246 + }, + { + "epoch": 0.32, + "learning_rate": 4.995312233568989e-05, + "logits/chosen": -2.279015302658081, + "logits/rejected": -2.4627904891967773, + "logps/chosen": -205.08226013183594, + "logps/rejected": -245.180908203125, + "loss": 0.6492, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.37771591544151306, + "rewards/margins": 0.17377406358718872, + "rewards/rejected": -0.5514900088310242, + "step": 247 + }, + { + "epoch": 0.32, + "learning_rate": 4.9952482577785545e-05, + "logits/chosen": -2.4535770416259766, + "logits/rejected": -2.5108795166015625, + "logps/chosen": -186.94288635253906, + "logps/rejected": -191.04786682128906, + "loss": 0.7335, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6674529314041138, + "rewards/margins": -0.03079444169998169, + "rewards/rejected": -0.6366585493087769, + "step": 248 + }, + { + "epoch": 0.33, + "learning_rate": 4.9951838488057134e-05, + "logits/chosen": -2.2177467346191406, + "logits/rejected": -2.3985226154327393, + "logps/chosen": -151.24977111816406, + "logps/rejected": -185.8676300048828, + "loss": 0.6516, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3900182843208313, + "rewards/margins": 0.13504652678966522, + "rewards/rejected": -0.5250648260116577, + "step": 249 + }, + { + "epoch": 0.33, + "learning_rate": 4.9951190066616495e-05, + "logits/chosen": -2.42427396774292, + "logits/rejected": -2.4583323001861572, + "logps/chosen": -255.53150939941406, + "logps/rejected": -261.932373046875, + "loss": 0.6442, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.3976048231124878, + "rewards/margins": 0.12972235679626465, + "rewards/rejected": -0.5273271799087524, + "step": 250 + }, + { + "epoch": 0.33, + "learning_rate": 4.995053731357618e-05, + "logits/chosen": -2.462376832962036, + "logits/rejected": -2.568171262741089, + "logps/chosen": -197.1337432861328, + "logps/rejected": -202.1964569091797, + "loss": 0.6419, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6016325354576111, + "rewards/margins": 0.15742458403110504, + "rewards/rejected": -0.7590572237968445, + "step": 251 + }, + { + "epoch": 0.33, + "learning_rate": 4.9949880229049526e-05, + "logits/chosen": -2.4696521759033203, + "logits/rejected": -2.4089298248291016, + "logps/chosen": -187.97976684570312, + "logps/rejected": -178.5655059814453, + "loss": 0.7341, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.43453776836395264, + "rewards/margins": -0.025840837508440018, + "rewards/rejected": -0.4086969494819641, + "step": 252 + }, + { + "epoch": 0.33, + "learning_rate": 4.994921881315059e-05, + "logits/chosen": -2.292558431625366, + "logits/rejected": -2.199901580810547, + "logps/chosen": -192.6279754638672, + "logps/rejected": -203.34487915039062, + "loss": 0.6863, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.48495134711265564, + "rewards/margins": 0.07142490148544312, + "rewards/rejected": -0.5563762784004211, + "step": 253 + }, + { + "epoch": 0.33, + "learning_rate": 4.9948553065994197e-05, + "logits/chosen": -2.231633424758911, + "logits/rejected": -2.2734503746032715, + "logps/chosen": -184.4136962890625, + "logps/rejected": -194.4268341064453, + "loss": 0.6465, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.44051364064216614, + "rewards/margins": 0.19739174842834473, + "rewards/rejected": -0.6379053592681885, + "step": 254 + }, + { + "epoch": 0.33, + "learning_rate": 4.994788298769593e-05, + "logits/chosen": -2.5150344371795654, + "logits/rejected": -2.4598844051361084, + "logps/chosen": -238.438232421875, + "logps/rejected": -217.13897705078125, + "loss": 0.6945, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.43908870220184326, + "rewards/margins": 0.03722059726715088, + "rewards/rejected": -0.47630926966667175, + "step": 255 + }, + { + "epoch": 0.34, + "learning_rate": 4.994720857837211e-05, + "logits/chosen": -2.2310400009155273, + "logits/rejected": -2.1882810592651367, + "logps/chosen": -184.02981567382812, + "logps/rejected": -208.0157470703125, + "loss": 0.666, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5550325512886047, + "rewards/margins": 0.08377734571695328, + "rewards/rejected": -0.638809859752655, + "step": 256 + }, + { + "epoch": 0.34, + "learning_rate": 4.994652983813982e-05, + "logits/chosen": -2.4377846717834473, + "logits/rejected": -2.5074000358581543, + "logps/chosen": -187.0503387451172, + "logps/rejected": -195.54049682617188, + "loss": 0.5873, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5530596971511841, + "rewards/margins": 0.2584923803806305, + "rewards/rejected": -0.811552107334137, + "step": 257 + }, + { + "epoch": 0.34, + "learning_rate": 4.994584676711689e-05, + "logits/chosen": -2.464853286743164, + "logits/rejected": -2.504178047180176, + "logps/chosen": -186.5401611328125, + "logps/rejected": -227.3590087890625, + "loss": 0.7102, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5265690088272095, + "rewards/margins": 0.009033482521772385, + "rewards/rejected": -0.5356025695800781, + "step": 258 + }, + { + "epoch": 0.34, + "learning_rate": 4.994515936542191e-05, + "logits/chosen": -2.2838244438171387, + "logits/rejected": -2.3256828784942627, + "logps/chosen": -173.54270935058594, + "logps/rejected": -166.37547302246094, + "loss": 0.7262, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4617432951927185, + "rewards/margins": -0.03625304624438286, + "rewards/rejected": -0.42549026012420654, + "step": 259 + }, + { + "epoch": 0.34, + "learning_rate": 4.99444676331742e-05, + "logits/chosen": -2.407790422439575, + "logits/rejected": -2.3871612548828125, + "logps/chosen": -248.3179931640625, + "logps/rejected": -235.79884338378906, + "loss": 0.6761, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.458143413066864, + "rewards/margins": 0.10666979104280472, + "rewards/rejected": -0.5648132562637329, + "step": 260 + }, + { + "epoch": 0.34, + "learning_rate": 4.9943771570493856e-05, + "logits/chosen": -2.560073137283325, + "logits/rejected": -2.51522159576416, + "logps/chosen": -191.05734252929688, + "logps/rejected": -195.287841796875, + "loss": 0.7288, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6246404051780701, + "rewards/margins": -0.01798657327890396, + "rewards/rejected": -0.6066538095474243, + "step": 261 + }, + { + "epoch": 0.34, + "learning_rate": 4.9943071177501724e-05, + "logits/chosen": -2.364337682723999, + "logits/rejected": -2.39058780670166, + "logps/chosen": -200.97915649414062, + "logps/rejected": -183.07151794433594, + "loss": 0.6839, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5552209615707397, + "rewards/margins": 0.09874990582466125, + "rewards/rejected": -0.6539708375930786, + "step": 262 + }, + { + "epoch": 0.34, + "learning_rate": 4.994236645431938e-05, + "logits/chosen": -2.6555843353271484, + "logits/rejected": -2.7670748233795166, + "logps/chosen": -244.5951690673828, + "logps/rejected": -246.04124450683594, + "loss": 0.688, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.45607852935791016, + "rewards/margins": 0.08103760331869125, + "rewards/rejected": -0.5371161103248596, + "step": 263 + }, + { + "epoch": 0.35, + "learning_rate": 4.994165740106918e-05, + "logits/chosen": -2.527970314025879, + "logits/rejected": -2.6416518688201904, + "logps/chosen": -231.80966186523438, + "logps/rejected": -246.3621826171875, + "loss": 0.6086, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5444769263267517, + "rewards/margins": 0.21878674626350403, + "rewards/rejected": -0.7632635831832886, + "step": 264 + }, + { + "epoch": 0.35, + "learning_rate": 4.99409440178742e-05, + "logits/chosen": -2.584487199783325, + "logits/rejected": -2.5451977252960205, + "logps/chosen": -184.02330017089844, + "logps/rejected": -187.47161865234375, + "loss": 0.7151, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6932749152183533, + "rewards/margins": -0.005857221782207489, + "rewards/rejected": -0.6874176263809204, + "step": 265 + }, + { + "epoch": 0.35, + "learning_rate": 4.9940226304858296e-05, + "logits/chosen": -2.556795597076416, + "logits/rejected": -2.5516929626464844, + "logps/chosen": -187.25588989257812, + "logps/rejected": -206.25067138671875, + "loss": 0.7035, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.563089907169342, + "rewards/margins": 0.009414192289113998, + "rewards/rejected": -0.5725040435791016, + "step": 266 + }, + { + "epoch": 0.35, + "learning_rate": 4.993950426214606e-05, + "logits/chosen": -2.3932271003723145, + "logits/rejected": -2.393653154373169, + "logps/chosen": -218.82334899902344, + "logps/rejected": -226.3861541748047, + "loss": 0.6841, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6887062788009644, + "rewards/margins": 0.031964417546987534, + "rewards/rejected": -0.720670759677887, + "step": 267 + }, + { + "epoch": 0.35, + "learning_rate": 4.993877788986285e-05, + "logits/chosen": -2.3766613006591797, + "logits/rejected": -2.507246971130371, + "logps/chosen": -165.4411163330078, + "logps/rejected": -199.56741333007812, + "loss": 0.575, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.49406641721725464, + "rewards/margins": 0.3596181869506836, + "rewards/rejected": -0.853684663772583, + "step": 268 + }, + { + "epoch": 0.35, + "learning_rate": 4.9938047188134776e-05, + "logits/chosen": -2.4775924682617188, + "logits/rejected": -2.4857354164123535, + "logps/chosen": -195.7847442626953, + "logps/rejected": -216.16220092773438, + "loss": 0.6795, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5620791912078857, + "rewards/margins": 0.07040975242853165, + "rewards/rejected": -0.632489025592804, + "step": 269 + }, + { + "epoch": 0.35, + "learning_rate": 4.993731215708866e-05, + "logits/chosen": -2.637573719024658, + "logits/rejected": -2.700226306915283, + "logps/chosen": -200.697509765625, + "logps/rejected": -214.4364471435547, + "loss": 0.6027, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5127323269844055, + "rewards/margins": 0.24725483357906342, + "rewards/rejected": -0.7599871754646301, + "step": 270 + }, + { + "epoch": 0.35, + "learning_rate": 4.993657279685212e-05, + "logits/chosen": -2.4737141132354736, + "logits/rejected": -2.576711654663086, + "logps/chosen": -198.31393432617188, + "logps/rejected": -219.9713592529297, + "loss": 0.7162, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7182321548461914, + "rewards/margins": 0.04057084769010544, + "rewards/rejected": -0.7588030099868774, + "step": 271 + }, + { + "epoch": 0.36, + "learning_rate": 4.9935829107553516e-05, + "logits/chosen": -2.3163719177246094, + "logits/rejected": -2.3124592304229736, + "logps/chosen": -198.31076049804688, + "logps/rejected": -210.8530731201172, + "loss": 0.6581, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5568541288375854, + "rewards/margins": 0.1273648738861084, + "rewards/rejected": -0.6842190623283386, + "step": 272 + }, + { + "epoch": 0.36, + "learning_rate": 4.993508108932195e-05, + "logits/chosen": -2.4512782096862793, + "logits/rejected": -2.508169651031494, + "logps/chosen": -202.17613220214844, + "logps/rejected": -203.27789306640625, + "loss": 0.6612, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7900542616844177, + "rewards/margins": 0.12749117612838745, + "rewards/rejected": -0.91754549741745, + "step": 273 + }, + { + "epoch": 0.36, + "learning_rate": 4.9934328742287285e-05, + "logits/chosen": -2.391414165496826, + "logits/rejected": -2.3899242877960205, + "logps/chosen": -231.99913024902344, + "logps/rejected": -232.77243041992188, + "loss": 0.7863, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.6857910752296448, + "rewards/margins": -0.05944066494703293, + "rewards/rejected": -0.626350462436676, + "step": 274 + }, + { + "epoch": 0.36, + "learning_rate": 4.993357206658011e-05, + "logits/chosen": -2.5768861770629883, + "logits/rejected": -2.6738839149475098, + "logps/chosen": -175.33010864257812, + "logps/rejected": -185.26443481445312, + "loss": 0.7157, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6001064777374268, + "rewards/margins": 0.052334412932395935, + "rewards/rejected": -0.6524409651756287, + "step": 275 + }, + { + "epoch": 0.36, + "learning_rate": 4.993281106233182e-05, + "logits/chosen": -2.3842387199401855, + "logits/rejected": -2.425611972808838, + "logps/chosen": -165.09873962402344, + "logps/rejected": -185.34361267089844, + "loss": 0.5746, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.4550285339355469, + "rewards/margins": 0.2833191156387329, + "rewards/rejected": -0.738347589969635, + "step": 276 + }, + { + "epoch": 0.36, + "learning_rate": 4.9932045729674505e-05, + "logits/chosen": -2.370333671569824, + "logits/rejected": -2.343536376953125, + "logps/chosen": -211.58517456054688, + "logps/rejected": -191.9432373046875, + "loss": 0.7727, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7500544786453247, + "rewards/margins": -0.05222831293940544, + "rewards/rejected": -0.697826087474823, + "step": 277 + }, + { + "epoch": 0.36, + "learning_rate": 4.993127606874104e-05, + "logits/chosen": -2.3698928356170654, + "logits/rejected": -2.459376573562622, + "logps/chosen": -241.92343139648438, + "logps/rejected": -265.6181945800781, + "loss": 0.6437, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6877275705337524, + "rewards/margins": 0.16285517811775208, + "rewards/rejected": -0.8505828380584717, + "step": 278 + }, + { + "epoch": 0.37, + "learning_rate": 4.9930502079665025e-05, + "logits/chosen": -2.3997507095336914, + "logits/rejected": -2.4078052043914795, + "logps/chosen": -228.6896514892578, + "logps/rejected": -221.53338623046875, + "loss": 0.7984, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6329948306083679, + "rewards/margins": -0.1108212098479271, + "rewards/rejected": -0.5221735835075378, + "step": 279 + }, + { + "epoch": 0.37, + "learning_rate": 4.9929723762580835e-05, + "logits/chosen": -2.498244285583496, + "logits/rejected": -2.571398973464966, + "logps/chosen": -192.15635681152344, + "logps/rejected": -176.19012451171875, + "loss": 0.7497, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5479970574378967, + "rewards/margins": -0.0442587286233902, + "rewards/rejected": -0.5037383437156677, + "step": 280 + }, + { + "epoch": 0.37, + "learning_rate": 4.9928941117623604e-05, + "logits/chosen": -2.6329376697540283, + "logits/rejected": -2.635847806930542, + "logps/chosen": -207.86293029785156, + "logps/rejected": -224.18490600585938, + "loss": 0.6649, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5313466787338257, + "rewards/margins": 0.07021217793226242, + "rewards/rejected": -0.6015588045120239, + "step": 281 + }, + { + "epoch": 0.37, + "learning_rate": 4.9928154144929175e-05, + "logits/chosen": -2.5966103076934814, + "logits/rejected": -2.571523666381836, + "logps/chosen": -192.47384643554688, + "logps/rejected": -180.02134704589844, + "loss": 0.8799, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.8716793656349182, + "rewards/margins": -0.2290811538696289, + "rewards/rejected": -0.6425981521606445, + "step": 282 + }, + { + "epoch": 0.37, + "learning_rate": 4.9927362844634186e-05, + "logits/chosen": -2.6513888835906982, + "logits/rejected": -2.606253147125244, + "logps/chosen": -208.56338500976562, + "logps/rejected": -200.686279296875, + "loss": 0.7609, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5643545389175415, + "rewards/margins": -0.08666092902421951, + "rewards/rejected": -0.4776935577392578, + "step": 283 + }, + { + "epoch": 0.37, + "learning_rate": 4.9926567216876e-05, + "logits/chosen": -2.5372819900512695, + "logits/rejected": -2.5824756622314453, + "logps/chosen": -187.18386840820312, + "logps/rejected": -211.16427612304688, + "loss": 0.6838, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5520645380020142, + "rewards/margins": 0.10136236250400543, + "rewards/rejected": -0.6534268856048584, + "step": 284 + }, + { + "epoch": 0.37, + "learning_rate": 4.992576726179274e-05, + "logits/chosen": -2.5787792205810547, + "logits/rejected": -2.5688636302948, + "logps/chosen": -199.59490966796875, + "logps/rejected": -221.02023315429688, + "loss": 0.7033, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5855346918106079, + "rewards/margins": 0.010282933712005615, + "rewards/rejected": -0.5958175659179688, + "step": 285 + }, + { + "epoch": 0.37, + "learning_rate": 4.9924962979523296e-05, + "logits/chosen": -2.588792085647583, + "logits/rejected": -2.5663416385650635, + "logps/chosen": -197.04592895507812, + "logps/rejected": -184.49290466308594, + "loss": 0.6537, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.46529656648635864, + "rewards/margins": 0.1208263412117958, + "rewards/rejected": -0.5861228704452515, + "step": 286 + }, + { + "epoch": 0.38, + "learning_rate": 4.992415437020727e-05, + "logits/chosen": -2.6553092002868652, + "logits/rejected": -2.763751268386841, + "logps/chosen": -217.5886993408203, + "logps/rejected": -233.33489990234375, + "loss": 0.6027, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4947764575481415, + "rewards/margins": 0.2827147841453552, + "rewards/rejected": -0.7774912118911743, + "step": 287 + }, + { + "epoch": 0.38, + "learning_rate": 4.992334143398506e-05, + "logits/chosen": -2.5335421562194824, + "logits/rejected": -2.600069999694824, + "logps/chosen": -204.98912048339844, + "logps/rejected": -212.0712890625, + "loss": 0.6079, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5413306355476379, + "rewards/margins": 0.22219431400299072, + "rewards/rejected": -0.7635249495506287, + "step": 288 + }, + { + "epoch": 0.38, + "learning_rate": 4.992252417099778e-05, + "logits/chosen": -2.638511896133423, + "logits/rejected": -2.600456476211548, + "logps/chosen": -172.59390258789062, + "logps/rejected": -192.1689910888672, + "loss": 0.7194, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5202851891517639, + "rewards/margins": 0.017805464565753937, + "rewards/rejected": -0.5380906462669373, + "step": 289 + }, + { + "epoch": 0.38, + "learning_rate": 4.992170258138732e-05, + "logits/chosen": -2.5361521244049072, + "logits/rejected": -2.4191856384277344, + "logps/chosen": -177.23558044433594, + "logps/rejected": -169.40321350097656, + "loss": 0.6089, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.20310896635055542, + "rewards/margins": 0.2397383451461792, + "rewards/rejected": -0.4428473711013794, + "step": 290 + }, + { + "epoch": 0.38, + "learning_rate": 4.99208766652963e-05, + "logits/chosen": -2.727092742919922, + "logits/rejected": -2.613562822341919, + "logps/chosen": -228.9248504638672, + "logps/rejected": -222.22886657714844, + "loss": 0.6345, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5637404322624207, + "rewards/margins": 0.21417354047298431, + "rewards/rejected": -0.7779139280319214, + "step": 291 + }, + { + "epoch": 0.38, + "learning_rate": 4.99200464228681e-05, + "logits/chosen": -2.530290126800537, + "logits/rejected": -2.49137282371521, + "logps/chosen": -218.17156982421875, + "logps/rejected": -225.41952514648438, + "loss": 0.7465, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.567755937576294, + "rewards/margins": -0.07816261053085327, + "rewards/rejected": -0.4895933270454407, + "step": 292 + }, + { + "epoch": 0.38, + "learning_rate": 4.9919211854246874e-05, + "logits/chosen": -2.683086633682251, + "logits/rejected": -2.588696002960205, + "logps/chosen": -268.5683898925781, + "logps/rejected": -255.58445739746094, + "loss": 0.7069, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5820844769477844, + "rewards/margins": 0.0005566291511058807, + "rewards/rejected": -0.582641065120697, + "step": 293 + }, + { + "epoch": 0.38, + "learning_rate": 4.9918372959577486e-05, + "logits/chosen": -2.7265396118164062, + "logits/rejected": -2.7635843753814697, + "logps/chosen": -206.09573364257812, + "logps/rejected": -214.75784301757812, + "loss": 0.6975, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6429769396781921, + "rewards/margins": 0.1274566352367401, + "rewards/rejected": -0.7704335451126099, + "step": 294 + }, + { + "epoch": 0.39, + "learning_rate": 4.9917529739005574e-05, + "logits/chosen": -2.507643938064575, + "logits/rejected": -2.4392926692962646, + "logps/chosen": -219.877197265625, + "logps/rejected": -217.41537475585938, + "loss": 0.729, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5904738306999207, + "rewards/margins": -0.04813346266746521, + "rewards/rejected": -0.5423403382301331, + "step": 295 + }, + { + "epoch": 0.39, + "learning_rate": 4.991668219267752e-05, + "logits/chosen": -2.613016128540039, + "logits/rejected": -2.7239303588867188, + "logps/chosen": -235.21534729003906, + "logps/rejected": -266.6222839355469, + "loss": 0.7173, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6593362092971802, + "rewards/margins": 0.04644089192152023, + "rewards/rejected": -0.7057771682739258, + "step": 296 + }, + { + "epoch": 0.39, + "learning_rate": 4.991583032074047e-05, + "logits/chosen": -2.774909257888794, + "logits/rejected": -2.7659873962402344, + "logps/chosen": -250.04261779785156, + "logps/rejected": -242.6505584716797, + "loss": 0.7062, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6024019718170166, + "rewards/margins": 0.010418819263577461, + "rewards/rejected": -0.6128207445144653, + "step": 297 + }, + { + "epoch": 0.39, + "learning_rate": 4.99149741233423e-05, + "logits/chosen": -2.447873592376709, + "logits/rejected": -2.468458414077759, + "logps/chosen": -214.55784606933594, + "logps/rejected": -189.22238159179688, + "loss": 0.6822, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6992676854133606, + "rewards/margins": 0.09424494951963425, + "rewards/rejected": -0.7935126423835754, + "step": 298 + }, + { + "epoch": 0.39, + "learning_rate": 4.9914113600631665e-05, + "logits/chosen": -2.5641584396362305, + "logits/rejected": -2.602008104324341, + "logps/chosen": -310.73077392578125, + "logps/rejected": -290.17352294921875, + "loss": 0.6765, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5222407579421997, + "rewards/margins": 0.06505537033081055, + "rewards/rejected": -0.587296187877655, + "step": 299 + }, + { + "epoch": 0.39, + "learning_rate": 4.991324875275794e-05, + "logits/chosen": -2.5343639850616455, + "logits/rejected": -2.610719919204712, + "logps/chosen": -199.84573364257812, + "logps/rejected": -203.96478271484375, + "loss": 0.718, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5667111873626709, + "rewards/margins": -0.001658361405134201, + "rewards/rejected": -0.5650528073310852, + "step": 300 + }, + { + "epoch": 0.39, + "learning_rate": 4.991237957987127e-05, + "logits/chosen": -2.379605293273926, + "logits/rejected": -2.3678669929504395, + "logps/chosen": -211.86761474609375, + "logps/rejected": -213.41464233398438, + "loss": 0.6325, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.46346691250801086, + "rewards/margins": 0.20269399881362915, + "rewards/rejected": -0.6661609411239624, + "step": 301 + }, + { + "epoch": 0.4, + "learning_rate": 4.991150608212254e-05, + "logits/chosen": -2.6828835010528564, + "logits/rejected": -2.7126364707946777, + "logps/chosen": -196.61216735839844, + "logps/rejected": -223.1612548828125, + "loss": 0.7503, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5514590740203857, + "rewards/margins": -0.08642277121543884, + "rewards/rejected": -0.4650362730026245, + "step": 302 + }, + { + "epoch": 0.4, + "learning_rate": 4.9910628259663404e-05, + "logits/chosen": -2.4960732460021973, + "logits/rejected": -2.608139991760254, + "logps/chosen": -197.00277709960938, + "logps/rejected": -196.33804321289062, + "loss": 0.564, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.35482868552207947, + "rewards/margins": 0.38187745213508606, + "rewards/rejected": -0.7367061376571655, + "step": 303 + }, + { + "epoch": 0.4, + "learning_rate": 4.990974611264625e-05, + "logits/chosen": -2.3627309799194336, + "logits/rejected": -2.3418447971343994, + "logps/chosen": -232.52740478515625, + "logps/rejected": -227.15753173828125, + "loss": 0.6344, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.6158957481384277, + "rewards/margins": 0.18052013218402863, + "rewards/rejected": -0.7964158654212952, + "step": 304 + }, + { + "epoch": 0.4, + "learning_rate": 4.990885964122421e-05, + "logits/chosen": -2.594547748565674, + "logits/rejected": -2.6074790954589844, + "logps/chosen": -170.09300231933594, + "logps/rejected": -190.0100860595703, + "loss": 0.7467, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5137202739715576, + "rewards/margins": -0.07406175136566162, + "rewards/rejected": -0.439658522605896, + "step": 305 + }, + { + "epoch": 0.4, + "learning_rate": 4.990796884555119e-05, + "logits/chosen": -2.632927417755127, + "logits/rejected": -2.6694023609161377, + "logps/chosen": -215.45489501953125, + "logps/rejected": -203.8738555908203, + "loss": 0.6641, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4411565661430359, + "rewards/margins": 0.1269129067659378, + "rewards/rejected": -0.5680694580078125, + "step": 306 + }, + { + "epoch": 0.4, + "learning_rate": 4.9907073725781836e-05, + "logits/chosen": -2.4353699684143066, + "logits/rejected": -2.537163734436035, + "logps/chosen": -279.5519714355469, + "logps/rejected": -290.08135986328125, + "loss": 0.6756, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.32637274265289307, + "rewards/margins": 0.053382109850645065, + "rewards/rejected": -0.379754900932312, + "step": 307 + }, + { + "epoch": 0.4, + "learning_rate": 4.9906174282071535e-05, + "logits/chosen": -2.586729049682617, + "logits/rejected": -2.583314895629883, + "logps/chosen": -197.04298400878906, + "logps/rejected": -195.334228515625, + "loss": 0.6718, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.515608549118042, + "rewards/margins": 0.06315431743860245, + "rewards/rejected": -0.5787628293037415, + "step": 308 + }, + { + "epoch": 0.4, + "learning_rate": 4.990527051457644e-05, + "logits/chosen": -2.45725154876709, + "logits/rejected": -2.445600748062134, + "logps/chosen": -204.52867126464844, + "logps/rejected": -290.3397216796875, + "loss": 0.6908, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6991719007492065, + "rewards/margins": 0.08316923677921295, + "rewards/rejected": -0.7823411226272583, + "step": 309 + }, + { + "epoch": 0.41, + "learning_rate": 4.9904362423453446e-05, + "logits/chosen": -2.604722738265991, + "logits/rejected": -2.6120543479919434, + "logps/chosen": -193.1224822998047, + "logps/rejected": -189.89694213867188, + "loss": 0.6076, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.3250601589679718, + "rewards/margins": 0.19382469356060028, + "rewards/rejected": -0.5188848376274109, + "step": 310 + }, + { + "epoch": 0.41, + "learning_rate": 4.990345000886019e-05, + "logits/chosen": -2.6838831901550293, + "logits/rejected": -2.6718058586120605, + "logps/chosen": -177.02090454101562, + "logps/rejected": -173.667724609375, + "loss": 0.6993, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5827906131744385, + "rewards/margins": 0.027324199676513672, + "rewards/rejected": -0.6101148128509521, + "step": 311 + }, + { + "epoch": 0.41, + "learning_rate": 4.990253327095509e-05, + "logits/chosen": -2.60197377204895, + "logits/rejected": -2.522291660308838, + "logps/chosen": -173.03811645507812, + "logps/rejected": -149.1508026123047, + "loss": 0.7939, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.4563007950782776, + "rewards/margins": -0.13545696437358856, + "rewards/rejected": -0.32084381580352783, + "step": 312 + }, + { + "epoch": 0.41, + "learning_rate": 4.9901612209897275e-05, + "logits/chosen": -2.643686532974243, + "logits/rejected": -2.6443119049072266, + "logps/chosen": -187.93931579589844, + "logps/rejected": -208.4308624267578, + "loss": 0.6686, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6419711709022522, + "rewards/margins": 0.10658866912126541, + "rewards/rejected": -0.748559832572937, + "step": 313 + }, + { + "epoch": 0.41, + "learning_rate": 4.990068682584666e-05, + "logits/chosen": -2.469681739807129, + "logits/rejected": -2.4439382553100586, + "logps/chosen": -186.7505340576172, + "logps/rejected": -204.6378936767578, + "loss": 0.7857, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5965012311935425, + "rewards/margins": -0.12374071031808853, + "rewards/rejected": -0.47276046872138977, + "step": 314 + }, + { + "epoch": 0.41, + "learning_rate": 4.989975711896388e-05, + "logits/chosen": -2.3346047401428223, + "logits/rejected": -2.323143720626831, + "logps/chosen": -223.70867919921875, + "logps/rejected": -238.58311462402344, + "loss": 0.7061, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5982979536056519, + "rewards/margins": -0.005420597270131111, + "rewards/rejected": -0.5928773283958435, + "step": 315 + }, + { + "epoch": 0.41, + "learning_rate": 4.989882308941034e-05, + "logits/chosen": -2.759549379348755, + "logits/rejected": -2.7803878784179688, + "logps/chosen": -239.48947143554688, + "logps/rejected": -181.70718383789062, + "loss": 0.6671, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.46423962712287903, + "rewards/margins": 0.11677178740501404, + "rewards/rejected": -0.5810113549232483, + "step": 316 + }, + { + "epoch": 0.41, + "learning_rate": 4.9897884737348196e-05, + "logits/chosen": -2.485264778137207, + "logits/rejected": -2.5983753204345703, + "logps/chosen": -175.43971252441406, + "logps/rejected": -219.08517456054688, + "loss": 0.7124, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5487624406814575, + "rewards/margins": 0.0002348199486732483, + "rewards/rejected": -0.5489972829818726, + "step": 317 + }, + { + "epoch": 0.42, + "learning_rate": 4.989694206294035e-05, + "logits/chosen": -2.6732518672943115, + "logits/rejected": -2.6291115283966064, + "logps/chosen": -193.59197998046875, + "logps/rejected": -207.9779052734375, + "loss": 0.6528, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4739716649055481, + "rewards/margins": 0.12173515558242798, + "rewards/rejected": -0.5957068204879761, + "step": 318 + }, + { + "epoch": 0.42, + "learning_rate": 4.989599506635044e-05, + "logits/chosen": -2.4193854331970215, + "logits/rejected": -2.565293073654175, + "logps/chosen": -263.6156311035156, + "logps/rejected": -251.44192504882812, + "loss": 0.6307, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.47510460019111633, + "rewards/margins": 0.25979098677635193, + "rewards/rejected": -0.734895646572113, + "step": 319 + }, + { + "epoch": 0.42, + "learning_rate": 4.989504374774288e-05, + "logits/chosen": -2.3681142330169678, + "logits/rejected": -2.4691953659057617, + "logps/chosen": -163.5777587890625, + "logps/rejected": -194.8612060546875, + "loss": 0.6185, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4559204876422882, + "rewards/margins": 0.19214320182800293, + "rewards/rejected": -0.648063600063324, + "step": 320 + }, + { + "epoch": 0.42, + "learning_rate": 4.989408810728281e-05, + "logits/chosen": -2.477172613143921, + "logits/rejected": -2.5207676887512207, + "logps/chosen": -141.12867736816406, + "logps/rejected": -146.79791259765625, + "loss": 0.657, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.22462037205696106, + "rewards/margins": 0.09705987572669983, + "rewards/rejected": -0.3216802775859833, + "step": 321 + }, + { + "epoch": 0.42, + "learning_rate": 4.989312814513614e-05, + "logits/chosen": -2.534242630004883, + "logits/rejected": -2.47320294380188, + "logps/chosen": -223.2980499267578, + "logps/rejected": -224.9616241455078, + "loss": 0.7372, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5472955107688904, + "rewards/margins": -0.037168219685554504, + "rewards/rejected": -0.5101273059844971, + "step": 322 + }, + { + "epoch": 0.42, + "learning_rate": 4.989216386146953e-05, + "logits/chosen": -2.3438358306884766, + "logits/rejected": -2.623857021331787, + "logps/chosen": -190.59585571289062, + "logps/rejected": -234.1730194091797, + "loss": 0.6873, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3276049494743347, + "rewards/margins": 0.06552110612392426, + "rewards/rejected": -0.39312607049942017, + "step": 323 + }, + { + "epoch": 0.42, + "learning_rate": 4.9891195256450366e-05, + "logits/chosen": -2.210477828979492, + "logits/rejected": -2.195955991744995, + "logps/chosen": -185.43902587890625, + "logps/rejected": -224.94488525390625, + "loss": 0.678, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.38528165221214294, + "rewards/margins": 0.09291733056306839, + "rewards/rejected": -0.47819897532463074, + "step": 324 + }, + { + "epoch": 0.43, + "learning_rate": 4.989022233024681e-05, + "logits/chosen": -2.7818360328674316, + "logits/rejected": -2.764118194580078, + "logps/chosen": -224.84347534179688, + "logps/rejected": -205.856201171875, + "loss": 0.6028, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.3358863592147827, + "rewards/margins": 0.23018960654735565, + "rewards/rejected": -0.5660759806632996, + "step": 325 + }, + { + "epoch": 0.43, + "learning_rate": 4.9889245083027755e-05, + "logits/chosen": -2.3800809383392334, + "logits/rejected": -2.3793954849243164, + "logps/chosen": -168.0996551513672, + "logps/rejected": -197.4810791015625, + "loss": 0.7001, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.36885035037994385, + "rewards/margins": 0.015535619109869003, + "rewards/rejected": -0.3843860328197479, + "step": 326 + }, + { + "epoch": 0.43, + "learning_rate": 4.988826351496287e-05, + "logits/chosen": -2.5323662757873535, + "logits/rejected": -2.5999341011047363, + "logps/chosen": -215.6177978515625, + "logps/rejected": -255.65231323242188, + "loss": 0.6679, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6092446446418762, + "rewards/margins": 0.09709976613521576, + "rewards/rejected": -0.7063443660736084, + "step": 327 + }, + { + "epoch": 0.43, + "learning_rate": 4.988727762622255e-05, + "logits/chosen": -2.7337067127227783, + "logits/rejected": -2.742793083190918, + "logps/chosen": -214.25506591796875, + "logps/rejected": -221.33468627929688, + "loss": 0.6614, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6302060484886169, + "rewards/margins": 0.12371520698070526, + "rewards/rejected": -0.7539212107658386, + "step": 328 + }, + { + "epoch": 0.43, + "learning_rate": 4.9886287416977936e-05, + "logits/chosen": -2.510671377182007, + "logits/rejected": -2.5760793685913086, + "logps/chosen": -169.99656677246094, + "logps/rejected": -181.1496124267578, + "loss": 0.6957, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4545098841190338, + "rewards/margins": 0.014530147425830364, + "rewards/rejected": -0.46904003620147705, + "step": 329 + }, + { + "epoch": 0.43, + "learning_rate": 4.988529288740096e-05, + "logits/chosen": -2.6438777446746826, + "logits/rejected": -2.6174240112304688, + "logps/chosen": -187.69232177734375, + "logps/rejected": -198.16409301757812, + "loss": 0.6247, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.37409377098083496, + "rewards/margins": 0.1910334676504135, + "rewards/rejected": -0.5651272535324097, + "step": 330 + }, + { + "epoch": 0.43, + "learning_rate": 4.9884294037664245e-05, + "logits/chosen": -2.456332206726074, + "logits/rejected": -2.4327914714813232, + "logps/chosen": -221.3645782470703, + "logps/rejected": -220.4224853515625, + "loss": 0.726, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.9011685848236084, + "rewards/margins": 0.07650195062160492, + "rewards/rejected": -0.9776705503463745, + "step": 331 + }, + { + "epoch": 0.43, + "learning_rate": 4.988329086794122e-05, + "logits/chosen": -2.463491439819336, + "logits/rejected": -2.49444842338562, + "logps/chosen": -197.96144104003906, + "logps/rejected": -214.404296875, + "loss": 0.6547, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.42488348484039307, + "rewards/margins": 0.11507010459899902, + "rewards/rejected": -0.5399536490440369, + "step": 332 + }, + { + "epoch": 0.44, + "learning_rate": 4.9882283378406015e-05, + "logits/chosen": -2.5106682777404785, + "logits/rejected": -2.5157594680786133, + "logps/chosen": -167.09947204589844, + "logps/rejected": -174.6180419921875, + "loss": 0.6179, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4473687410354614, + "rewards/margins": 0.22422267496585846, + "rewards/rejected": -0.6715914011001587, + "step": 333 + }, + { + "epoch": 0.44, + "learning_rate": 4.988127156923355e-05, + "logits/chosen": -2.4497499465942383, + "logits/rejected": -2.491909980773926, + "logps/chosen": -167.1006622314453, + "logps/rejected": -165.177978515625, + "loss": 0.6855, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5522390604019165, + "rewards/margins": 0.06803999096155167, + "rewards/rejected": -0.6202789545059204, + "step": 334 + }, + { + "epoch": 0.44, + "learning_rate": 4.9880255440599476e-05, + "logits/chosen": -2.445422649383545, + "logits/rejected": -2.4025139808654785, + "logps/chosen": -199.83444213867188, + "logps/rejected": -203.16355895996094, + "loss": 0.7404, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.658203125, + "rewards/margins": -0.06064491346478462, + "rewards/rejected": -0.5975580811500549, + "step": 335 + }, + { + "epoch": 0.44, + "learning_rate": 4.987923499268018e-05, + "logits/chosen": -2.540513753890991, + "logits/rejected": -2.6426820755004883, + "logps/chosen": -181.93406677246094, + "logps/rejected": -203.6474609375, + "loss": 0.6465, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.45222824811935425, + "rewards/margins": 0.17588049173355103, + "rewards/rejected": -0.6281087398529053, + "step": 336 + }, + { + "epoch": 0.44, + "learning_rate": 4.987821022565284e-05, + "logits/chosen": -2.608978509902954, + "logits/rejected": -2.632282257080078, + "logps/chosen": -195.6852569580078, + "logps/rejected": -216.0919952392578, + "loss": 0.6137, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4888995289802551, + "rewards/margins": 0.3496631681919098, + "rewards/rejected": -0.8385627269744873, + "step": 337 + }, + { + "epoch": 0.44, + "learning_rate": 4.987718113969534e-05, + "logits/chosen": -2.637908458709717, + "logits/rejected": -2.6849193572998047, + "logps/chosen": -307.9834289550781, + "logps/rejected": -321.95867919921875, + "loss": 0.5909, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.4099658131599426, + "rewards/margins": 0.3652358055114746, + "rewards/rejected": -0.7752016186714172, + "step": 338 + }, + { + "epoch": 0.44, + "learning_rate": 4.9876147734986335e-05, + "logits/chosen": -2.6805038452148438, + "logits/rejected": -2.659087657928467, + "logps/chosen": -235.20855712890625, + "logps/rejected": -222.38107299804688, + "loss": 0.7633, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7779167294502258, + "rewards/margins": -0.11177889257669449, + "rewards/rejected": -0.6661379337310791, + "step": 339 + }, + { + "epoch": 0.44, + "learning_rate": 4.987511001170523e-05, + "logits/chosen": -2.4979588985443115, + "logits/rejected": -2.509103775024414, + "logps/chosen": -205.0665283203125, + "logps/rejected": -197.5661163330078, + "loss": 0.584, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.603629469871521, + "rewards/margins": 0.3201472759246826, + "rewards/rejected": -0.9237766861915588, + "step": 340 + }, + { + "epoch": 0.45, + "learning_rate": 4.987406797003218e-05, + "logits/chosen": -2.6034300327301025, + "logits/rejected": -2.606010913848877, + "logps/chosen": -212.59959411621094, + "logps/rejected": -266.6811218261719, + "loss": 0.7099, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.7883586287498474, + "rewards/margins": 0.047954261302948, + "rewards/rejected": -0.8363128900527954, + "step": 341 + }, + { + "epoch": 0.45, + "learning_rate": 4.987302161014808e-05, + "logits/chosen": -2.26737380027771, + "logits/rejected": -2.2702267169952393, + "logps/chosen": -149.69259643554688, + "logps/rejected": -207.59146118164062, + "loss": 0.718, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6731799244880676, + "rewards/margins": 0.02036801353096962, + "rewards/rejected": -0.6935478448867798, + "step": 342 + }, + { + "epoch": 0.45, + "learning_rate": 4.9871970932234586e-05, + "logits/chosen": -2.3482635021209717, + "logits/rejected": -2.363096237182617, + "logps/chosen": -196.5614013671875, + "logps/rejected": -206.21414184570312, + "loss": 0.6163, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5754539966583252, + "rewards/margins": 0.2127307951450348, + "rewards/rejected": -0.7881847620010376, + "step": 343 + }, + { + "epoch": 0.45, + "learning_rate": 4.9870915936474095e-05, + "logits/chosen": -2.7459514141082764, + "logits/rejected": -2.7396438121795654, + "logps/chosen": -206.6090087890625, + "logps/rejected": -211.6893768310547, + "loss": 0.6763, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7424849271774292, + "rewards/margins": 0.08226263523101807, + "rewards/rejected": -0.8247475624084473, + "step": 344 + }, + { + "epoch": 0.45, + "learning_rate": 4.986985662304976e-05, + "logits/chosen": -2.769526243209839, + "logits/rejected": -2.807554244995117, + "logps/chosen": -184.0075225830078, + "logps/rejected": -225.2920379638672, + "loss": 0.6217, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7790610194206238, + "rewards/margins": 0.24687500298023224, + "rewards/rejected": -1.0259360074996948, + "step": 345 + }, + { + "epoch": 0.45, + "learning_rate": 4.9868792992145484e-05, + "logits/chosen": -2.6607320308685303, + "logits/rejected": -2.6989316940307617, + "logps/chosen": -218.40530395507812, + "logps/rejected": -227.24240112304688, + "loss": 0.7374, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6406981945037842, + "rewards/margins": 0.009773064404726028, + "rewards/rejected": -0.6504712104797363, + "step": 346 + }, + { + "epoch": 0.45, + "learning_rate": 4.9867725043945904e-05, + "logits/chosen": -2.5469489097595215, + "logits/rejected": -2.539520740509033, + "logps/chosen": -222.92864990234375, + "logps/rejected": -236.92977905273438, + "loss": 0.6992, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.640974223613739, + "rewards/margins": 0.09771876782178879, + "rewards/rejected": -0.7386929988861084, + "step": 347 + }, + { + "epoch": 0.46, + "learning_rate": 4.9866652778636436e-05, + "logits/chosen": -2.545572280883789, + "logits/rejected": -2.541029453277588, + "logps/chosen": -182.341552734375, + "logps/rejected": -197.535888671875, + "loss": 0.7149, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7396100163459778, + "rewards/margins": 0.06020259112119675, + "rewards/rejected": -0.7998126149177551, + "step": 348 + }, + { + "epoch": 0.46, + "learning_rate": 4.986557619640322e-05, + "logits/chosen": -2.479189157485962, + "logits/rejected": -2.504956007003784, + "logps/chosen": -184.10472106933594, + "logps/rejected": -184.99070739746094, + "loss": 0.5709, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6914107799530029, + "rewards/margins": 0.33003267645835876, + "rewards/rejected": -1.021443486213684, + "step": 349 + }, + { + "epoch": 0.46, + "learning_rate": 4.986449529743314e-05, + "logits/chosen": -2.7267708778381348, + "logits/rejected": -2.711848735809326, + "logps/chosen": -225.98228454589844, + "logps/rejected": -204.31724548339844, + "loss": 0.7177, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.8359745740890503, + "rewards/margins": 0.011185385286808014, + "rewards/rejected": -0.8471599221229553, + "step": 350 + }, + { + "epoch": 0.46, + "learning_rate": 4.9863410081913875e-05, + "logits/chosen": -2.691619396209717, + "logits/rejected": -2.669311285018921, + "logps/chosen": -157.73721313476562, + "logps/rejected": -161.97061157226562, + "loss": 0.7285, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.5839475393295288, + "rewards/margins": -0.027979355305433273, + "rewards/rejected": -0.555968165397644, + "step": 351 + }, + { + "epoch": 0.46, + "learning_rate": 4.98623205500338e-05, + "logits/chosen": -2.6664159297943115, + "logits/rejected": -2.653743267059326, + "logps/chosen": -155.03836059570312, + "logps/rejected": -141.4946746826172, + "loss": 0.7569, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5298622250556946, + "rewards/margins": -0.08286149799823761, + "rewards/rejected": -0.44700077176094055, + "step": 352 + }, + { + "epoch": 0.46, + "learning_rate": 4.986122670198205e-05, + "logits/chosen": -2.588876485824585, + "logits/rejected": -2.706108808517456, + "logps/chosen": -170.23751831054688, + "logps/rejected": -172.2996826171875, + "loss": 0.8284, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.9628427624702454, + "rewards/margins": -0.18341386318206787, + "rewards/rejected": -0.779429018497467, + "step": 353 + }, + { + "epoch": 0.46, + "learning_rate": 4.9860128537948555e-05, + "logits/chosen": -2.8683338165283203, + "logits/rejected": -2.814868211746216, + "logps/chosen": -194.464599609375, + "logps/rejected": -197.70077514648438, + "loss": 0.7405, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6296394467353821, + "rewards/margins": -0.0025831498205661774, + "rewards/rejected": -0.6270563006401062, + "step": 354 + }, + { + "epoch": 0.46, + "learning_rate": 4.9859026058123925e-05, + "logits/chosen": -2.978065013885498, + "logits/rejected": -2.9368367195129395, + "logps/chosen": -217.73974609375, + "logps/rejected": -220.19183349609375, + "loss": 0.7014, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6989356279373169, + "rewards/margins": 0.007979679852724075, + "rewards/rejected": -0.7069153189659119, + "step": 355 + }, + { + "epoch": 0.47, + "learning_rate": 4.985791926269958e-05, + "logits/chosen": -2.666344165802002, + "logits/rejected": -2.8283472061157227, + "logps/chosen": -163.43722534179688, + "logps/rejected": -190.4180908203125, + "loss": 0.6778, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5998314023017883, + "rewards/margins": 0.11102181673049927, + "rewards/rejected": -0.7108532190322876, + "step": 356 + }, + { + "epoch": 0.47, + "learning_rate": 4.985680815186764e-05, + "logits/chosen": -2.6322436332702637, + "logits/rejected": -2.6792352199554443, + "logps/chosen": -225.80972290039062, + "logps/rejected": -221.8551025390625, + "loss": 0.6464, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6963925957679749, + "rewards/margins": 0.1484571099281311, + "rewards/rejected": -0.844849705696106, + "step": 357 + }, + { + "epoch": 0.47, + "learning_rate": 4.985569272582101e-05, + "logits/chosen": -2.829981803894043, + "logits/rejected": -2.8077688217163086, + "logps/chosen": -245.94029235839844, + "logps/rejected": -279.1697082519531, + "loss": 0.6446, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6325417757034302, + "rewards/margins": 0.1735651046037674, + "rewards/rejected": -0.8061069250106812, + "step": 358 + }, + { + "epoch": 0.47, + "learning_rate": 4.9854572984753334e-05, + "logits/chosen": -2.8024842739105225, + "logits/rejected": -2.750588893890381, + "logps/chosen": -161.65240478515625, + "logps/rejected": -201.25054931640625, + "loss": 0.6502, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.46800869703292847, + "rewards/margins": 0.12990553677082062, + "rewards/rejected": -0.5979142189025879, + "step": 359 + }, + { + "epoch": 0.47, + "learning_rate": 4.985344892885899e-05, + "logits/chosen": -2.347231149673462, + "logits/rejected": -2.4064972400665283, + "logps/chosen": -158.72801208496094, + "logps/rejected": -167.15805053710938, + "loss": 0.7264, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.7969176173210144, + "rewards/margins": 0.016100093722343445, + "rewards/rejected": -0.8130176663398743, + "step": 360 + }, + { + "epoch": 0.47, + "learning_rate": 4.985232055833313e-05, + "logits/chosen": -2.603712320327759, + "logits/rejected": -2.6721928119659424, + "logps/chosen": -211.3256378173828, + "logps/rejected": -189.5877685546875, + "loss": 0.7658, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6097235679626465, + "rewards/margins": -0.10059280693531036, + "rewards/rejected": -0.5091307163238525, + "step": 361 + }, + { + "epoch": 0.47, + "learning_rate": 4.985118787337164e-05, + "logits/chosen": -2.6673433780670166, + "logits/rejected": -2.766993284225464, + "logps/chosen": -162.41415405273438, + "logps/rejected": -218.15077209472656, + "loss": 0.6236, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5241696238517761, + "rewards/margins": 0.28995370864868164, + "rewards/rejected": -0.8141233921051025, + "step": 362 + }, + { + "epoch": 0.48, + "learning_rate": 4.985005087417115e-05, + "logits/chosen": -2.4291069507598877, + "logits/rejected": -2.382821798324585, + "logps/chosen": -221.48736572265625, + "logps/rejected": -205.37940979003906, + "loss": 0.6503, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4502621591091156, + "rewards/margins": 0.2195020467042923, + "rewards/rejected": -0.6697642207145691, + "step": 363 + }, + { + "epoch": 0.48, + "learning_rate": 4.984890956092905e-05, + "logits/chosen": -2.5559186935424805, + "logits/rejected": -2.496387481689453, + "logps/chosen": -253.63047790527344, + "logps/rejected": -244.56698608398438, + "loss": 0.9484, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.9435632228851318, + "rewards/margins": -0.268410325050354, + "rewards/rejected": -0.6751528978347778, + "step": 364 + }, + { + "epoch": 0.48, + "learning_rate": 4.984776393384348e-05, + "logits/chosen": -2.407367706298828, + "logits/rejected": -2.3134310245513916, + "logps/chosen": -173.61795043945312, + "logps/rejected": -157.41207885742188, + "loss": 0.7949, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5948637127876282, + "rewards/margins": -0.14962351322174072, + "rewards/rejected": -0.44524019956588745, + "step": 365 + }, + { + "epoch": 0.48, + "learning_rate": 4.984661399311332e-05, + "logits/chosen": -2.6404154300689697, + "logits/rejected": -2.7036595344543457, + "logps/chosen": -155.3403778076172, + "logps/rejected": -141.01382446289062, + "loss": 0.7475, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4795231521129608, + "rewards/margins": -0.06228278949856758, + "rewards/rejected": -0.4172403812408447, + "step": 366 + }, + { + "epoch": 0.48, + "learning_rate": 4.9845459738938204e-05, + "logits/chosen": -2.5403473377227783, + "logits/rejected": -2.5674545764923096, + "logps/chosen": -154.85910034179688, + "logps/rejected": -143.22906494140625, + "loss": 0.6505, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4790140390396118, + "rewards/margins": 0.1586410105228424, + "rewards/rejected": -0.6376550197601318, + "step": 367 + }, + { + "epoch": 0.48, + "learning_rate": 4.9844301171518516e-05, + "logits/chosen": -2.6081302165985107, + "logits/rejected": -2.641979217529297, + "logps/chosen": -185.13006591796875, + "logps/rejected": -198.2803955078125, + "loss": 0.6726, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3776107430458069, + "rewards/margins": 0.08190791308879852, + "rewards/rejected": -0.4595187306404114, + "step": 368 + }, + { + "epoch": 0.48, + "learning_rate": 4.984313829105538e-05, + "logits/chosen": -2.6662757396698, + "logits/rejected": -2.7398993968963623, + "logps/chosen": -201.8738555908203, + "logps/rejected": -195.76390075683594, + "loss": 0.5956, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.38797736167907715, + "rewards/margins": 0.2769092321395874, + "rewards/rejected": -0.6648865342140198, + "step": 369 + }, + { + "epoch": 0.48, + "learning_rate": 4.984197109775068e-05, + "logits/chosen": -2.4933226108551025, + "logits/rejected": -2.484591245651245, + "logps/chosen": -202.7592315673828, + "logps/rejected": -199.02105712890625, + "loss": 0.6536, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.35886555910110474, + "rewards/margins": 0.1531214416027069, + "rewards/rejected": -0.511987030506134, + "step": 370 + }, + { + "epoch": 0.49, + "learning_rate": 4.984079959180705e-05, + "logits/chosen": -2.3560287952423096, + "logits/rejected": -2.385936975479126, + "logps/chosen": -198.06044006347656, + "logps/rejected": -203.5518798828125, + "loss": 0.7203, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.35552191734313965, + "rewards/margins": 0.03736201673746109, + "rewards/rejected": -0.39288395643234253, + "step": 371 + }, + { + "epoch": 0.49, + "learning_rate": 4.983962377342786e-05, + "logits/chosen": -2.456765651702881, + "logits/rejected": -2.522660970687866, + "logps/chosen": -236.66744995117188, + "logps/rejected": -241.58963012695312, + "loss": 0.6049, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.551971435546875, + "rewards/margins": 0.30021384358406067, + "rewards/rejected": -0.8521853089332581, + "step": 372 + }, + { + "epoch": 0.49, + "learning_rate": 4.983844364281723e-05, + "logits/chosen": -2.5745279788970947, + "logits/rejected": -2.5961902141571045, + "logps/chosen": -186.73411560058594, + "logps/rejected": -194.74974060058594, + "loss": 0.6354, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.312610000371933, + "rewards/margins": 0.14659735560417175, + "rewards/rejected": -0.45920735597610474, + "step": 373 + }, + { + "epoch": 0.49, + "learning_rate": 4.983725920018004e-05, + "logits/chosen": -2.3747611045837402, + "logits/rejected": -2.4170022010803223, + "logps/chosen": -179.91162109375, + "logps/rejected": -170.89920043945312, + "loss": 0.6529, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.43839430809020996, + "rewards/margins": 0.16247621178627014, + "rewards/rejected": -0.6008704900741577, + "step": 374 + }, + { + "epoch": 0.49, + "learning_rate": 4.9836070445721924e-05, + "logits/chosen": -2.573714017868042, + "logits/rejected": -2.567589282989502, + "logps/chosen": -204.4001922607422, + "logps/rejected": -197.39869689941406, + "loss": 0.751, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.350676029920578, + "rewards/margins": -0.06606191396713257, + "rewards/rejected": -0.28461411595344543, + "step": 375 + }, + { + "epoch": 0.49, + "learning_rate": 4.983487737964924e-05, + "logits/chosen": -2.398193597793579, + "logits/rejected": -2.4593253135681152, + "logps/chosen": -199.0366973876953, + "logps/rejected": -199.51275634765625, + "loss": 0.6772, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.397927463054657, + "rewards/margins": 0.08031069487333298, + "rewards/rejected": -0.47823822498321533, + "step": 376 + }, + { + "epoch": 0.49, + "learning_rate": 4.9833680002169105e-05, + "logits/chosen": -2.5120034217834473, + "logits/rejected": -2.553926706314087, + "logps/chosen": -226.15040588378906, + "logps/rejected": -249.55255126953125, + "loss": 0.6996, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.3695937395095825, + "rewards/margins": 0.0394027940928936, + "rewards/rejected": -0.4089965522289276, + "step": 377 + }, + { + "epoch": 0.49, + "learning_rate": 4.983247831348939e-05, + "logits/chosen": -2.289033889770508, + "logits/rejected": -2.2501816749572754, + "logps/chosen": -201.75392150878906, + "logps/rejected": -217.8618621826172, + "loss": 0.6645, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.28367459774017334, + "rewards/margins": 0.1500779092311859, + "rewards/rejected": -0.43375247716903687, + "step": 378 + }, + { + "epoch": 0.5, + "learning_rate": 4.9831272313818716e-05, + "logits/chosen": -2.4449117183685303, + "logits/rejected": -2.522752046585083, + "logps/chosen": -201.63587951660156, + "logps/rejected": -215.33270263671875, + "loss": 0.6512, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.23286518454551697, + "rewards/margins": 0.127617746591568, + "rewards/rejected": -0.36048293113708496, + "step": 379 + }, + { + "epoch": 0.5, + "learning_rate": 4.983006200336645e-05, + "logits/chosen": -2.4317235946655273, + "logits/rejected": -2.3696205615997314, + "logps/chosen": -243.03509521484375, + "logps/rejected": -186.61427307128906, + "loss": 0.7762, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5067302584648132, + "rewards/margins": -0.11989758908748627, + "rewards/rejected": -0.3868326246738434, + "step": 380 + }, + { + "epoch": 0.5, + "learning_rate": 4.98288473823427e-05, + "logits/chosen": -2.4131155014038086, + "logits/rejected": -2.498098611831665, + "logps/chosen": -190.73397827148438, + "logps/rejected": -213.9364471435547, + "loss": 0.6336, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.24977169930934906, + "rewards/margins": 0.1747249811887741, + "rewards/rejected": -0.42449668049812317, + "step": 381 + }, + { + "epoch": 0.5, + "learning_rate": 4.982762845095833e-05, + "logits/chosen": -2.5675809383392334, + "logits/rejected": -2.6289899349212646, + "logps/chosen": -233.9481658935547, + "logps/rejected": -245.3311767578125, + "loss": 0.6879, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3384949564933777, + "rewards/margins": 0.07099074125289917, + "rewards/rejected": -0.40948566794395447, + "step": 382 + }, + { + "epoch": 0.5, + "learning_rate": 4.982640520942494e-05, + "logits/chosen": -2.3582189083099365, + "logits/rejected": -2.405932903289795, + "logps/chosen": -157.98828125, + "logps/rejected": -179.5477294921875, + "loss": 0.6256, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.30075955390930176, + "rewards/margins": 0.2314756065607071, + "rewards/rejected": -0.5322352051734924, + "step": 383 + }, + { + "epoch": 0.5, + "learning_rate": 4.9825177657954914e-05, + "logits/chosen": -2.6346352100372314, + "logits/rejected": -2.7042407989501953, + "logps/chosen": -174.1151123046875, + "logps/rejected": -203.845458984375, + "loss": 0.6727, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3591979742050171, + "rewards/margins": 0.10004588216543198, + "rewards/rejected": -0.45924389362335205, + "step": 384 + }, + { + "epoch": 0.5, + "learning_rate": 4.982394579676133e-05, + "logits/chosen": -2.3420321941375732, + "logits/rejected": -2.3010876178741455, + "logps/chosen": -186.43467712402344, + "logps/rejected": -183.117431640625, + "loss": 0.6836, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.3466501235961914, + "rewards/margins": 0.06071464717388153, + "rewards/rejected": -0.40736478567123413, + "step": 385 + }, + { + "epoch": 0.51, + "learning_rate": 4.982270962605806e-05, + "logits/chosen": -2.4120569229125977, + "logits/rejected": -2.449502944946289, + "logps/chosen": -168.2112579345703, + "logps/rejected": -227.885986328125, + "loss": 0.5666, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.22820252180099487, + "rewards/margins": 0.35979852080345154, + "rewards/rejected": -0.588001012802124, + "step": 386 + }, + { + "epoch": 0.51, + "learning_rate": 4.9821469146059704e-05, + "logits/chosen": -2.3874173164367676, + "logits/rejected": -2.4257519245147705, + "logps/chosen": -158.00424194335938, + "logps/rejected": -176.4921417236328, + "loss": 0.6581, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.28017669916152954, + "rewards/margins": 0.168703094124794, + "rewards/rejected": -0.44887974858283997, + "step": 387 + }, + { + "epoch": 0.51, + "learning_rate": 4.982022435698161e-05, + "logits/chosen": -2.444685697555542, + "logits/rejected": -2.653350591659546, + "logps/chosen": -207.88034057617188, + "logps/rejected": -231.86582946777344, + "loss": 0.7048, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5877898931503296, + "rewards/margins": 0.10215066373348236, + "rewards/rejected": -0.6899405121803284, + "step": 388 + }, + { + "epoch": 0.51, + "learning_rate": 4.981897525903988e-05, + "logits/chosen": -2.5300025939941406, + "logits/rejected": -2.4865880012512207, + "logps/chosen": -155.8424835205078, + "logps/rejected": -152.6171875, + "loss": 0.8172, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.5343464612960815, + "rewards/margins": -0.17394664883613586, + "rewards/rejected": -0.3603998124599457, + "step": 389 + }, + { + "epoch": 0.51, + "learning_rate": 4.981772185245135e-05, + "logits/chosen": -2.61796236038208, + "logits/rejected": -2.662163734436035, + "logps/chosen": -198.32403564453125, + "logps/rejected": -198.2169647216797, + "loss": 0.751, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4620332419872284, + "rewards/margins": -0.09868823736906052, + "rewards/rejected": -0.3633449673652649, + "step": 390 + }, + { + "epoch": 0.51, + "learning_rate": 4.981646413743363e-05, + "logits/chosen": -2.386425733566284, + "logits/rejected": -2.452526092529297, + "logps/chosen": -187.49838256835938, + "logps/rejected": -181.8330535888672, + "loss": 0.606, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.27474457025527954, + "rewards/margins": 0.2754724621772766, + "rewards/rejected": -0.5502170324325562, + "step": 391 + }, + { + "epoch": 0.51, + "learning_rate": 4.981520211420506e-05, + "logits/chosen": -2.4169254302978516, + "logits/rejected": -2.4883086681365967, + "logps/chosen": -175.3878173828125, + "logps/rejected": -210.00119018554688, + "loss": 0.7642, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.532840371131897, + "rewards/margins": -0.09793217480182648, + "rewards/rejected": -0.4349081814289093, + "step": 392 + }, + { + "epoch": 0.51, + "learning_rate": 4.9813935782984724e-05, + "logits/chosen": -2.528134346008301, + "logits/rejected": -2.491497039794922, + "logps/chosen": -239.28878784179688, + "logps/rejected": -241.08883666992188, + "loss": 0.7798, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6152268648147583, + "rewards/margins": -0.10364706814289093, + "rewards/rejected": -0.5115798711776733, + "step": 393 + }, + { + "epoch": 0.52, + "learning_rate": 4.9812665143992466e-05, + "logits/chosen": -2.5107626914978027, + "logits/rejected": -2.591826915740967, + "logps/chosen": -146.5967559814453, + "logps/rejected": -154.36585998535156, + "loss": 0.6751, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3082922697067261, + "rewards/margins": 0.11212008446455002, + "rewards/rejected": -0.4204123616218567, + "step": 394 + }, + { + "epoch": 0.52, + "learning_rate": 4.981139019744887e-05, + "logits/chosen": -2.5607235431671143, + "logits/rejected": -2.562152862548828, + "logps/chosen": -185.95343017578125, + "logps/rejected": -194.59483337402344, + "loss": 0.6469, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.2080746442079544, + "rewards/margins": 0.12636929750442505, + "rewards/rejected": -0.33444392681121826, + "step": 395 + }, + { + "epoch": 0.52, + "learning_rate": 4.981011094357527e-05, + "logits/chosen": -2.4996774196624756, + "logits/rejected": -2.5331335067749023, + "logps/chosen": -215.7071075439453, + "logps/rejected": -231.0633087158203, + "loss": 0.7228, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4276806116104126, + "rewards/margins": 0.03171852231025696, + "rewards/rejected": -0.45939913392066956, + "step": 396 + }, + { + "epoch": 0.52, + "learning_rate": 4.980882738259376e-05, + "logits/chosen": -2.3114399909973145, + "logits/rejected": -2.3782169818878174, + "logps/chosen": -169.04246520996094, + "logps/rejected": -188.5185089111328, + "loss": 0.7661, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.31136631965637207, + "rewards/margins": -0.09910126030445099, + "rewards/rejected": -0.21226505935192108, + "step": 397 + }, + { + "epoch": 0.52, + "learning_rate": 4.980753951472715e-05, + "logits/chosen": -2.466881275177002, + "logits/rejected": -2.5022642612457275, + "logps/chosen": -168.63385009765625, + "logps/rejected": -192.87521362304688, + "loss": 0.6487, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3558157682418823, + "rewards/margins": 0.13948221504688263, + "rewards/rejected": -0.49529796838760376, + "step": 398 + }, + { + "epoch": 0.52, + "learning_rate": 4.980624734019903e-05, + "logits/chosen": -2.5671474933624268, + "logits/rejected": -2.6410720348358154, + "logps/chosen": -183.6251220703125, + "logps/rejected": -214.81272888183594, + "loss": 0.7078, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4311671555042267, + "rewards/margins": 0.0398234948515892, + "rewards/rejected": -0.4709906280040741, + "step": 399 + }, + { + "epoch": 0.52, + "learning_rate": 4.980495085923372e-05, + "logits/chosen": -2.602818012237549, + "logits/rejected": -2.5820987224578857, + "logps/chosen": -167.52012634277344, + "logps/rejected": -180.94418334960938, + "loss": 0.7179, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.41106754541397095, + "rewards/margins": 0.013390736654400826, + "rewards/rejected": -0.42445826530456543, + "step": 400 + }, + { + "epoch": 0.52, + "learning_rate": 4.980365007205631e-05, + "logits/chosen": -2.4487011432647705, + "logits/rejected": -2.5343518257141113, + "logps/chosen": -201.9692840576172, + "logps/rejected": -244.95530700683594, + "loss": 0.5837, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3986441195011139, + "rewards/margins": 0.3018190264701843, + "rewards/rejected": -0.700463056564331, + "step": 401 + }, + { + "epoch": 0.53, + "learning_rate": 4.980234497889259e-05, + "logits/chosen": -1.9015774726867676, + "logits/rejected": -1.9911203384399414, + "logps/chosen": -165.83837890625, + "logps/rejected": -178.608154296875, + "loss": 0.7531, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.47146371006965637, + "rewards/margins": -0.03502827137708664, + "rewards/rejected": -0.43643543124198914, + "step": 402 + }, + { + "epoch": 0.53, + "learning_rate": 4.980103557996915e-05, + "logits/chosen": -2.609720468521118, + "logits/rejected": -2.631443738937378, + "logps/chosen": -235.0199737548828, + "logps/rejected": -234.86245727539062, + "loss": 0.6376, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.2306605726480484, + "rewards/margins": 0.16340221464633942, + "rewards/rejected": -0.3940627872943878, + "step": 403 + }, + { + "epoch": 0.53, + "learning_rate": 4.9799721875513306e-05, + "logits/chosen": -2.283406972885132, + "logits/rejected": -2.386524200439453, + "logps/chosen": -226.02059936523438, + "logps/rejected": -220.51963806152344, + "loss": 0.7208, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5439196228981018, + "rewards/margins": 0.06111856549978256, + "rewards/rejected": -0.6050382256507874, + "step": 404 + }, + { + "epoch": 0.53, + "learning_rate": 4.979840386575311e-05, + "logits/chosen": -2.2238452434539795, + "logits/rejected": -2.3374826908111572, + "logps/chosen": -169.82667541503906, + "logps/rejected": -197.92726135253906, + "loss": 0.7292, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.46263885498046875, + "rewards/margins": 0.007239609956741333, + "rewards/rejected": -0.46987849473953247, + "step": 405 + }, + { + "epoch": 0.53, + "learning_rate": 4.979708155091737e-05, + "logits/chosen": -2.381389856338501, + "logits/rejected": -2.358883857727051, + "logps/chosen": -153.38771057128906, + "logps/rejected": -147.0846405029297, + "loss": 0.6555, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.2459157109260559, + "rewards/margins": 0.1274775117635727, + "rewards/rejected": -0.3733932375907898, + "step": 406 + }, + { + "epoch": 0.53, + "learning_rate": 4.979575493123566e-05, + "logits/chosen": -2.4942612648010254, + "logits/rejected": -2.4526309967041016, + "logps/chosen": -222.35780334472656, + "logps/rejected": -205.19630432128906, + "loss": 0.7278, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4296140670776367, + "rewards/margins": -0.018014922738075256, + "rewards/rejected": -0.41159915924072266, + "step": 407 + }, + { + "epoch": 0.53, + "learning_rate": 4.979442400693827e-05, + "logits/chosen": -2.4821577072143555, + "logits/rejected": -2.4502267837524414, + "logps/chosen": -157.32427978515625, + "logps/rejected": -167.9297332763672, + "loss": 0.7119, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4199235439300537, + "rewards/margins": 0.024679476395249367, + "rewards/rejected": -0.4446030259132385, + "step": 408 + }, + { + "epoch": 0.54, + "learning_rate": 4.979308877825626e-05, + "logits/chosen": -2.1102793216705322, + "logits/rejected": -2.100888729095459, + "logps/chosen": -235.02105712890625, + "logps/rejected": -232.0230712890625, + "loss": 0.747, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4274646043777466, + "rewards/margins": 0.0070232609286904335, + "rewards/rejected": -0.43448784947395325, + "step": 409 + }, + { + "epoch": 0.54, + "learning_rate": 4.9791749245421434e-05, + "logits/chosen": -2.4029147624969482, + "logits/rejected": -2.436694622039795, + "logps/chosen": -160.94296264648438, + "logps/rejected": -219.91183471679688, + "loss": 0.7659, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6857771873474121, + "rewards/margins": -0.07571852207183838, + "rewards/rejected": -0.6100587248802185, + "step": 410 + }, + { + "epoch": 0.54, + "learning_rate": 4.979040540866632e-05, + "logits/chosen": -2.423585891723633, + "logits/rejected": -2.583218574523926, + "logps/chosen": -152.8751220703125, + "logps/rejected": -195.98130798339844, + "loss": 0.6592, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.29784753918647766, + "rewards/margins": 0.13703800737857819, + "rewards/rejected": -0.43488556146621704, + "step": 411 + }, + { + "epoch": 0.54, + "learning_rate": 4.9789057268224234e-05, + "logits/chosen": -2.0688211917877197, + "logits/rejected": -2.1250336170196533, + "logps/chosen": -192.48826599121094, + "logps/rejected": -219.7627716064453, + "loss": 0.7074, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.16141924262046814, + "rewards/margins": 0.035948604345321655, + "rewards/rejected": -0.197367861866951, + "step": 412 + }, + { + "epoch": 0.54, + "learning_rate": 4.978770482432921e-05, + "logits/chosen": -2.4140584468841553, + "logits/rejected": -2.394195079803467, + "logps/chosen": -177.2361297607422, + "logps/rejected": -201.13475036621094, + "loss": 0.7109, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.41423746943473816, + "rewards/margins": 0.01436915248632431, + "rewards/rejected": -0.42860662937164307, + "step": 413 + }, + { + "epoch": 0.54, + "learning_rate": 4.9786348077216024e-05, + "logits/chosen": -2.399442672729492, + "logits/rejected": -2.4223692417144775, + "logps/chosen": -184.38839721679688, + "logps/rejected": -176.9226531982422, + "loss": 0.6572, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.2739775776863098, + "rewards/margins": 0.14736850559711456, + "rewards/rejected": -0.4213460683822632, + "step": 414 + }, + { + "epoch": 0.54, + "learning_rate": 4.9784987027120236e-05, + "logits/chosen": -2.331202745437622, + "logits/rejected": -2.3324198722839355, + "logps/chosen": -199.08740234375, + "logps/rejected": -171.50718688964844, + "loss": 0.7564, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4433014988899231, + "rewards/margins": -0.06983453035354614, + "rewards/rejected": -0.37346696853637695, + "step": 415 + }, + { + "epoch": 0.54, + "learning_rate": 4.9783621674278104e-05, + "logits/chosen": -2.402174949645996, + "logits/rejected": -2.5383033752441406, + "logps/chosen": -174.37954711914062, + "logps/rejected": -165.9345703125, + "loss": 0.6405, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5206565856933594, + "rewards/margins": 0.13473454117774963, + "rewards/rejected": -0.6553912162780762, + "step": 416 + }, + { + "epoch": 0.55, + "learning_rate": 4.978225201892667e-05, + "logits/chosen": -2.4111886024475098, + "logits/rejected": -2.3553833961486816, + "logps/chosen": -220.5755615234375, + "logps/rejected": -195.2405242919922, + "loss": 0.7531, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.4778462052345276, + "rewards/margins": -0.041694507002830505, + "rewards/rejected": -0.4361516833305359, + "step": 417 + }, + { + "epoch": 0.55, + "learning_rate": 4.97808780613037e-05, + "logits/chosen": -2.307465076446533, + "logits/rejected": -2.3099443912506104, + "logps/chosen": -171.25372314453125, + "logps/rejected": -200.85305786132812, + "loss": 0.6326, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.445933997631073, + "rewards/margins": 0.2173961102962494, + "rewards/rejected": -0.663330078125, + "step": 418 + }, + { + "epoch": 0.55, + "learning_rate": 4.977949980164773e-05, + "logits/chosen": -2.3666648864746094, + "logits/rejected": -2.433722496032715, + "logps/chosen": -217.6551513671875, + "logps/rejected": -234.6415557861328, + "loss": 0.5846, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3358496427536011, + "rewards/margins": 0.309315949678421, + "rewards/rejected": -0.6451655626296997, + "step": 419 + }, + { + "epoch": 0.55, + "learning_rate": 4.977811724019802e-05, + "logits/chosen": -2.336599111557007, + "logits/rejected": -2.409275770187378, + "logps/chosen": -191.2588653564453, + "logps/rejected": -208.126953125, + "loss": 0.6455, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4646199941635132, + "rewards/margins": 0.13836784660816193, + "rewards/rejected": -0.6029877662658691, + "step": 420 + }, + { + "epoch": 0.55, + "learning_rate": 4.9776730377194596e-05, + "logits/chosen": -2.1639509201049805, + "logits/rejected": -2.2069835662841797, + "logps/chosen": -203.13632202148438, + "logps/rejected": -169.3188018798828, + "loss": 0.7455, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.23145976662635803, + "rewards/margins": 0.013234168291091919, + "rewards/rejected": -0.24469394981861115, + "step": 421 + }, + { + "epoch": 0.55, + "learning_rate": 4.9775339212878215e-05, + "logits/chosen": -2.1670339107513428, + "logits/rejected": -2.1488430500030518, + "logps/chosen": -166.50318908691406, + "logps/rejected": -169.4437713623047, + "loss": 0.698, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.2539255619049072, + "rewards/margins": 0.030521919950842857, + "rewards/rejected": -0.28444746136665344, + "step": 422 + }, + { + "epoch": 0.55, + "learning_rate": 4.977394374749039e-05, + "logits/chosen": -2.336040735244751, + "logits/rejected": -2.3590757846832275, + "logps/chosen": -150.48399353027344, + "logps/rejected": -154.54652404785156, + "loss": 0.667, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.30479297041893005, + "rewards/margins": 0.10228873789310455, + "rewards/rejected": -0.4070816934108734, + "step": 423 + }, + { + "epoch": 0.55, + "learning_rate": 4.9772543981273374e-05, + "logits/chosen": -2.25268292427063, + "logits/rejected": -2.267082691192627, + "logps/chosen": -246.07342529296875, + "logps/rejected": -230.28424072265625, + "loss": 0.7204, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.42524397373199463, + "rewards/margins": -0.008991291746497154, + "rewards/rejected": -0.4162527322769165, + "step": 424 + }, + { + "epoch": 0.56, + "learning_rate": 4.977113991447017e-05, + "logits/chosen": -2.35402250289917, + "logits/rejected": -2.3599345684051514, + "logps/chosen": -163.17568969726562, + "logps/rejected": -175.94029235839844, + "loss": 0.7346, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.4037836790084839, + "rewards/margins": 0.02548382803797722, + "rewards/rejected": -0.4292675256729126, + "step": 425 + }, + { + "epoch": 0.56, + "learning_rate": 4.976973154732454e-05, + "logits/chosen": -2.3611769676208496, + "logits/rejected": -2.3299450874328613, + "logps/chosen": -205.6555633544922, + "logps/rejected": -213.8665771484375, + "loss": 0.7124, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.22228708863258362, + "rewards/margins": 0.031078480184078217, + "rewards/rejected": -0.25336551666259766, + "step": 426 + }, + { + "epoch": 0.56, + "learning_rate": 4.976831888008096e-05, + "logits/chosen": -2.316866636276245, + "logits/rejected": -2.299907684326172, + "logps/chosen": -168.08087158203125, + "logps/rejected": -160.0249786376953, + "loss": 0.6877, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.13129538297653198, + "rewards/margins": 0.04759233817458153, + "rewards/rejected": -0.17888770997524261, + "step": 427 + }, + { + "epoch": 0.56, + "learning_rate": 4.976690191298469e-05, + "logits/chosen": -2.1294949054718018, + "logits/rejected": -2.1100010871887207, + "logps/chosen": -176.1876678466797, + "logps/rejected": -175.78619384765625, + "loss": 0.7597, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.16827335953712463, + "rewards/margins": -0.110628642141819, + "rewards/rejected": -0.05764475464820862, + "step": 428 + }, + { + "epoch": 0.56, + "learning_rate": 4.9765480646281716e-05, + "logits/chosen": -2.120499849319458, + "logits/rejected": -2.2186028957366943, + "logps/chosen": -178.4534912109375, + "logps/rejected": -200.6439971923828, + "loss": 0.6143, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11943019181489944, + "rewards/margins": 0.28324708342552185, + "rewards/rejected": -0.4026772677898407, + "step": 429 + }, + { + "epoch": 0.56, + "learning_rate": 4.976405508021877e-05, + "logits/chosen": -2.430680274963379, + "logits/rejected": -2.5126051902770996, + "logps/chosen": -201.0321502685547, + "logps/rejected": -231.42811584472656, + "loss": 0.7632, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.47548148036003113, + "rewards/margins": -0.046219632029533386, + "rewards/rejected": -0.42926183342933655, + "step": 430 + }, + { + "epoch": 0.56, + "learning_rate": 4.9762625215043334e-05, + "logits/chosen": -2.396533966064453, + "logits/rejected": -2.4125595092773438, + "logps/chosen": -236.99049377441406, + "logps/rejected": -253.93577575683594, + "loss": 0.7165, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4821574091911316, + "rewards/margins": -0.007029315456748009, + "rewards/rejected": -0.4751281142234802, + "step": 431 + }, + { + "epoch": 0.57, + "learning_rate": 4.9761191051003644e-05, + "logits/chosen": -2.1031394004821777, + "logits/rejected": -2.143960475921631, + "logps/chosen": -168.68551635742188, + "logps/rejected": -209.39537048339844, + "loss": 0.6219, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1691710501909256, + "rewards/margins": 0.2633129060268402, + "rewards/rejected": -0.4324839413166046, + "step": 432 + }, + { + "epoch": 0.57, + "learning_rate": 4.975975258834867e-05, + "logits/chosen": -2.451735496520996, + "logits/rejected": -2.410836696624756, + "logps/chosen": -219.5807342529297, + "logps/rejected": -238.2914276123047, + "loss": 0.6939, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.26019036769866943, + "rewards/margins": 0.09737718105316162, + "rewards/rejected": -0.35756757855415344, + "step": 433 + }, + { + "epoch": 0.57, + "learning_rate": 4.9758309827328134e-05, + "logits/chosen": -2.3229422569274902, + "logits/rejected": -2.3514108657836914, + "logps/chosen": -181.05836486816406, + "logps/rejected": -211.49786376953125, + "loss": 0.7734, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.2722405195236206, + "rewards/margins": -0.07979361712932587, + "rewards/rejected": -0.19244688749313354, + "step": 434 + }, + { + "epoch": 0.57, + "learning_rate": 4.9756862768192504e-05, + "logits/chosen": -2.3667490482330322, + "logits/rejected": -2.328806161880493, + "logps/chosen": -201.50511169433594, + "logps/rejected": -179.98643493652344, + "loss": 0.7353, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.3051126301288605, + "rewards/margins": -0.007734470069408417, + "rewards/rejected": -0.29737818241119385, + "step": 435 + }, + { + "epoch": 0.57, + "learning_rate": 4.9755411411192996e-05, + "logits/chosen": -2.4068377017974854, + "logits/rejected": -2.532482862472534, + "logps/chosen": -186.65342712402344, + "logps/rejected": -205.83154296875, + "loss": 0.7144, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.17129601538181305, + "rewards/margins": -0.002290443517267704, + "rewards/rejected": -0.1690055876970291, + "step": 436 + }, + { + "epoch": 0.57, + "learning_rate": 4.975395575658156e-05, + "logits/chosen": -2.382935047149658, + "logits/rejected": -2.4527859687805176, + "logps/chosen": -200.10215759277344, + "logps/rejected": -207.49302673339844, + "loss": 0.7113, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.33162015676498413, + "rewards/margins": 0.029873518273234367, + "rewards/rejected": -0.36149367690086365, + "step": 437 + }, + { + "epoch": 0.57, + "learning_rate": 4.9752495804610916e-05, + "logits/chosen": -2.2459821701049805, + "logits/rejected": -2.2536497116088867, + "logps/chosen": -249.44638061523438, + "logps/rejected": -216.59483337402344, + "loss": 0.8425, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.3279973268508911, + "rewards/margins": -0.22765566408634186, + "rewards/rejected": -0.10034167766571045, + "step": 438 + }, + { + "epoch": 0.57, + "learning_rate": 4.9751031555534504e-05, + "logits/chosen": -2.416114330291748, + "logits/rejected": -2.5028560161590576, + "logps/chosen": -186.53961181640625, + "logps/rejected": -160.81617736816406, + "loss": 0.674, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.1654774248600006, + "rewards/margins": 0.07480952888727188, + "rewards/rejected": -0.2402869611978531, + "step": 439 + }, + { + "epoch": 0.58, + "learning_rate": 4.9749563009606534e-05, + "logits/chosen": -2.4489054679870605, + "logits/rejected": -2.527320623397827, + "logps/chosen": -202.46507263183594, + "logps/rejected": -185.72853088378906, + "loss": 0.6135, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.22387953102588654, + "rewards/margins": 0.19476865231990814, + "rewards/rejected": -0.4186481833457947, + "step": 440 + }, + { + "epoch": 0.58, + "learning_rate": 4.9748090167081936e-05, + "logits/chosen": -2.147719144821167, + "logits/rejected": -2.249868154525757, + "logps/chosen": -152.69786071777344, + "logps/rejected": -190.48863220214844, + "loss": 0.6568, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.29554668068885803, + "rewards/margins": 0.11645788699388504, + "rewards/rejected": -0.4120045602321625, + "step": 441 + }, + { + "epoch": 0.58, + "learning_rate": 4.974661302821641e-05, + "logits/chosen": -2.142338275909424, + "logits/rejected": -2.2330148220062256, + "logps/chosen": -157.3348388671875, + "logps/rejected": -194.91302490234375, + "loss": 0.7378, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.13629794120788574, + "rewards/margins": -0.030040191486477852, + "rewards/rejected": -0.10625775158405304, + "step": 442 + }, + { + "epoch": 0.58, + "learning_rate": 4.974513159326638e-05, + "logits/chosen": -2.551227569580078, + "logits/rejected": -2.5135083198547363, + "logps/chosen": -201.37612915039062, + "logps/rejected": -167.43287658691406, + "loss": 0.6417, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.25598540902137756, + "rewards/margins": 0.18629388511180878, + "rewards/rejected": -0.4422793388366699, + "step": 443 + }, + { + "epoch": 0.58, + "learning_rate": 4.974364586248904e-05, + "logits/chosen": -2.174506425857544, + "logits/rejected": -2.1649577617645264, + "logps/chosen": -181.4037628173828, + "logps/rejected": -171.9210968017578, + "loss": 0.8572, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3619764745235443, + "rewards/margins": -0.19866357743740082, + "rewards/rejected": -0.1633128821849823, + "step": 444 + }, + { + "epoch": 0.58, + "learning_rate": 4.974215583614232e-05, + "logits/chosen": -2.3266444206237793, + "logits/rejected": -2.418710470199585, + "logps/chosen": -207.11448669433594, + "logps/rejected": -287.95574951171875, + "loss": 0.7347, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.41178515553474426, + "rewards/margins": -0.002230718731880188, + "rewards/rejected": -0.40955445170402527, + "step": 445 + }, + { + "epoch": 0.58, + "learning_rate": 4.974066151448488e-05, + "logits/chosen": -2.5598104000091553, + "logits/rejected": -2.559662342071533, + "logps/chosen": -231.59133911132812, + "logps/rejected": -239.7825469970703, + "loss": 0.7079, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.42168766260147095, + "rewards/margins": 0.02927880734205246, + "rewards/rejected": -0.4509664475917816, + "step": 446 + }, + { + "epoch": 0.58, + "learning_rate": 4.9739162897776146e-05, + "logits/chosen": -2.3569555282592773, + "logits/rejected": -2.3967788219451904, + "logps/chosen": -185.6054229736328, + "logps/rejected": -192.92169189453125, + "loss": 0.7437, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4934594929218292, + "rewards/margins": -0.05449381470680237, + "rewards/rejected": -0.43896567821502686, + "step": 447 + }, + { + "epoch": 0.59, + "learning_rate": 4.973765998627628e-05, + "logits/chosen": -2.5715792179107666, + "logits/rejected": -2.5525949001312256, + "logps/chosen": -252.23037719726562, + "logps/rejected": -268.3929443359375, + "loss": 0.7203, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3784821331501007, + "rewards/margins": -0.01087496429681778, + "rewards/rejected": -0.3676071763038635, + "step": 448 + }, + { + "epoch": 0.59, + "learning_rate": 4.973615278024619e-05, + "logits/chosen": -2.441962718963623, + "logits/rejected": -2.481163501739502, + "logps/chosen": -161.66839599609375, + "logps/rejected": -192.90371704101562, + "loss": 0.6712, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3044736981391907, + "rewards/margins": 0.08197857439517975, + "rewards/rejected": -0.38645222783088684, + "step": 449 + }, + { + "epoch": 0.59, + "learning_rate": 4.9734641279947535e-05, + "logits/chosen": -2.500675916671753, + "logits/rejected": -2.487545967102051, + "logps/chosen": -215.66336059570312, + "logps/rejected": -232.06967163085938, + "loss": 0.6796, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4935031235218048, + "rewards/margins": 0.12285958230495453, + "rewards/rejected": -0.6163626909255981, + "step": 450 + }, + { + "epoch": 0.59, + "learning_rate": 4.973312548564272e-05, + "logits/chosen": -2.4314112663269043, + "logits/rejected": -2.4294910430908203, + "logps/chosen": -170.31471252441406, + "logps/rejected": -159.71585083007812, + "loss": 0.7036, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.32274457812309265, + "rewards/margins": 0.005533523857593536, + "rewards/rejected": -0.328278124332428, + "step": 451 + }, + { + "epoch": 0.59, + "learning_rate": 4.9731605397594884e-05, + "logits/chosen": -2.4075756072998047, + "logits/rejected": -2.32165789604187, + "logps/chosen": -186.2813720703125, + "logps/rejected": -199.41989135742188, + "loss": 0.655, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3598267138004303, + "rewards/margins": 0.11941798776388168, + "rewards/rejected": -0.4792447090148926, + "step": 452 + }, + { + "epoch": 0.59, + "learning_rate": 4.973008101606792e-05, + "logits/chosen": -2.422717571258545, + "logits/rejected": -2.5152597427368164, + "logps/chosen": -173.51361083984375, + "logps/rejected": -203.63848876953125, + "loss": 0.7204, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.25253090262413025, + "rewards/margins": 0.029171746224164963, + "rewards/rejected": -0.2817026376724243, + "step": 453 + }, + { + "epoch": 0.59, + "learning_rate": 4.972855234132646e-05, + "logits/chosen": -2.517770767211914, + "logits/rejected": -2.501011848449707, + "logps/chosen": -202.05300903320312, + "logps/rejected": -191.65887451171875, + "loss": 0.7912, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.42360594868659973, + "rewards/margins": -0.1400626003742218, + "rewards/rejected": -0.28354334831237793, + "step": 454 + }, + { + "epoch": 0.6, + "learning_rate": 4.9727019373635895e-05, + "logits/chosen": -2.612367630004883, + "logits/rejected": -2.617859125137329, + "logps/chosen": -210.05746459960938, + "logps/rejected": -199.4877166748047, + "loss": 0.621, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12031766027212143, + "rewards/margins": 0.193304181098938, + "rewards/rejected": -0.3136218190193176, + "step": 455 + }, + { + "epoch": 0.6, + "learning_rate": 4.972548211326235e-05, + "logits/chosen": -2.5540568828582764, + "logits/rejected": -2.587779998779297, + "logps/chosen": -183.94273376464844, + "logps/rejected": -219.0413818359375, + "loss": 0.6669, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.1985998898744583, + "rewards/margins": 0.12929418683052063, + "rewards/rejected": -0.32789406180381775, + "step": 456 + }, + { + "epoch": 0.6, + "learning_rate": 4.9723940560472705e-05, + "logits/chosen": -2.573051691055298, + "logits/rejected": -2.6791186332702637, + "logps/chosen": -193.5257568359375, + "logps/rejected": -189.68743896484375, + "loss": 0.7393, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.31839922070503235, + "rewards/margins": -0.04755366966128349, + "rewards/rejected": -0.27084553241729736, + "step": 457 + }, + { + "epoch": 0.6, + "learning_rate": 4.972239471553457e-05, + "logits/chosen": -2.127181053161621, + "logits/rejected": -2.1366987228393555, + "logps/chosen": -180.3720703125, + "logps/rejected": -206.37628173828125, + "loss": 0.6858, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.14177769422531128, + "rewards/margins": 0.04955483227968216, + "rewards/rejected": -0.19133250415325165, + "step": 458 + }, + { + "epoch": 0.6, + "learning_rate": 4.97208445787163e-05, + "logits/chosen": -2.4841840267181396, + "logits/rejected": -2.576199531555176, + "logps/chosen": -175.8242645263672, + "logps/rejected": -205.9224090576172, + "loss": 0.7229, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.39802610874176025, + "rewards/margins": -0.00046849995851516724, + "rewards/rejected": -0.3975576162338257, + "step": 459 + }, + { + "epoch": 0.6, + "learning_rate": 4.9719290150287026e-05, + "logits/chosen": -2.4974851608276367, + "logits/rejected": -2.552859306335449, + "logps/chosen": -204.64480590820312, + "logps/rejected": -229.373291015625, + "loss": 0.7028, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.09486077725887299, + "rewards/margins": 0.017538849264383316, + "rewards/rejected": -0.1123996153473854, + "step": 460 + }, + { + "epoch": 0.6, + "learning_rate": 4.9717731430516576e-05, + "logits/chosen": -2.5486257076263428, + "logits/rejected": -2.505126953125, + "logps/chosen": -216.0121307373047, + "logps/rejected": -204.5452880859375, + "loss": 0.6996, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1612711399793625, + "rewards/margins": 0.06856641173362732, + "rewards/rejected": -0.229837566614151, + "step": 461 + }, + { + "epoch": 0.6, + "learning_rate": 4.9716168419675555e-05, + "logits/chosen": -2.5866286754608154, + "logits/rejected": -2.667116403579712, + "logps/chosen": -198.39340209960938, + "logps/rejected": -250.51675415039062, + "loss": 0.6783, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3453162908554077, + "rewards/margins": 0.07628250867128372, + "rewards/rejected": -0.42159876227378845, + "step": 462 + }, + { + "epoch": 0.61, + "learning_rate": 4.9714601118035325e-05, + "logits/chosen": -2.339556932449341, + "logits/rejected": -2.2843379974365234, + "logps/chosen": -202.150634765625, + "logps/rejected": -192.74267578125, + "loss": 0.674, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1799849569797516, + "rewards/margins": 0.09412900358438492, + "rewards/rejected": -0.2741139531135559, + "step": 463 + }, + { + "epoch": 0.61, + "learning_rate": 4.971302952586796e-05, + "logits/chosen": -2.3514528274536133, + "logits/rejected": -2.313333749771118, + "logps/chosen": -216.48309326171875, + "logps/rejected": -205.1566925048828, + "loss": 0.7625, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.3306673467159271, + "rewards/margins": -0.07343533635139465, + "rewards/rejected": -0.25723204016685486, + "step": 464 + }, + { + "epoch": 0.61, + "learning_rate": 4.971145364344628e-05, + "logits/chosen": -2.475745916366577, + "logits/rejected": -2.5058016777038574, + "logps/chosen": -198.23568725585938, + "logps/rejected": -226.68544006347656, + "loss": 0.6412, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.38944318890571594, + "rewards/margins": 0.15158149600028992, + "rewards/rejected": -0.5410246849060059, + "step": 465 + }, + { + "epoch": 0.61, + "learning_rate": 4.970987347104389e-05, + "logits/chosen": -2.490628242492676, + "logits/rejected": -2.5108678340911865, + "logps/chosen": -180.23779296875, + "logps/rejected": -232.67515563964844, + "loss": 0.6863, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.3212130665779114, + "rewards/margins": 0.09368535876274109, + "rewards/rejected": -0.41489848494529724, + "step": 466 + }, + { + "epoch": 0.61, + "learning_rate": 4.9708289008935096e-05, + "logits/chosen": -2.6224091053009033, + "logits/rejected": -2.6361618041992188, + "logps/chosen": -254.05960083007812, + "logps/rejected": -247.78634643554688, + "loss": 0.6909, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.40414828062057495, + "rewards/margins": 0.051116712391376495, + "rewards/rejected": -0.4552650451660156, + "step": 467 + }, + { + "epoch": 0.61, + "learning_rate": 4.9706700257394966e-05, + "logits/chosen": -2.6673390865325928, + "logits/rejected": -2.6409029960632324, + "logps/chosen": -160.94146728515625, + "logps/rejected": -228.09808349609375, + "loss": 0.7103, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4117045998573303, + "rewards/margins": 0.047504693269729614, + "rewards/rejected": -0.45920926332473755, + "step": 468 + }, + { + "epoch": 0.61, + "learning_rate": 4.970510721669932e-05, + "logits/chosen": -2.12448787689209, + "logits/rejected": -2.209928512573242, + "logps/chosen": -247.7496337890625, + "logps/rejected": -256.54254150390625, + "loss": 0.602, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.13797114789485931, + "rewards/margins": 0.25247055292129517, + "rewards/rejected": -0.3904416561126709, + "step": 469 + }, + { + "epoch": 0.62, + "learning_rate": 4.97035098871247e-05, + "logits/chosen": -2.531524896621704, + "logits/rejected": -2.5506911277770996, + "logps/chosen": -239.89340209960938, + "logps/rejected": -232.53662109375, + "loss": 0.6477, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.2409316897392273, + "rewards/margins": 0.11869845539331436, + "rewards/rejected": -0.35963016748428345, + "step": 470 + }, + { + "epoch": 0.62, + "learning_rate": 4.970190826894842e-05, + "logits/chosen": -2.4936788082122803, + "logits/rejected": -2.6132445335388184, + "logps/chosen": -193.3863067626953, + "logps/rejected": -196.8754119873047, + "loss": 0.6012, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.33218735456466675, + "rewards/margins": 0.244198739528656, + "rewards/rejected": -0.5763860940933228, + "step": 471 + }, + { + "epoch": 0.62, + "learning_rate": 4.9700302362448517e-05, + "logits/chosen": -2.7358896732330322, + "logits/rejected": -2.860752582550049, + "logps/chosen": -193.43789672851562, + "logps/rejected": -240.47451782226562, + "loss": 0.6757, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.2744450271129608, + "rewards/margins": 0.07577859610319138, + "rewards/rejected": -0.3502236306667328, + "step": 472 + }, + { + "epoch": 0.62, + "learning_rate": 4.9698692167903794e-05, + "logits/chosen": -2.581242799758911, + "logits/rejected": -2.7139995098114014, + "logps/chosen": -206.38645935058594, + "logps/rejected": -210.1461639404297, + "loss": 0.5954, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.3665695786476135, + "rewards/margins": 0.26593247056007385, + "rewards/rejected": -0.632502019405365, + "step": 473 + }, + { + "epoch": 0.62, + "learning_rate": 4.9697077685593766e-05, + "logits/chosen": -2.6083261966705322, + "logits/rejected": -2.6498053073883057, + "logps/chosen": -131.37237548828125, + "logps/rejected": -151.5311737060547, + "loss": 0.649, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15027746558189392, + "rewards/margins": 0.126561239361763, + "rewards/rejected": -0.2768386900424957, + "step": 474 + }, + { + "epoch": 0.62, + "learning_rate": 4.969545891579873e-05, + "logits/chosen": -2.359971046447754, + "logits/rejected": -2.3982698917388916, + "logps/chosen": -157.39181518554688, + "logps/rejected": -158.3177032470703, + "loss": 0.6222, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.10072011500597, + "rewards/margins": 0.24248063564300537, + "rewards/rejected": -0.34320077300071716, + "step": 475 + }, + { + "epoch": 0.62, + "learning_rate": 4.9693835858799696e-05, + "logits/chosen": -2.408668279647827, + "logits/rejected": -2.4540035724639893, + "logps/chosen": -190.36312866210938, + "logps/rejected": -204.20359802246094, + "loss": 0.6997, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.18467962741851807, + "rewards/margins": 0.015133664011955261, + "rewards/rejected": -0.19981329143047333, + "step": 476 + }, + { + "epoch": 0.62, + "learning_rate": 4.9692208514878444e-05, + "logits/chosen": -2.270054340362549, + "logits/rejected": -2.177553653717041, + "logps/chosen": -197.311767578125, + "logps/rejected": -205.99948120117188, + "loss": 0.7255, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.2719980776309967, + "rewards/margins": -0.018820375204086304, + "rewards/rejected": -0.2531776428222656, + "step": 477 + }, + { + "epoch": 0.63, + "learning_rate": 4.969057688431748e-05, + "logits/chosen": -2.4876270294189453, + "logits/rejected": -2.516055107116699, + "logps/chosen": -150.5498046875, + "logps/rejected": -173.20225524902344, + "loss": 0.6482, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.2806074023246765, + "rewards/margins": 0.18431711196899414, + "rewards/rejected": -0.46492451429367065, + "step": 478 + }, + { + "epoch": 0.63, + "learning_rate": 4.968894096740006e-05, + "logits/chosen": -2.699005365371704, + "logits/rejected": -2.706855297088623, + "logps/chosen": -208.8563232421875, + "logps/rejected": -229.45355224609375, + "loss": 0.6569, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3221612572669983, + "rewards/margins": 0.1469050496816635, + "rewards/rejected": -0.4690663516521454, + "step": 479 + }, + { + "epoch": 0.63, + "learning_rate": 4.968730076441017e-05, + "logits/chosen": -2.752363681793213, + "logits/rejected": -2.8258612155914307, + "logps/chosen": -233.5675811767578, + "logps/rejected": -261.7062683105469, + "loss": 0.7128, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.604475200176239, + "rewards/margins": 0.022365085780620575, + "rewards/rejected": -0.6268402338027954, + "step": 480 + }, + { + "epoch": 0.63, + "learning_rate": 4.9685656275632575e-05, + "logits/chosen": -2.5177550315856934, + "logits/rejected": -2.510917901992798, + "logps/chosen": -217.02195739746094, + "logps/rejected": -189.55174255371094, + "loss": 0.6434, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3391067683696747, + "rewards/margins": 0.13497701287269592, + "rewards/rejected": -0.4740837812423706, + "step": 481 + }, + { + "epoch": 0.63, + "learning_rate": 4.968400750135276e-05, + "logits/chosen": -2.540860652923584, + "logits/rejected": -2.582563877105713, + "logps/chosen": -194.17721557617188, + "logps/rejected": -198.79684448242188, + "loss": 0.6679, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.574460506439209, + "rewards/margins": 0.10229263454675674, + "rewards/rejected": -0.6767531633377075, + "step": 482 + }, + { + "epoch": 0.63, + "learning_rate": 4.968235444185695e-05, + "logits/chosen": -2.7020184993743896, + "logits/rejected": -2.7175564765930176, + "logps/chosen": -211.08863830566406, + "logps/rejected": -235.41836547851562, + "loss": 0.7235, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5369959473609924, + "rewards/margins": 0.011504769325256348, + "rewards/rejected": -0.5485007762908936, + "step": 483 + }, + { + "epoch": 0.63, + "learning_rate": 4.968069709743212e-05, + "logits/chosen": -2.4362404346466064, + "logits/rejected": -2.5247411727905273, + "logps/chosen": -149.7013397216797, + "logps/rejected": -185.23251342773438, + "loss": 0.5556, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.1017133966088295, + "rewards/margins": 0.3292151391506195, + "rewards/rejected": -0.4309285283088684, + "step": 484 + }, + { + "epoch": 0.63, + "learning_rate": 4.9679035468365986e-05, + "logits/chosen": -2.3711907863616943, + "logits/rejected": -2.412506580352783, + "logps/chosen": -202.7202911376953, + "logps/rejected": -219.67977905273438, + "loss": 0.6658, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3317970931529999, + "rewards/margins": 0.12470696866512299, + "rewards/rejected": -0.45650404691696167, + "step": 485 + }, + { + "epoch": 0.64, + "learning_rate": 4.967736955494703e-05, + "logits/chosen": -2.204392910003662, + "logits/rejected": -2.2046332359313965, + "logps/chosen": -195.41537475585938, + "logps/rejected": -196.8472442626953, + "loss": 0.7358, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.43655478954315186, + "rewards/margins": -0.000651337206363678, + "rewards/rejected": -0.4359034597873688, + "step": 486 + }, + { + "epoch": 0.64, + "learning_rate": 4.9675699357464445e-05, + "logits/chosen": -2.198075771331787, + "logits/rejected": -2.1894662380218506, + "logps/chosen": -248.62149047851562, + "logps/rejected": -267.9733581542969, + "loss": 0.5947, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14261871576309204, + "rewards/margins": 0.31586068868637085, + "rewards/rejected": -0.4584794044494629, + "step": 487 + }, + { + "epoch": 0.64, + "learning_rate": 4.967402487620818e-05, + "logits/chosen": -2.5536584854125977, + "logits/rejected": -2.577322006225586, + "logps/chosen": -224.8876190185547, + "logps/rejected": -227.71383666992188, + "loss": 0.663, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.40421125292778015, + "rewards/margins": 0.11385571211576462, + "rewards/rejected": -0.5180670022964478, + "step": 488 + }, + { + "epoch": 0.64, + "learning_rate": 4.9672346111468934e-05, + "logits/chosen": -2.508915424346924, + "logits/rejected": -2.624011993408203, + "logps/chosen": -181.42379760742188, + "logps/rejected": -231.30458068847656, + "loss": 0.6309, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.31741198897361755, + "rewards/margins": 0.24159038066864014, + "rewards/rejected": -0.5590023994445801, + "step": 489 + }, + { + "epoch": 0.64, + "learning_rate": 4.967066306353816e-05, + "logits/chosen": -2.4504952430725098, + "logits/rejected": -2.4388234615325928, + "logps/chosen": -229.2549591064453, + "logps/rejected": -238.2850341796875, + "loss": 0.6922, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.47322726249694824, + "rewards/margins": 0.07009989768266678, + "rewards/rejected": -0.5433271527290344, + "step": 490 + }, + { + "epoch": 0.64, + "learning_rate": 4.966897573270801e-05, + "logits/chosen": -2.555962562561035, + "logits/rejected": -2.5629987716674805, + "logps/chosen": -172.14407348632812, + "logps/rejected": -168.1295166015625, + "loss": 0.7069, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.2661295533180237, + "rewards/margins": 0.07032840698957443, + "rewards/rejected": -0.3364579379558563, + "step": 491 + }, + { + "epoch": 0.64, + "learning_rate": 4.966728411927144e-05, + "logits/chosen": -2.4634742736816406, + "logits/rejected": -2.4717013835906982, + "logps/chosen": -206.756591796875, + "logps/rejected": -202.884033203125, + "loss": 0.7013, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.5119624733924866, + "rewards/margins": 0.043263014405965805, + "rewards/rejected": -0.5552253723144531, + "step": 492 + }, + { + "epoch": 0.65, + "learning_rate": 4.9665588223522096e-05, + "logits/chosen": -2.333207607269287, + "logits/rejected": -2.3922958374023438, + "logps/chosen": -210.04153442382812, + "logps/rejected": -239.76979064941406, + "loss": 0.7059, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.32687515020370483, + "rewards/margins": 0.09048546105623245, + "rewards/rejected": -0.41736066341400146, + "step": 493 + }, + { + "epoch": 0.65, + "learning_rate": 4.96638880457544e-05, + "logits/chosen": -2.414531707763672, + "logits/rejected": -2.4198668003082275, + "logps/chosen": -225.68203735351562, + "logps/rejected": -252.04661560058594, + "loss": 0.6259, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.44954052567481995, + "rewards/margins": 0.19488830864429474, + "rewards/rejected": -0.6444287896156311, + "step": 494 + }, + { + "epoch": 0.65, + "learning_rate": 4.9662183586263514e-05, + "logits/chosen": -2.523651361465454, + "logits/rejected": -2.4278194904327393, + "logps/chosen": -222.92274475097656, + "logps/rejected": -203.31668090820312, + "loss": 0.8255, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6954134106636047, + "rewards/margins": -0.12540754675865173, + "rewards/rejected": -0.5700058937072754, + "step": 495 + }, + { + "epoch": 0.65, + "learning_rate": 4.966047484534533e-05, + "logits/chosen": -2.465467929840088, + "logits/rejected": -2.4773805141448975, + "logps/chosen": -215.38780212402344, + "logps/rejected": -222.23593139648438, + "loss": 0.6348, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.43675118684768677, + "rewards/margins": 0.19925275444984436, + "rewards/rejected": -0.6360039710998535, + "step": 496 + }, + { + "epoch": 0.65, + "learning_rate": 4.965876182329648e-05, + "logits/chosen": -2.4241979122161865, + "logits/rejected": -2.4904754161834717, + "logps/chosen": -176.40281677246094, + "logps/rejected": -189.89398193359375, + "loss": 0.7316, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.35698071122169495, + "rewards/margins": 0.12830819189548492, + "rewards/rejected": -0.48528894782066345, + "step": 497 + }, + { + "epoch": 0.65, + "learning_rate": 4.965704452041437e-05, + "logits/chosen": -2.4424967765808105, + "logits/rejected": -2.447554588317871, + "logps/chosen": -186.7346954345703, + "logps/rejected": -202.56729125976562, + "loss": 0.6988, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.379626601934433, + "rewards/margins": 0.05914265662431717, + "rewards/rejected": -0.43876922130584717, + "step": 498 + }, + { + "epoch": 0.65, + "learning_rate": 4.9655322936997115e-05, + "logits/chosen": -2.384314775466919, + "logits/rejected": -2.6753270626068115, + "logps/chosen": -168.5001220703125, + "logps/rejected": -201.58462524414062, + "loss": 0.5711, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.30074769258499146, + "rewards/margins": 0.31708773970603943, + "rewards/rejected": -0.6178354024887085, + "step": 499 + }, + { + "epoch": 0.65, + "learning_rate": 4.9653597073343594e-05, + "logits/chosen": -2.44107723236084, + "logits/rejected": -2.518571615219116, + "logps/chosen": -190.8749542236328, + "logps/rejected": -195.59471130371094, + "loss": 0.6465, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.3541903495788574, + "rewards/margins": 0.2186165750026703, + "rewards/rejected": -0.5728069543838501, + "step": 500 + }, + { + "epoch": 0.66, + "learning_rate": 4.9651866929753424e-05, + "logits/chosen": -2.580932140350342, + "logits/rejected": -2.6375184059143066, + "logps/chosen": -201.61231994628906, + "logps/rejected": -195.1832275390625, + "loss": 0.6788, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4448980689048767, + "rewards/margins": 0.1541019082069397, + "rewards/rejected": -0.5990000367164612, + "step": 501 + }, + { + "epoch": 0.66, + "learning_rate": 4.9650132506526944e-05, + "logits/chosen": -2.601297378540039, + "logits/rejected": -2.5584542751312256, + "logps/chosen": -188.46102905273438, + "logps/rejected": -176.1941680908203, + "loss": 0.6884, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.32628124952316284, + "rewards/margins": 0.053992465138435364, + "rewards/rejected": -0.3802737295627594, + "step": 502 + }, + { + "epoch": 0.66, + "learning_rate": 4.9648393803965284e-05, + "logits/chosen": -2.594449281692505, + "logits/rejected": -2.6243650913238525, + "logps/chosen": -180.65838623046875, + "logps/rejected": -173.06399536132812, + "loss": 0.608, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.3709149658679962, + "rewards/margins": 0.24527978897094727, + "rewards/rejected": -0.6161947846412659, + "step": 503 + }, + { + "epoch": 0.66, + "learning_rate": 4.9646650822370265e-05, + "logits/chosen": -2.623140811920166, + "logits/rejected": -2.6087071895599365, + "logps/chosen": -175.10757446289062, + "logps/rejected": -198.5271759033203, + "loss": 0.6623, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.34849774837493896, + "rewards/margins": 0.2720443308353424, + "rewards/rejected": -0.6205421090126038, + "step": 504 + }, + { + "epoch": 0.66, + "learning_rate": 4.964490356204449e-05, + "logits/chosen": -2.3962643146514893, + "logits/rejected": -2.4296982288360596, + "logps/chosen": -171.71841430664062, + "logps/rejected": -189.31715393066406, + "loss": 0.7731, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5484548211097717, + "rewards/margins": 0.05871535837650299, + "rewards/rejected": -0.6071702241897583, + "step": 505 + }, + { + "epoch": 0.66, + "learning_rate": 4.964315202329127e-05, + "logits/chosen": -2.1054232120513916, + "logits/rejected": -2.0613820552825928, + "logps/chosen": -196.4398956298828, + "logps/rejected": -197.6771240234375, + "loss": 0.6993, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4299621880054474, + "rewards/margins": 0.10146878659725189, + "rewards/rejected": -0.5314310193061829, + "step": 506 + }, + { + "epoch": 0.66, + "learning_rate": 4.96413962064147e-05, + "logits/chosen": -2.4437365531921387, + "logits/rejected": -2.6009879112243652, + "logps/chosen": -218.52403259277344, + "logps/rejected": -253.1461639404297, + "loss": 0.5733, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.4768885374069214, + "rewards/margins": 0.29149848222732544, + "rewards/rejected": -0.7683870196342468, + "step": 507 + }, + { + "epoch": 0.66, + "learning_rate": 4.963963611171957e-05, + "logits/chosen": -2.737818479537964, + "logits/rejected": -2.8296992778778076, + "logps/chosen": -194.77084350585938, + "logps/rejected": -196.74485778808594, + "loss": 0.6388, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4071928560733795, + "rewards/margins": 0.17611998319625854, + "rewards/rejected": -0.5833128690719604, + "step": 508 + }, + { + "epoch": 0.67, + "learning_rate": 4.9637871739511454e-05, + "logits/chosen": -2.5671932697296143, + "logits/rejected": -2.4917585849761963, + "logps/chosen": -208.9228057861328, + "logps/rejected": -204.2896270751953, + "loss": 0.7151, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.38905590772628784, + "rewards/margins": 0.02341422438621521, + "rewards/rejected": -0.41247016191482544, + "step": 509 + }, + { + "epoch": 0.67, + "learning_rate": 4.963610309009665e-05, + "logits/chosen": -2.3518598079681396, + "logits/rejected": -2.3721611499786377, + "logps/chosen": -154.779296875, + "logps/rejected": -190.75613403320312, + "loss": 0.6552, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.35764750838279724, + "rewards/margins": 0.1814635694026947, + "rewards/rejected": -0.5391110777854919, + "step": 510 + }, + { + "epoch": 0.67, + "learning_rate": 4.9634330163782205e-05, + "logits/chosen": -2.432107448577881, + "logits/rejected": -2.495102882385254, + "logps/chosen": -248.51341247558594, + "logps/rejected": -233.74266052246094, + "loss": 0.7034, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.45434871315956116, + "rewards/margins": 0.08803940564393997, + "rewards/rejected": -0.5423881411552429, + "step": 511 + }, + { + "epoch": 0.67, + "learning_rate": 4.963255296087589e-05, + "logits/chosen": -2.4448506832122803, + "logits/rejected": -2.462013006210327, + "logps/chosen": -143.73773193359375, + "logps/rejected": -182.57113647460938, + "loss": 0.6068, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.3137112855911255, + "rewards/margins": 0.21627672016620636, + "rewards/rejected": -0.5299879908561707, + "step": 512 + }, + { + "epoch": 0.67, + "learning_rate": 4.9630771481686244e-05, + "logits/chosen": -2.4147915840148926, + "logits/rejected": -2.368267297744751, + "logps/chosen": -189.82333374023438, + "logps/rejected": -175.15501403808594, + "loss": 0.6099, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.34441184997558594, + "rewards/margins": 0.29255932569503784, + "rewards/rejected": -0.636971116065979, + "step": 513 + }, + { + "epoch": 0.67, + "learning_rate": 4.9628985726522535e-05, + "logits/chosen": -2.424818992614746, + "logits/rejected": -2.4481663703918457, + "logps/chosen": -204.38375854492188, + "logps/rejected": -204.50376892089844, + "loss": 0.6846, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4307776689529419, + "rewards/margins": 0.06390251964330673, + "rewards/rejected": -0.4946801960468292, + "step": 514 + }, + { + "epoch": 0.67, + "learning_rate": 4.9627195695694774e-05, + "logits/chosen": -2.523521661758423, + "logits/rejected": -2.5060675144195557, + "logps/chosen": -163.70565795898438, + "logps/rejected": -207.99964904785156, + "loss": 0.6256, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.32005366683006287, + "rewards/margins": 0.24324071407318115, + "rewards/rejected": -0.5632944107055664, + "step": 515 + }, + { + "epoch": 0.68, + "learning_rate": 4.9625401389513715e-05, + "logits/chosen": -2.4181556701660156, + "logits/rejected": -2.369680881500244, + "logps/chosen": -205.77340698242188, + "logps/rejected": -155.3488311767578, + "loss": 0.8924, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.7356570363044739, + "rewards/margins": -0.26000097393989563, + "rewards/rejected": -0.47565609216690063, + "step": 516 + }, + { + "epoch": 0.68, + "learning_rate": 4.9623602808290855e-05, + "logits/chosen": -2.544684410095215, + "logits/rejected": -2.5888850688934326, + "logps/chosen": -229.81246948242188, + "logps/rejected": -251.07574462890625, + "loss": 0.6031, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4079195559024811, + "rewards/margins": 0.29860368371009827, + "rewards/rejected": -0.7065231800079346, + "step": 517 + }, + { + "epoch": 0.68, + "learning_rate": 4.962179995233843e-05, + "logits/chosen": -2.397979259490967, + "logits/rejected": -2.5106217861175537, + "logps/chosen": -157.80418395996094, + "logps/rejected": -175.9936065673828, + "loss": 0.6093, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.44883906841278076, + "rewards/margins": 0.32764846086502075, + "rewards/rejected": -0.7764875888824463, + "step": 518 + }, + { + "epoch": 0.68, + "learning_rate": 4.961999282196943e-05, + "logits/chosen": -2.4970529079437256, + "logits/rejected": -2.582210063934326, + "logps/chosen": -194.96694946289062, + "logps/rejected": -198.24681091308594, + "loss": 0.6883, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.37289613485336304, + "rewards/margins": 0.0580718033015728, + "rewards/rejected": -0.4309679865837097, + "step": 519 + }, + { + "epoch": 0.68, + "learning_rate": 4.9618181417497566e-05, + "logits/chosen": -2.469606399536133, + "logits/rejected": -2.54701566696167, + "logps/chosen": -215.56979370117188, + "logps/rejected": -233.58209228515625, + "loss": 0.691, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5863683819770813, + "rewards/margins": 0.03697293624281883, + "rewards/rejected": -0.6233413815498352, + "step": 520 + }, + { + "epoch": 0.68, + "learning_rate": 4.961636573923732e-05, + "logits/chosen": -2.4047884941101074, + "logits/rejected": -2.414797782897949, + "logps/chosen": -226.51625061035156, + "logps/rejected": -212.8037872314453, + "loss": 0.6559, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4074936807155609, + "rewards/margins": 0.16430170834064484, + "rewards/rejected": -0.5717953443527222, + "step": 521 + }, + { + "epoch": 0.68, + "learning_rate": 4.9614545787503886e-05, + "logits/chosen": -2.4261462688446045, + "logits/rejected": -2.482429027557373, + "logps/chosen": -197.58526611328125, + "logps/rejected": -215.66322326660156, + "loss": 0.6091, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4302670359611511, + "rewards/margins": 0.2678428888320923, + "rewards/rejected": -0.6981098651885986, + "step": 522 + }, + { + "epoch": 0.68, + "learning_rate": 4.961272156261321e-05, + "logits/chosen": -2.22928786277771, + "logits/rejected": -2.3709988594055176, + "logps/chosen": -157.08120727539062, + "logps/rejected": -186.32533264160156, + "loss": 0.5613, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.32108116149902344, + "rewards/margins": 0.476359486579895, + "rewards/rejected": -0.7974405884742737, + "step": 523 + }, + { + "epoch": 0.69, + "learning_rate": 4.961089306488199e-05, + "logits/chosen": -2.3764328956604004, + "logits/rejected": -2.423450231552124, + "logps/chosen": -225.89686584472656, + "logps/rejected": -250.8079376220703, + "loss": 0.6351, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4355972707271576, + "rewards/margins": 0.2791078984737396, + "rewards/rejected": -0.7147052884101868, + "step": 524 + }, + { + "epoch": 0.69, + "learning_rate": 4.960906029462766e-05, + "logits/chosen": -2.338782548904419, + "logits/rejected": -2.423982858657837, + "logps/chosen": -210.87510681152344, + "logps/rejected": -207.5828857421875, + "loss": 0.7054, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5041165351867676, + "rewards/margins": 0.092525914311409, + "rewards/rejected": -0.5966423749923706, + "step": 525 + }, + { + "epoch": 0.69, + "learning_rate": 4.96072232521684e-05, + "logits/chosen": -2.4865126609802246, + "logits/rejected": -2.55895733833313, + "logps/chosen": -290.5992431640625, + "logps/rejected": -275.2317810058594, + "loss": 0.6608, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5548466444015503, + "rewards/margins": 0.17232416570186615, + "rewards/rejected": -0.7271708250045776, + "step": 526 + }, + { + "epoch": 0.69, + "learning_rate": 4.96053819378231e-05, + "logits/chosen": -2.6567628383636475, + "logits/rejected": -2.5933961868286133, + "logps/chosen": -183.54367065429688, + "logps/rejected": -175.34194946289062, + "loss": 0.7362, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.7040901780128479, + "rewards/margins": -0.04544142633676529, + "rewards/rejected": -0.6586487889289856, + "step": 527 + }, + { + "epoch": 0.69, + "learning_rate": 4.960353635191145e-05, + "logits/chosen": -2.514308452606201, + "logits/rejected": -2.5751612186431885, + "logps/chosen": -171.15457153320312, + "logps/rejected": -163.67311096191406, + "loss": 0.606, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.340698778629303, + "rewards/margins": 0.42078471183776855, + "rewards/rejected": -0.7614834904670715, + "step": 528 + }, + { + "epoch": 0.69, + "learning_rate": 4.9601686494753826e-05, + "logits/chosen": -2.318474292755127, + "logits/rejected": -2.2866580486297607, + "logps/chosen": -155.76071166992188, + "logps/rejected": -156.44679260253906, + "loss": 0.7173, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.30408087372779846, + "rewards/margins": -0.002507832832634449, + "rewards/rejected": -0.3015730381011963, + "step": 529 + }, + { + "epoch": 0.69, + "learning_rate": 4.959983236667138e-05, + "logits/chosen": -2.577028274536133, + "logits/rejected": -2.6590182781219482, + "logps/chosen": -142.8551483154297, + "logps/rejected": -166.59149169921875, + "loss": 0.6867, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5840120315551758, + "rewards/margins": 0.07422871142625809, + "rewards/rejected": -0.6582406759262085, + "step": 530 + }, + { + "epoch": 0.69, + "learning_rate": 4.959797396798599e-05, + "logits/chosen": -2.409472703933716, + "logits/rejected": -2.4960474967956543, + "logps/chosen": -150.38006591796875, + "logps/rejected": -176.5684814453125, + "loss": 0.6631, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5444418787956238, + "rewards/margins": 0.12680965662002563, + "rewards/rejected": -0.6712515354156494, + "step": 531 + }, + { + "epoch": 0.7, + "learning_rate": 4.9596111299020284e-05, + "logits/chosen": -2.336719512939453, + "logits/rejected": -2.3440167903900146, + "logps/chosen": -184.50473022460938, + "logps/rejected": -187.92713928222656, + "loss": 0.627, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.33718907833099365, + "rewards/margins": 0.19500240683555603, + "rewards/rejected": -0.5321914553642273, + "step": 532 + }, + { + "epoch": 0.7, + "learning_rate": 4.959424436009762e-05, + "logits/chosen": -2.1330745220184326, + "logits/rejected": -2.2025442123413086, + "logps/chosen": -255.2789306640625, + "logps/rejected": -287.85736083984375, + "loss": 0.5781, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.6389483213424683, + "rewards/margins": 0.5605921149253845, + "rewards/rejected": -1.199540376663208, + "step": 533 + }, + { + "epoch": 0.7, + "learning_rate": 4.9592373151542105e-05, + "logits/chosen": -2.783942699432373, + "logits/rejected": -2.766120672225952, + "logps/chosen": -185.41404724121094, + "logps/rejected": -203.95669555664062, + "loss": 0.7158, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4609447717666626, + "rewards/margins": 0.003200806677341461, + "rewards/rejected": -0.46414560079574585, + "step": 534 + }, + { + "epoch": 0.7, + "learning_rate": 4.959049767367859e-05, + "logits/chosen": -2.4105114936828613, + "logits/rejected": -2.6107630729675293, + "logps/chosen": -172.59347534179688, + "logps/rejected": -212.30836486816406, + "loss": 0.6269, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5956838726997375, + "rewards/margins": 0.2615227699279785, + "rewards/rejected": -0.8572065830230713, + "step": 535 + }, + { + "epoch": 0.7, + "learning_rate": 4.958861792683266e-05, + "logits/chosen": -2.1016757488250732, + "logits/rejected": -2.1774649620056152, + "logps/chosen": -204.3402862548828, + "logps/rejected": -215.18890380859375, + "loss": 0.6164, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.49696028232574463, + "rewards/margins": 0.2144584059715271, + "rewards/rejected": -0.7114187479019165, + "step": 536 + }, + { + "epoch": 0.7, + "learning_rate": 4.958673391133065e-05, + "logits/chosen": -2.6260104179382324, + "logits/rejected": -2.648331880569458, + "logps/chosen": -180.17063903808594, + "logps/rejected": -175.27981567382812, + "loss": 0.7315, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7630718946456909, + "rewards/margins": 0.05031493306159973, + "rewards/rejected": -0.813386857509613, + "step": 537 + }, + { + "epoch": 0.7, + "learning_rate": 4.958484562749963e-05, + "logits/chosen": -2.3528480529785156, + "logits/rejected": -2.35026216506958, + "logps/chosen": -240.7522735595703, + "logps/rejected": -284.01177978515625, + "loss": 0.6558, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5610861778259277, + "rewards/margins": 0.1676333099603653, + "rewards/rejected": -0.7287194132804871, + "step": 538 + }, + { + "epoch": 0.71, + "learning_rate": 4.9582953075667406e-05, + "logits/chosen": -2.7893500328063965, + "logits/rejected": -2.695694923400879, + "logps/chosen": -213.7672119140625, + "logps/rejected": -197.13304138183594, + "loss": 0.7034, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7025653719902039, + "rewards/margins": 0.08315228670835495, + "rewards/rejected": -0.7857176661491394, + "step": 539 + }, + { + "epoch": 0.71, + "learning_rate": 4.958105625616253e-05, + "logits/chosen": -2.6460516452789307, + "logits/rejected": -2.6226911544799805, + "logps/chosen": -186.30870056152344, + "logps/rejected": -169.23074340820312, + "loss": 0.684, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4503140449523926, + "rewards/margins": 0.0906258150935173, + "rewards/rejected": -0.5409399271011353, + "step": 540 + }, + { + "epoch": 0.71, + "learning_rate": 4.95791551693143e-05, + "logits/chosen": -2.6068363189697266, + "logits/rejected": -2.645115852355957, + "logps/chosen": -225.66354370117188, + "logps/rejected": -248.78900146484375, + "loss": 0.7087, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.7523027062416077, + "rewards/margins": 0.06183048337697983, + "rewards/rejected": -0.8141331672668457, + "step": 541 + }, + { + "epoch": 0.71, + "learning_rate": 4.957724981545275e-05, + "logits/chosen": -2.472942352294922, + "logits/rejected": -2.5449466705322266, + "logps/chosen": -165.9869384765625, + "logps/rejected": -195.7375946044922, + "loss": 0.6983, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5627427697181702, + "rewards/margins": 0.17754918336868286, + "rewards/rejected": -0.7402920126914978, + "step": 542 + }, + { + "epoch": 0.71, + "learning_rate": 4.957534019490865e-05, + "logits/chosen": -2.6267778873443604, + "logits/rejected": -2.8627214431762695, + "logps/chosen": -208.47406005859375, + "logps/rejected": -230.64390563964844, + "loss": 0.5822, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.6202707290649414, + "rewards/margins": 0.35299497842788696, + "rewards/rejected": -0.9732657074928284, + "step": 543 + }, + { + "epoch": 0.71, + "learning_rate": 4.957342630801353e-05, + "logits/chosen": -2.302680730819702, + "logits/rejected": -2.4394922256469727, + "logps/chosen": -169.56790161132812, + "logps/rejected": -198.37738037109375, + "loss": 0.6269, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.44470933079719543, + "rewards/margins": 0.24785885214805603, + "rewards/rejected": -0.6925681233406067, + "step": 544 + }, + { + "epoch": 0.71, + "learning_rate": 4.957150815509963e-05, + "logits/chosen": -2.454188823699951, + "logits/rejected": -2.548893451690674, + "logps/chosen": -170.51040649414062, + "logps/rejected": -232.72727966308594, + "loss": 0.7024, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.3639673590660095, + "rewards/margins": 0.021631799638271332, + "rewards/rejected": -0.38559919595718384, + "step": 545 + }, + { + "epoch": 0.71, + "learning_rate": 4.9569585736499945e-05, + "logits/chosen": -2.337388515472412, + "logits/rejected": -2.4017117023468018, + "logps/chosen": -177.1454620361328, + "logps/rejected": -191.3271484375, + "loss": 0.5661, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3925960958003998, + "rewards/margins": 0.34767282009124756, + "rewards/rejected": -0.740268886089325, + "step": 546 + }, + { + "epoch": 0.72, + "learning_rate": 4.9567659052548234e-05, + "logits/chosen": -2.653782367706299, + "logits/rejected": -2.6659772396087646, + "logps/chosen": -198.43539428710938, + "logps/rejected": -176.4965362548828, + "loss": 0.7005, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5790706872940063, + "rewards/margins": 0.11143729835748672, + "rewards/rejected": -0.6905080080032349, + "step": 547 + }, + { + "epoch": 0.72, + "learning_rate": 4.956572810357896e-05, + "logits/chosen": -2.5503158569335938, + "logits/rejected": -2.520476818084717, + "logps/chosen": -190.98855590820312, + "logps/rejected": -173.04501342773438, + "loss": 0.7238, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5375619530677795, + "rewards/margins": 0.01934710703790188, + "rewards/rejected": -0.5569091439247131, + "step": 548 + }, + { + "epoch": 0.72, + "learning_rate": 4.956379288992733e-05, + "logits/chosen": -2.2868709564208984, + "logits/rejected": -2.358032464981079, + "logps/chosen": -163.09713745117188, + "logps/rejected": -187.16030883789062, + "loss": 0.5837, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3862106502056122, + "rewards/margins": 0.4660491943359375, + "rewards/rejected": -0.8522598147392273, + "step": 549 + }, + { + "epoch": 0.72, + "learning_rate": 4.956185341192933e-05, + "logits/chosen": -2.522181987762451, + "logits/rejected": -2.5097920894622803, + "logps/chosen": -167.14633178710938, + "logps/rejected": -205.06365966796875, + "loss": 0.7648, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5081343650817871, + "rewards/margins": -0.012615025043487549, + "rewards/rejected": -0.4955193102359772, + "step": 550 + }, + { + "epoch": 0.72, + "learning_rate": 4.9559909669921635e-05, + "logits/chosen": -2.3636226654052734, + "logits/rejected": -2.438326120376587, + "logps/chosen": -197.59310913085938, + "logps/rejected": -186.26112365722656, + "loss": 0.7066, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5027388334274292, + "rewards/margins": 0.07569919526576996, + "rewards/rejected": -0.5784379839897156, + "step": 551 + }, + { + "epoch": 0.72, + "learning_rate": 4.9557961664241694e-05, + "logits/chosen": -2.425126552581787, + "logits/rejected": -2.5314579010009766, + "logps/chosen": -181.5507049560547, + "logps/rejected": -211.9708251953125, + "loss": 0.7133, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6780929565429688, + "rewards/margins": 0.19112253189086914, + "rewards/rejected": -0.8692154884338379, + "step": 552 + }, + { + "epoch": 0.72, + "learning_rate": 4.955600939522769e-05, + "logits/chosen": -2.52880597114563, + "logits/rejected": -2.5222525596618652, + "logps/chosen": -178.28521728515625, + "logps/rejected": -210.90565490722656, + "loss": 0.587, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.5996692180633545, + "rewards/margins": 0.25250738859176636, + "rewards/rejected": -0.8521764874458313, + "step": 553 + }, + { + "epoch": 0.73, + "learning_rate": 4.955405286321854e-05, + "logits/chosen": -2.3732471466064453, + "logits/rejected": -2.4644832611083984, + "logps/chosen": -210.51051330566406, + "logps/rejected": -208.96826171875, + "loss": 0.5982, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5088058114051819, + "rewards/margins": 0.2920345067977905, + "rewards/rejected": -0.8008402585983276, + "step": 554 + }, + { + "epoch": 0.73, + "learning_rate": 4.95520920685539e-05, + "logits/chosen": -2.4812893867492676, + "logits/rejected": -2.5562007427215576, + "logps/chosen": -170.83322143554688, + "logps/rejected": -192.9698486328125, + "loss": 0.5716, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6304159164428711, + "rewards/margins": 0.37207356095314026, + "rewards/rejected": -1.002489447593689, + "step": 555 + }, + { + "epoch": 0.73, + "learning_rate": 4.9550127011574176e-05, + "logits/chosen": -2.281599760055542, + "logits/rejected": -2.3443262577056885, + "logps/chosen": -188.46249389648438, + "logps/rejected": -179.5142364501953, + "loss": 0.6338, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.41307827830314636, + "rewards/margins": 0.17765894532203674, + "rewards/rejected": -0.5907372236251831, + "step": 556 + }, + { + "epoch": 0.73, + "learning_rate": 4.95481576926205e-05, + "logits/chosen": -2.584645986557007, + "logits/rejected": -2.604243755340576, + "logps/chosen": -181.7239990234375, + "logps/rejected": -209.90145874023438, + "loss": 0.5821, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7002069354057312, + "rewards/margins": 0.43418359756469727, + "rewards/rejected": -1.1343904733657837, + "step": 557 + }, + { + "epoch": 0.73, + "learning_rate": 4.954618411203476e-05, + "logits/chosen": -2.679819107055664, + "logits/rejected": -2.7520828247070312, + "logps/chosen": -217.54901123046875, + "logps/rejected": -236.81210327148438, + "loss": 0.6187, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.734368085861206, + "rewards/margins": 0.29621487855911255, + "rewards/rejected": -1.0305829048156738, + "step": 558 + }, + { + "epoch": 0.73, + "learning_rate": 4.954420627015957e-05, + "logits/chosen": -2.530925750732422, + "logits/rejected": -2.5949485301971436, + "logps/chosen": -190.8072509765625, + "logps/rejected": -226.07455444335938, + "loss": 0.5481, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.5543414950370789, + "rewards/margins": 0.42910295724868774, + "rewards/rejected": -0.9834444522857666, + "step": 559 + }, + { + "epoch": 0.73, + "learning_rate": 4.954222416733829e-05, + "logits/chosen": -2.4444892406463623, + "logits/rejected": -2.4189417362213135, + "logps/chosen": -275.56109619140625, + "logps/rejected": -248.05233764648438, + "loss": 0.7178, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5780796408653259, + "rewards/margins": 0.0615503191947937, + "rewards/rejected": -0.6396299600601196, + "step": 560 + }, + { + "epoch": 0.73, + "learning_rate": 4.954023780391501e-05, + "logits/chosen": -2.360807180404663, + "logits/rejected": -2.308104991912842, + "logps/chosen": -172.66574096679688, + "logps/rejected": -191.79412841796875, + "loss": 0.5558, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.709125816822052, + "rewards/margins": 0.38629868626594543, + "rewards/rejected": -1.0954244136810303, + "step": 561 + }, + { + "epoch": 0.74, + "learning_rate": 4.953824718023459e-05, + "logits/chosen": -2.140808582305908, + "logits/rejected": -2.2450931072235107, + "logps/chosen": -135.81964111328125, + "logps/rejected": -176.7286376953125, + "loss": 0.7815, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7152988910675049, + "rewards/margins": 0.006897151470184326, + "rewards/rejected": -0.722196102142334, + "step": 562 + }, + { + "epoch": 0.74, + "learning_rate": 4.953625229664259e-05, + "logits/chosen": -2.2745089530944824, + "logits/rejected": -2.2524397373199463, + "logps/chosen": -174.69879150390625, + "logps/rejected": -193.57516479492188, + "loss": 0.8268, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6095337867736816, + "rewards/margins": -0.15611900389194489, + "rewards/rejected": -0.45341476798057556, + "step": 563 + }, + { + "epoch": 0.74, + "learning_rate": 4.953425315348534e-05, + "logits/chosen": -2.4158096313476562, + "logits/rejected": -2.5616989135742188, + "logps/chosen": -161.41812133789062, + "logps/rejected": -195.5721893310547, + "loss": 0.5776, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4711029529571533, + "rewards/margins": 0.4865836799144745, + "rewards/rejected": -0.957686722278595, + "step": 564 + }, + { + "epoch": 0.74, + "learning_rate": 4.953224975110988e-05, + "logits/chosen": -2.3829874992370605, + "logits/rejected": -2.3937342166900635, + "logps/chosen": -168.71534729003906, + "logps/rejected": -178.2794189453125, + "loss": 0.7187, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.5582124590873718, + "rewards/margins": -0.010660316795110703, + "rewards/rejected": -0.5475521087646484, + "step": 565 + }, + { + "epoch": 0.74, + "learning_rate": 4.9530242089864026e-05, + "logits/chosen": -2.3829197883605957, + "logits/rejected": -2.462663173675537, + "logps/chosen": -173.0697021484375, + "logps/rejected": -207.88348388671875, + "loss": 0.6547, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4435291290283203, + "rewards/margins": 0.13191349804401398, + "rewards/rejected": -0.5754426717758179, + "step": 566 + }, + { + "epoch": 0.74, + "learning_rate": 4.9528230170096305e-05, + "logits/chosen": -2.319032907485962, + "logits/rejected": -2.3326690196990967, + "logps/chosen": -169.28501892089844, + "logps/rejected": -194.19927978515625, + "loss": 0.7851, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7481305599212646, + "rewards/margins": 0.09495264291763306, + "rewards/rejected": -0.8430831432342529, + "step": 567 + }, + { + "epoch": 0.74, + "learning_rate": 4.952621399215598e-05, + "logits/chosen": -2.630171537399292, + "logits/rejected": -2.617821455001831, + "logps/chosen": -192.14907836914062, + "logps/rejected": -193.33782958984375, + "loss": 0.687, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.47104740142822266, + "rewards/margins": 0.11374848335981369, + "rewards/rejected": -0.5847958922386169, + "step": 568 + }, + { + "epoch": 0.74, + "learning_rate": 4.9524193556393083e-05, + "logits/chosen": -2.3562679290771484, + "logits/rejected": -2.388498306274414, + "logps/chosen": -212.62057495117188, + "logps/rejected": -230.94476318359375, + "loss": 0.6921, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.575402557849884, + "rewards/margins": 0.19730643928050995, + "rewards/rejected": -0.7727090120315552, + "step": 569 + }, + { + "epoch": 0.75, + "learning_rate": 4.952216886315837e-05, + "logits/chosen": -2.414254903793335, + "logits/rejected": -2.4329581260681152, + "logps/chosen": -175.03147888183594, + "logps/rejected": -214.68109130859375, + "loss": 0.6274, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6071965098381042, + "rewards/margins": 0.29113632440567017, + "rewards/rejected": -0.8983328342437744, + "step": 570 + }, + { + "epoch": 0.75, + "learning_rate": 4.952013991280332e-05, + "logits/chosen": -2.5487287044525146, + "logits/rejected": -2.563288688659668, + "logps/chosen": -226.3212890625, + "logps/rejected": -214.44525146484375, + "loss": 0.746, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6687456965446472, + "rewards/margins": -0.017180969938635826, + "rewards/rejected": -0.6515647172927856, + "step": 571 + }, + { + "epoch": 0.75, + "learning_rate": 4.951810670568017e-05, + "logits/chosen": -2.445035219192505, + "logits/rejected": -2.3831865787506104, + "logps/chosen": -214.5970001220703, + "logps/rejected": -228.41064453125, + "loss": 0.7696, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.62672358751297, + "rewards/margins": -0.040074095129966736, + "rewards/rejected": -0.5866495370864868, + "step": 572 + }, + { + "epoch": 0.75, + "learning_rate": 4.9516069242141885e-05, + "logits/chosen": -2.321072578430176, + "logits/rejected": -2.4207139015197754, + "logps/chosen": -203.04151916503906, + "logps/rejected": -206.2476806640625, + "loss": 0.766, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6442697644233704, + "rewards/margins": -0.051033951342105865, + "rewards/rejected": -0.5932357907295227, + "step": 573 + }, + { + "epoch": 0.75, + "learning_rate": 4.951402752254219e-05, + "logits/chosen": -2.3598363399505615, + "logits/rejected": -2.510725498199463, + "logps/chosen": -160.54183959960938, + "logps/rejected": -181.72743225097656, + "loss": 0.613, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.45199206471443176, + "rewards/margins": 0.4041283130645752, + "rewards/rejected": -0.8561204075813293, + "step": 574 + }, + { + "epoch": 0.75, + "learning_rate": 4.951198154723552e-05, + "logits/chosen": -2.550727605819702, + "logits/rejected": -2.5166475772857666, + "logps/chosen": -150.58238220214844, + "logps/rejected": -157.48434448242188, + "loss": 0.6906, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.3471527397632599, + "rewards/margins": 0.06844709068536758, + "rewards/rejected": -0.41559985280036926, + "step": 575 + }, + { + "epoch": 0.75, + "learning_rate": 4.950993131657706e-05, + "logits/chosen": -2.32822847366333, + "logits/rejected": -2.3608272075653076, + "logps/chosen": -183.2533721923828, + "logps/rejected": -155.5178680419922, + "loss": 0.8822, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.7413718700408936, + "rewards/margins": -0.2268693447113037, + "rewards/rejected": -0.5145025849342346, + "step": 576 + }, + { + "epoch": 0.76, + "learning_rate": 4.9507876830922755e-05, + "logits/chosen": -2.426633358001709, + "logits/rejected": -2.350067138671875, + "logps/chosen": -159.07473754882812, + "logps/rejected": -177.24722290039062, + "loss": 0.6643, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4699227511882782, + "rewards/margins": 0.37284964323043823, + "rewards/rejected": -0.842772364616394, + "step": 577 + }, + { + "epoch": 0.76, + "learning_rate": 4.950581809062925e-05, + "logits/chosen": -2.3766255378723145, + "logits/rejected": -2.411029815673828, + "logps/chosen": -211.9445343017578, + "logps/rejected": -224.93450927734375, + "loss": 0.7549, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5890372395515442, + "rewards/margins": 0.09359408915042877, + "rewards/rejected": -0.6826313138008118, + "step": 578 + }, + { + "epoch": 0.76, + "learning_rate": 4.950375509605396e-05, + "logits/chosen": -2.3283326625823975, + "logits/rejected": -2.1340157985687256, + "logps/chosen": -159.19125366210938, + "logps/rejected": -161.9340362548828, + "loss": 0.7342, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4637477993965149, + "rewards/margins": -0.048650771379470825, + "rewards/rejected": -0.41509705781936646, + "step": 579 + }, + { + "epoch": 0.76, + "learning_rate": 4.9501687847555016e-05, + "logits/chosen": -2.4342098236083984, + "logits/rejected": -2.4008848667144775, + "logps/chosen": -214.58380126953125, + "logps/rejected": -184.8182373046875, + "loss": 0.7425, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.45641300082206726, + "rewards/margins": -0.036872051656246185, + "rewards/rejected": -0.4195409417152405, + "step": 580 + }, + { + "epoch": 0.76, + "learning_rate": 4.949961634549131e-05, + "logits/chosen": -2.4010324478149414, + "logits/rejected": -2.397801160812378, + "logps/chosen": -175.51223754882812, + "logps/rejected": -165.90208435058594, + "loss": 0.6852, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3178666830062866, + "rewards/margins": 0.06419295072555542, + "rewards/rejected": -0.38205963373184204, + "step": 581 + }, + { + "epoch": 0.76, + "learning_rate": 4.949754059022246e-05, + "logits/chosen": -2.494157075881958, + "logits/rejected": -2.560157537460327, + "logps/chosen": -172.84835815429688, + "logps/rejected": -178.97540283203125, + "loss": 0.6419, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.2800545394420624, + "rewards/margins": 0.1691555678844452, + "rewards/rejected": -0.44921010732650757, + "step": 582 + }, + { + "epoch": 0.76, + "learning_rate": 4.949546058210881e-05, + "logits/chosen": -2.209298610687256, + "logits/rejected": -2.293645143508911, + "logps/chosen": -194.64381408691406, + "logps/rejected": -237.14974975585938, + "loss": 0.5891, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.257599413394928, + "rewards/margins": 0.3168815076351166, + "rewards/rejected": -0.5744808912277222, + "step": 583 + }, + { + "epoch": 0.76, + "learning_rate": 4.9493376321511464e-05, + "logits/chosen": -2.446950912475586, + "logits/rejected": -2.4006083011627197, + "logps/chosen": -199.42282104492188, + "logps/rejected": -196.51849365234375, + "loss": 0.6841, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.35894283652305603, + "rewards/margins": 0.1623423993587494, + "rewards/rejected": -0.5212852358818054, + "step": 584 + }, + { + "epoch": 0.77, + "learning_rate": 4.9491287808792265e-05, + "logits/chosen": -2.3545334339141846, + "logits/rejected": -2.452188491821289, + "logps/chosen": -198.55056762695312, + "logps/rejected": -195.9398956298828, + "loss": 0.6768, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4273378551006317, + "rewards/margins": 0.11289644986391068, + "rewards/rejected": -0.5402343273162842, + "step": 585 + }, + { + "epoch": 0.77, + "learning_rate": 4.948919504431376e-05, + "logits/chosen": -2.687852144241333, + "logits/rejected": -2.6648173332214355, + "logps/chosen": -215.23638916015625, + "logps/rejected": -221.06588745117188, + "loss": 0.6981, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.30441591143608093, + "rewards/margins": 0.08797214180231094, + "rewards/rejected": -0.39238807559013367, + "step": 586 + }, + { + "epoch": 0.77, + "learning_rate": 4.948709802843929e-05, + "logits/chosen": -2.2244153022766113, + "logits/rejected": -2.2272024154663086, + "logps/chosen": -197.98904418945312, + "logps/rejected": -188.39773559570312, + "loss": 0.6626, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.27749666571617126, + "rewards/margins": 0.12206709384918213, + "rewards/rejected": -0.3995637595653534, + "step": 587 + }, + { + "epoch": 0.77, + "learning_rate": 4.9484996761532886e-05, + "logits/chosen": -2.471318006515503, + "logits/rejected": -2.550818920135498, + "logps/chosen": -192.51039123535156, + "logps/rejected": -192.6507568359375, + "loss": 0.7235, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.27331510186195374, + "rewards/margins": -0.019602246582508087, + "rewards/rejected": -0.25371283292770386, + "step": 588 + }, + { + "epoch": 0.77, + "learning_rate": 4.9482891243959335e-05, + "logits/chosen": -2.4945497512817383, + "logits/rejected": -2.554481029510498, + "logps/chosen": -187.52774047851562, + "logps/rejected": -227.6110076904297, + "loss": 0.5713, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.2509083151817322, + "rewards/margins": 0.3469720482826233, + "rewards/rejected": -0.5978803038597107, + "step": 589 + }, + { + "epoch": 0.77, + "learning_rate": 4.948078147608416e-05, + "logits/chosen": -2.5424046516418457, + "logits/rejected": -2.58070707321167, + "logps/chosen": -181.62966918945312, + "logps/rejected": -219.62698364257812, + "loss": 0.6805, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4028276205062866, + "rewards/margins": 0.0835791528224945, + "rewards/rejected": -0.48640677332878113, + "step": 590 + }, + { + "epoch": 0.77, + "learning_rate": 4.9478667458273624e-05, + "logits/chosen": -2.558363199234009, + "logits/rejected": -2.533869743347168, + "logps/chosen": -207.1812286376953, + "logps/rejected": -201.0562286376953, + "loss": 0.6612, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.24153894186019897, + "rewards/margins": 0.12561962008476257, + "rewards/rejected": -0.36715856194496155, + "step": 591 + }, + { + "epoch": 0.77, + "learning_rate": 4.9476549190894725e-05, + "logits/chosen": -2.4393560886383057, + "logits/rejected": -2.450369358062744, + "logps/chosen": -153.7821044921875, + "logps/rejected": -153.88775634765625, + "loss": 0.686, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.23368775844573975, + "rewards/margins": 0.0638914704322815, + "rewards/rejected": -0.29757922887802124, + "step": 592 + }, + { + "epoch": 0.78, + "learning_rate": 4.947442667431522e-05, + "logits/chosen": -2.6815226078033447, + "logits/rejected": -2.506643056869507, + "logps/chosen": -229.4060516357422, + "logps/rejected": -240.65362548828125, + "loss": 0.7512, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3724709153175354, + "rewards/margins": -0.04746837168931961, + "rewards/rejected": -0.325002521276474, + "step": 593 + }, + { + "epoch": 0.78, + "learning_rate": 4.9472299908903555e-05, + "logits/chosen": -2.3556230068206787, + "logits/rejected": -2.4480998516082764, + "logps/chosen": -128.82077026367188, + "logps/rejected": -170.01109313964844, + "loss": 0.6209, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.1109066978096962, + "rewards/margins": 0.2432006597518921, + "rewards/rejected": -0.3541073799133301, + "step": 594 + }, + { + "epoch": 0.78, + "learning_rate": 4.947016889502895e-05, + "logits/chosen": -2.5540051460266113, + "logits/rejected": -2.549333095550537, + "logps/chosen": -178.5426788330078, + "logps/rejected": -172.33694458007812, + "loss": 0.7123, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3693382441997528, + "rewards/margins": -0.0018721502274274826, + "rewards/rejected": -0.36746612191200256, + "step": 595 + }, + { + "epoch": 0.78, + "learning_rate": 4.946803363306137e-05, + "logits/chosen": -2.076066255569458, + "logits/rejected": -2.011690139770508, + "logps/chosen": -194.8577880859375, + "logps/rejected": -176.207763671875, + "loss": 0.7698, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.26399266719818115, + "rewards/margins": -0.08310166001319885, + "rewards/rejected": -0.1808909922838211, + "step": 596 + }, + { + "epoch": 0.78, + "learning_rate": 4.946589412337149e-05, + "logits/chosen": -2.2557950019836426, + "logits/rejected": -2.3377816677093506, + "logps/chosen": -148.5780029296875, + "logps/rejected": -170.98593139648438, + "loss": 0.6012, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.07843692600727081, + "rewards/margins": 0.26211532950401306, + "rewards/rejected": -0.18367838859558105, + "step": 597 + }, + { + "epoch": 0.78, + "learning_rate": 4.9463750366330734e-05, + "logits/chosen": -2.2877843379974365, + "logits/rejected": -2.359644889831543, + "logps/chosen": -187.41412353515625, + "logps/rejected": -194.01309204101562, + "loss": 0.734, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.3197644352912903, + "rewards/margins": -0.021138954907655716, + "rewards/rejected": -0.29862549901008606, + "step": 598 + }, + { + "epoch": 0.78, + "learning_rate": 4.9461602362311275e-05, + "logits/chosen": -2.493452548980713, + "logits/rejected": -2.5441408157348633, + "logps/chosen": -209.37081909179688, + "logps/rejected": -173.10861206054688, + "loss": 0.7112, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.324692964553833, + "rewards/margins": 0.05840418487787247, + "rewards/rejected": -0.3830971419811249, + "step": 599 + }, + { + "epoch": 0.79, + "learning_rate": 4.9459450111686e-05, + "logits/chosen": -2.4824540615081787, + "logits/rejected": -2.477165937423706, + "logps/chosen": -191.7369384765625, + "logps/rejected": -181.0056915283203, + "loss": 0.692, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.38107597827911377, + "rewards/margins": 0.05237661302089691, + "rewards/rejected": -0.4334526062011719, + "step": 600 + }, + { + "epoch": 0.79, + "learning_rate": 4.945729361482856e-05, + "logits/chosen": -2.450261116027832, + "logits/rejected": -2.500009536743164, + "logps/chosen": -232.3407440185547, + "logps/rejected": -212.10638427734375, + "loss": 0.5732, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.19914206862449646, + "rewards/margins": 0.3821813762187958, + "rewards/rejected": -0.5813234448432922, + "step": 601 + }, + { + "epoch": 0.79, + "learning_rate": 4.9455132872113316e-05, + "logits/chosen": -2.3420450687408447, + "logits/rejected": -2.3474817276000977, + "logps/chosen": -216.38914489746094, + "logps/rejected": -224.3979949951172, + "loss": 0.6047, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.1369827687740326, + "rewards/margins": 0.2419712096452713, + "rewards/rejected": -0.3789539933204651, + "step": 602 + }, + { + "epoch": 0.79, + "learning_rate": 4.945296788391539e-05, + "logits/chosen": -2.3396308422088623, + "logits/rejected": -2.3148958683013916, + "logps/chosen": -169.25946044921875, + "logps/rejected": -197.1276397705078, + "loss": 0.7964, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.32238638401031494, + "rewards/margins": -0.1470273733139038, + "rewards/rejected": -0.17535902559757233, + "step": 603 + }, + { + "epoch": 0.79, + "learning_rate": 4.945079865061062e-05, + "logits/chosen": -2.275747537612915, + "logits/rejected": -2.2709038257598877, + "logps/chosen": -139.6877899169922, + "logps/rejected": -148.5768585205078, + "loss": 0.6796, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.08058857917785645, + "rewards/margins": 0.04861634969711304, + "rewards/rejected": -0.12920492887496948, + "step": 604 + }, + { + "epoch": 0.79, + "learning_rate": 4.94486251725756e-05, + "logits/chosen": -2.3698954582214355, + "logits/rejected": -2.453796625137329, + "logps/chosen": -229.4521484375, + "logps/rejected": -210.6240234375, + "loss": 0.6212, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.22786223888397217, + "rewards/margins": 0.20501334965229034, + "rewards/rejected": -0.4328756034374237, + "step": 605 + }, + { + "epoch": 0.79, + "learning_rate": 4.944644745018765e-05, + "logits/chosen": -2.4110870361328125, + "logits/rejected": -2.423731565475464, + "logps/chosen": -219.2958221435547, + "logps/rejected": -202.20706176757812, + "loss": 0.7773, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.37566569447517395, + "rewards/margins": -0.0947473868727684, + "rewards/rejected": -0.28091832995414734, + "step": 606 + }, + { + "epoch": 0.79, + "learning_rate": 4.944426548382482e-05, + "logits/chosen": -2.4861879348754883, + "logits/rejected": -2.5459907054901123, + "logps/chosen": -237.8024444580078, + "logps/rejected": -260.2302551269531, + "loss": 0.7621, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.42855891585350037, + "rewards/margins": -0.048416607081890106, + "rewards/rejected": -0.38014233112335205, + "step": 607 + }, + { + "epoch": 0.8, + "learning_rate": 4.944207927386591e-05, + "logits/chosen": -2.478734254837036, + "logits/rejected": -2.5266644954681396, + "logps/chosen": -218.4932403564453, + "logps/rejected": -225.3731689453125, + "loss": 0.7075, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.23466593027114868, + "rewards/margins": 0.09529409557580948, + "rewards/rejected": -0.3299599885940552, + "step": 608 + }, + { + "epoch": 0.8, + "learning_rate": 4.9439888820690475e-05, + "logits/chosen": -2.5482990741729736, + "logits/rejected": -2.544922113418579, + "logps/chosen": -186.0775909423828, + "logps/rejected": -197.2762908935547, + "loss": 0.5692, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.01272206474095583, + "rewards/margins": 0.301639199256897, + "rewards/rejected": -0.3143612742424011, + "step": 609 + }, + { + "epoch": 0.8, + "learning_rate": 4.943769412467875e-05, + "logits/chosen": -2.2644476890563965, + "logits/rejected": -2.3761229515075684, + "logps/chosen": -215.5855712890625, + "logps/rejected": -246.0447998046875, + "loss": 0.5599, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19982381165027618, + "rewards/margins": 0.4122087359428406, + "rewards/rejected": -0.6120326519012451, + "step": 610 + }, + { + "epoch": 0.8, + "learning_rate": 4.943549518621176e-05, + "logits/chosen": -2.4406063556671143, + "logits/rejected": -2.529265880584717, + "logps/chosen": -175.3257598876953, + "logps/rejected": -188.4786376953125, + "loss": 0.6989, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.19547346234321594, + "rewards/margins": 0.05771362781524658, + "rewards/rejected": -0.2531871199607849, + "step": 611 + }, + { + "epoch": 0.8, + "learning_rate": 4.9433292005671236e-05, + "logits/chosen": -2.4887635707855225, + "logits/rejected": -2.48945951461792, + "logps/chosen": -187.1964111328125, + "logps/rejected": -231.7959747314453, + "loss": 0.6501, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.21680843830108643, + "rewards/margins": 0.15266810357570648, + "rewards/rejected": -0.3694765567779541, + "step": 612 + }, + { + "epoch": 0.8, + "learning_rate": 4.943108458343968e-05, + "logits/chosen": -2.4516611099243164, + "logits/rejected": -2.4493842124938965, + "logps/chosen": -175.4194793701172, + "logps/rejected": -185.060546875, + "loss": 0.6172, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.2488647699356079, + "rewards/margins": 0.22184765338897705, + "rewards/rejected": -0.47071248292922974, + "step": 613 + }, + { + "epoch": 0.8, + "learning_rate": 4.942887291990028e-05, + "logits/chosen": -2.636701822280884, + "logits/rejected": -2.5966298580169678, + "logps/chosen": -194.91783142089844, + "logps/rejected": -182.28858947753906, + "loss": 0.6595, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.04664868861436844, + "rewards/margins": 0.11729076504707336, + "rewards/rejected": -0.1639394611120224, + "step": 614 + }, + { + "epoch": 0.8, + "learning_rate": 4.9426657015436994e-05, + "logits/chosen": -2.303603172302246, + "logits/rejected": -2.2422983646392822, + "logps/chosen": -144.08590698242188, + "logps/rejected": -148.51541137695312, + "loss": 0.7636, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.18376705050468445, + "rewards/margins": -0.09818767011165619, + "rewards/rejected": -0.08557939529418945, + "step": 615 + }, + { + "epoch": 0.81, + "learning_rate": 4.9424436870434515e-05, + "logits/chosen": -2.7395613193511963, + "logits/rejected": -2.747368097305298, + "logps/chosen": -226.32354736328125, + "logps/rejected": -212.107421875, + "loss": 0.7233, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.19879278540611267, + "rewards/margins": 0.0006840275600552559, + "rewards/rejected": -0.19947680830955505, + "step": 616 + }, + { + "epoch": 0.81, + "learning_rate": 4.942221248527826e-05, + "logits/chosen": -2.493027448654175, + "logits/rejected": -2.686164140701294, + "logps/chosen": -166.07200622558594, + "logps/rejected": -189.38475036621094, + "loss": 0.8368, + "rewards/accuracies": 0.1875, + "rewards/chosen": -0.28490307927131653, + "rewards/margins": -0.22959473729133606, + "rewards/rejected": -0.05530836060643196, + "step": 617 + }, + { + "epoch": 0.81, + "learning_rate": 4.94199838603544e-05, + "logits/chosen": -2.3346376419067383, + "logits/rejected": -2.4675793647766113, + "logps/chosen": -177.6221160888672, + "logps/rejected": -186.67446899414062, + "loss": 0.4898, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.1148318499326706, + "rewards/margins": 0.5469637513160706, + "rewards/rejected": -0.4321318566799164, + "step": 618 + }, + { + "epoch": 0.81, + "learning_rate": 4.941775099604983e-05, + "logits/chosen": -2.5135788917541504, + "logits/rejected": -2.6071126461029053, + "logps/chosen": -186.59161376953125, + "logps/rejected": -205.1910400390625, + "loss": 0.6155, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.029235512018203735, + "rewards/margins": 0.20693793892860413, + "rewards/rejected": -0.23617346584796906, + "step": 619 + }, + { + "epoch": 0.81, + "learning_rate": 4.941551389275217e-05, + "logits/chosen": -2.49117374420166, + "logits/rejected": -2.5229427814483643, + "logps/chosen": -208.4278106689453, + "logps/rejected": -191.97398376464844, + "loss": 0.6074, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18853022158145905, + "rewards/margins": 0.2418961077928543, + "rewards/rejected": -0.43042635917663574, + "step": 620 + }, + { + "epoch": 0.81, + "learning_rate": 4.941327255084979e-05, + "logits/chosen": -2.4580295085906982, + "logits/rejected": -2.4623186588287354, + "logps/chosen": -191.52886962890625, + "logps/rejected": -169.84417724609375, + "loss": 0.7945, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.2514897584915161, + "rewards/margins": -0.04441142827272415, + "rewards/rejected": -0.20707830786705017, + "step": 621 + }, + { + "epoch": 0.81, + "learning_rate": 4.9411026970731805e-05, + "logits/chosen": -2.5821356773376465, + "logits/rejected": -2.619976758956909, + "logps/chosen": -203.17149353027344, + "logps/rejected": -217.1163330078125, + "loss": 0.6965, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.33491429686546326, + "rewards/margins": 0.05494851991534233, + "rewards/rejected": -0.3898628354072571, + "step": 622 + }, + { + "epoch": 0.82, + "learning_rate": 4.9408777152788044e-05, + "logits/chosen": -2.464843273162842, + "logits/rejected": -2.5318896770477295, + "logps/chosen": -169.26376342773438, + "logps/rejected": -182.11395263671875, + "loss": 0.6721, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.17076966166496277, + "rewards/margins": 0.10583178699016571, + "rewards/rejected": -0.2766014337539673, + "step": 623 + }, + { + "epoch": 0.82, + "learning_rate": 4.9406523097409085e-05, + "logits/chosen": -2.262134075164795, + "logits/rejected": -2.329742670059204, + "logps/chosen": -153.96200561523438, + "logps/rejected": -157.85238647460938, + "loss": 0.6468, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.13661958277225494, + "rewards/margins": 0.20581209659576416, + "rewards/rejected": -0.3424316644668579, + "step": 624 + }, + { + "epoch": 0.82, + "learning_rate": 4.940426480498623e-05, + "logits/chosen": -2.403775453567505, + "logits/rejected": -2.459458351135254, + "logps/chosen": -167.8171844482422, + "logps/rejected": -221.13282775878906, + "loss": 0.6038, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2118593454360962, + "rewards/margins": 0.26167717576026917, + "rewards/rejected": -0.47353655099868774, + "step": 625 + }, + { + "epoch": 0.82, + "learning_rate": 4.940200227591154e-05, + "logits/chosen": -1.9581964015960693, + "logits/rejected": -2.0107665061950684, + "logps/chosen": -124.81986236572266, + "logps/rejected": -138.32220458984375, + "loss": 0.6216, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.05847623199224472, + "rewards/margins": 0.22099967300891876, + "rewards/rejected": -0.2794759273529053, + "step": 626 + }, + { + "epoch": 0.82, + "learning_rate": 4.9399735510577796e-05, + "logits/chosen": -2.2749149799346924, + "logits/rejected": -2.283113718032837, + "logps/chosen": -155.7340087890625, + "logps/rejected": -150.0223388671875, + "loss": 0.6522, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1051105409860611, + "rewards/margins": 0.16768980026245117, + "rewards/rejected": -0.27280035614967346, + "step": 627 + }, + { + "epoch": 0.82, + "learning_rate": 4.93974645093785e-05, + "logits/chosen": -2.4072506427764893, + "logits/rejected": -2.4812541007995605, + "logps/chosen": -195.17230224609375, + "logps/rejected": -194.09622192382812, + "loss": 0.6298, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.06678425520658493, + "rewards/margins": 0.17194166779518127, + "rewards/rejected": -0.2387259304523468, + "step": 628 + }, + { + "epoch": 0.82, + "learning_rate": 4.9395189272707916e-05, + "logits/chosen": -2.428178310394287, + "logits/rejected": -2.5200040340423584, + "logps/chosen": -194.95848083496094, + "logps/rejected": -215.4308319091797, + "loss": 0.699, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.37522369623184204, + "rewards/margins": 0.045931555330753326, + "rewards/rejected": -0.4211552143096924, + "step": 629 + }, + { + "epoch": 0.82, + "learning_rate": 4.939290980096103e-05, + "logits/chosen": -2.243135690689087, + "logits/rejected": -2.25071120262146, + "logps/chosen": -188.89820861816406, + "logps/rejected": -219.1158447265625, + "loss": 0.7187, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.2669258415699005, + "rewards/margins": 0.03279988840222359, + "rewards/rejected": -0.2997257113456726, + "step": 630 + }, + { + "epoch": 0.83, + "learning_rate": 4.9390626094533565e-05, + "logits/chosen": -2.496492385864258, + "logits/rejected": -2.6048879623413086, + "logps/chosen": -169.56320190429688, + "logps/rejected": -184.27169799804688, + "loss": 0.6584, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.31219586730003357, + "rewards/margins": 0.16650965809822083, + "rewards/rejected": -0.4787055253982544, + "step": 631 + }, + { + "epoch": 0.83, + "learning_rate": 4.9388338153821976e-05, + "logits/chosen": -2.256747245788574, + "logits/rejected": -2.2907259464263916, + "logps/chosen": -163.09808349609375, + "logps/rejected": -231.26492309570312, + "loss": 0.5661, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15426135063171387, + "rewards/margins": 0.48811718821525574, + "rewards/rejected": -0.6423785090446472, + "step": 632 + }, + { + "epoch": 0.83, + "learning_rate": 4.938604597922346e-05, + "logits/chosen": -2.5338540077209473, + "logits/rejected": -2.580803871154785, + "logps/chosen": -221.49908447265625, + "logps/rejected": -222.09754943847656, + "loss": 0.7293, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.393398642539978, + "rewards/margins": 0.04478640481829643, + "rewards/rejected": -0.43818506598472595, + "step": 633 + }, + { + "epoch": 0.83, + "learning_rate": 4.9383749571135946e-05, + "logits/chosen": -2.4923906326293945, + "logits/rejected": -2.552914619445801, + "logps/chosen": -188.54759216308594, + "logps/rejected": -201.86276245117188, + "loss": 0.6803, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.19801469147205353, + "rewards/margins": 0.08076779544353485, + "rewards/rejected": -0.2787824869155884, + "step": 634 + }, + { + "epoch": 0.83, + "learning_rate": 4.938144892995809e-05, + "logits/chosen": -2.6368439197540283, + "logits/rejected": -2.681831121444702, + "logps/chosen": -172.92823791503906, + "logps/rejected": -153.6282501220703, + "loss": 0.6554, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1948256492614746, + "rewards/margins": 0.14521902799606323, + "rewards/rejected": -0.34004467725753784, + "step": 635 + }, + { + "epoch": 0.83, + "learning_rate": 4.93791440560893e-05, + "logits/chosen": -2.5344183444976807, + "logits/rejected": -2.5658974647521973, + "logps/chosen": -198.16390991210938, + "logps/rejected": -193.16055297851562, + "loss": 0.6981, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.06259515881538391, + "rewards/margins": 0.05056838318705559, + "rewards/rejected": -0.11316356062889099, + "step": 636 + }, + { + "epoch": 0.83, + "learning_rate": 4.93768349499297e-05, + "logits/chosen": -2.6175780296325684, + "logits/rejected": -2.561044216156006, + "logps/chosen": -194.93130493164062, + "logps/rejected": -171.37437438964844, + "loss": 0.865, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.44466546177864075, + "rewards/margins": -0.2321683019399643, + "rewards/rejected": -0.21249720454216003, + "step": 637 + }, + { + "epoch": 0.83, + "learning_rate": 4.9374521611880166e-05, + "logits/chosen": -2.552360773086548, + "logits/rejected": -2.585489273071289, + "logps/chosen": -157.82420349121094, + "logps/rejected": -171.7198028564453, + "loss": 0.6514, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.19169321656227112, + "rewards/margins": 0.18479710817337036, + "rewards/rejected": -0.3764903247356415, + "step": 638 + }, + { + "epoch": 0.84, + "learning_rate": 4.9372204042342295e-05, + "logits/chosen": -2.55067777633667, + "logits/rejected": -2.6302952766418457, + "logps/chosen": -186.92031860351562, + "logps/rejected": -184.224365234375, + "loss": 0.676, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.2839874029159546, + "rewards/margins": 0.20806796848773956, + "rewards/rejected": -0.49205532670021057, + "step": 639 + }, + { + "epoch": 0.84, + "learning_rate": 4.936988224171842e-05, + "logits/chosen": -2.6061015129089355, + "logits/rejected": -2.609809637069702, + "logps/chosen": -209.86328125, + "logps/rejected": -199.8352813720703, + "loss": 0.6114, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.38792091608047485, + "rewards/margins": 0.29765814542770386, + "rewards/rejected": -0.6855790615081787, + "step": 640 + }, + { + "epoch": 0.84, + "learning_rate": 4.936755621041163e-05, + "logits/chosen": -2.600673198699951, + "logits/rejected": -2.6286747455596924, + "logps/chosen": -217.86569213867188, + "logps/rejected": -208.2232208251953, + "loss": 0.5999, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3269140124320984, + "rewards/margins": 0.2362305074930191, + "rewards/rejected": -0.5631444454193115, + "step": 641 + }, + { + "epoch": 0.84, + "learning_rate": 4.93652259488257e-05, + "logits/chosen": -2.331010341644287, + "logits/rejected": -2.406320571899414, + "logps/chosen": -181.08682250976562, + "logps/rejected": -231.1434326171875, + "loss": 0.6473, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.18573978543281555, + "rewards/margins": 0.11375062167644501, + "rewards/rejected": -0.29949042201042175, + "step": 642 + }, + { + "epoch": 0.84, + "learning_rate": 4.9362891457365194e-05, + "logits/chosen": -2.5639798641204834, + "logits/rejected": -2.511385917663574, + "logps/chosen": -167.39529418945312, + "logps/rejected": -190.17266845703125, + "loss": 0.6687, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.23445367813110352, + "rewards/margins": 0.10250186920166016, + "rewards/rejected": -0.33695557713508606, + "step": 643 + }, + { + "epoch": 0.84, + "learning_rate": 4.9360552736435386e-05, + "logits/chosen": -2.484229326248169, + "logits/rejected": -2.5917751789093018, + "logps/chosen": -160.66372680664062, + "logps/rejected": -151.573974609375, + "loss": 0.7071, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3852943181991577, + "rewards/margins": 0.021035168319940567, + "rewards/rejected": -0.40632951259613037, + "step": 644 + }, + { + "epoch": 0.84, + "learning_rate": 4.935820978644228e-05, + "logits/chosen": -2.4788904190063477, + "logits/rejected": -2.4254837036132812, + "logps/chosen": -161.3828887939453, + "logps/rejected": -179.1514129638672, + "loss": 0.7489, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.27134445309638977, + "rewards/margins": -0.01518234983086586, + "rewards/rejected": -0.2561620771884918, + "step": 645 + }, + { + "epoch": 0.85, + "learning_rate": 4.9355862607792614e-05, + "logits/chosen": -2.3145663738250732, + "logits/rejected": -2.3181071281433105, + "logps/chosen": -175.7195281982422, + "logps/rejected": -180.3649139404297, + "loss": 0.7226, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.2857952117919922, + "rewards/margins": -0.0009603817015886307, + "rewards/rejected": -0.2848348617553711, + "step": 646 + }, + { + "epoch": 0.85, + "learning_rate": 4.9353511200893865e-05, + "logits/chosen": -2.5312018394470215, + "logits/rejected": -2.5117855072021484, + "logps/chosen": -147.04232788085938, + "logps/rejected": -153.8798065185547, + "loss": 0.7482, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.2138766497373581, + "rewards/margins": -0.0894002690911293, + "rewards/rejected": -0.12447639554738998, + "step": 647 + }, + { + "epoch": 0.85, + "learning_rate": 4.935115556615425e-05, + "logits/chosen": -2.5504438877105713, + "logits/rejected": -2.5526883602142334, + "logps/chosen": -198.34768676757812, + "logps/rejected": -190.35696411132812, + "loss": 0.6935, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.393574059009552, + "rewards/margins": 0.1438981145620346, + "rewards/rejected": -0.5374721884727478, + "step": 648 + }, + { + "epoch": 0.85, + "learning_rate": 4.934879570398272e-05, + "logits/chosen": -2.450056314468384, + "logits/rejected": -2.440789222717285, + "logps/chosen": -198.56884765625, + "logps/rejected": -234.9272918701172, + "loss": 0.7365, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.5248376131057739, + "rewards/margins": -0.04359120875597, + "rewards/rejected": -0.4812464118003845, + "step": 649 + }, + { + "epoch": 0.85, + "learning_rate": 4.9346431614788945e-05, + "logits/chosen": -2.481201648712158, + "logits/rejected": -2.540107250213623, + "logps/chosen": -193.42530822753906, + "logps/rejected": -196.03887939453125, + "loss": 0.7004, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.30591022968292236, + "rewards/margins": 0.07436452805995941, + "rewards/rejected": -0.3802747428417206, + "step": 650 + }, + { + "epoch": 0.85, + "learning_rate": 4.934406329898334e-05, + "logits/chosen": -2.4828920364379883, + "logits/rejected": -2.4486544132232666, + "logps/chosen": -174.25201416015625, + "logps/rejected": -167.93795776367188, + "loss": 0.8821, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.4259773790836334, + "rewards/margins": -0.30132219195365906, + "rewards/rejected": -0.12465521693229675, + "step": 651 + }, + { + "epoch": 0.85, + "learning_rate": 4.934169075697704e-05, + "logits/chosen": -2.5756962299346924, + "logits/rejected": -2.53938364982605, + "logps/chosen": -261.4378356933594, + "logps/rejected": -217.41473388671875, + "loss": 0.7193, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.35280755162239075, + "rewards/margins": 0.009605936706066132, + "rewards/rejected": -0.36241352558135986, + "step": 652 + }, + { + "epoch": 0.85, + "learning_rate": 4.9339313989181955e-05, + "logits/chosen": -2.479517936706543, + "logits/rejected": -2.4295461177825928, + "logps/chosen": -168.84185791015625, + "logps/rejected": -171.62991333007812, + "loss": 0.7111, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4041588306427002, + "rewards/margins": 0.014433953911066055, + "rewards/rejected": -0.41859275102615356, + "step": 653 + }, + { + "epoch": 0.86, + "learning_rate": 4.933693299601067e-05, + "logits/chosen": -2.2451162338256836, + "logits/rejected": -2.4006645679473877, + "logps/chosen": -178.17141723632812, + "logps/rejected": -210.85440063476562, + "loss": 0.5213, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.0021903254091739655, + "rewards/margins": 0.5191294550895691, + "rewards/rejected": -0.516939103603363, + "step": 654 + }, + { + "epoch": 0.86, + "learning_rate": 4.933454777787654e-05, + "logits/chosen": -2.6251702308654785, + "logits/rejected": -2.6193838119506836, + "logps/chosen": -261.71185302734375, + "logps/rejected": -258.5350646972656, + "loss": 0.6123, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3498530089855194, + "rewards/margins": 0.23792549967765808, + "rewards/rejected": -0.5877785086631775, + "step": 655 + }, + { + "epoch": 0.86, + "learning_rate": 4.933215833519366e-05, + "logits/chosen": -2.515723466873169, + "logits/rejected": -2.4995017051696777, + "logps/chosen": -170.08506774902344, + "logps/rejected": -171.64215087890625, + "loss": 0.6167, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.03253903239965439, + "rewards/margins": 0.2270340621471405, + "rewards/rejected": -0.2595731019973755, + "step": 656 + }, + { + "epoch": 0.86, + "learning_rate": 4.9329764668376824e-05, + "logits/chosen": -2.3417258262634277, + "logits/rejected": -2.5132431983947754, + "logps/chosen": -182.23594665527344, + "logps/rejected": -204.4223175048828, + "loss": 0.5153, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08154846727848053, + "rewards/margins": 0.5844264626502991, + "rewards/rejected": -0.6659749746322632, + "step": 657 + }, + { + "epoch": 0.86, + "learning_rate": 4.932736677784158e-05, + "logits/chosen": -2.406738758087158, + "logits/rejected": -2.4287314414978027, + "logps/chosen": -175.0724639892578, + "logps/rejected": -212.99569702148438, + "loss": 0.6727, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5249559879302979, + "rewards/margins": 0.10958628356456757, + "rewards/rejected": -0.6345422863960266, + "step": 658 + }, + { + "epoch": 0.86, + "learning_rate": 4.9324964664004226e-05, + "logits/chosen": -2.4244885444641113, + "logits/rejected": -2.511654853820801, + "logps/chosen": -181.31353759765625, + "logps/rejected": -189.31832885742188, + "loss": 0.7071, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.27095070481300354, + "rewards/margins": -0.0028193406760692596, + "rewards/rejected": -0.2681313753128052, + "step": 659 + }, + { + "epoch": 0.86, + "learning_rate": 4.9322558327281773e-05, + "logits/chosen": -2.5121233463287354, + "logits/rejected": -2.6510791778564453, + "logps/chosen": -230.39369201660156, + "logps/rejected": -241.02325439453125, + "loss": 0.4769, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.02516031824052334, + "rewards/margins": 0.551730751991272, + "rewards/rejected": -0.5768911242485046, + "step": 660 + }, + { + "epoch": 0.87, + "learning_rate": 4.9320147768091954e-05, + "logits/chosen": -2.076624631881714, + "logits/rejected": -2.117382287979126, + "logps/chosen": -228.64761352539062, + "logps/rejected": -211.5120849609375, + "loss": 0.696, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.37209880352020264, + "rewards/margins": 0.16731634736061096, + "rewards/rejected": -0.5394151210784912, + "step": 661 + }, + { + "epoch": 0.87, + "learning_rate": 4.931773298685326e-05, + "logits/chosen": -2.3631484508514404, + "logits/rejected": -2.424973249435425, + "logps/chosen": -174.7392120361328, + "logps/rejected": -198.92578125, + "loss": 0.6354, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.23427049815654755, + "rewards/margins": 0.1624229997396469, + "rewards/rejected": -0.39669349789619446, + "step": 662 + }, + { + "epoch": 0.87, + "learning_rate": 4.93153139839849e-05, + "logits/chosen": -2.271193742752075, + "logits/rejected": -2.2948765754699707, + "logps/chosen": -180.6439971923828, + "logps/rejected": -192.5905303955078, + "loss": 0.7982, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.23156273365020752, + "rewards/margins": -0.15015332400798798, + "rewards/rejected": -0.08140941709280014, + "step": 663 + }, + { + "epoch": 0.87, + "learning_rate": 4.931289075990682e-05, + "logits/chosen": -2.6266207695007324, + "logits/rejected": -2.6377651691436768, + "logps/chosen": -214.2593536376953, + "logps/rejected": -202.11019897460938, + "loss": 0.7092, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.25713613629341125, + "rewards/margins": 0.02131093665957451, + "rewards/rejected": -0.2784470319747925, + "step": 664 + }, + { + "epoch": 0.87, + "learning_rate": 4.93104633150397e-05, + "logits/chosen": -2.558809995651245, + "logits/rejected": -2.580552816390991, + "logps/chosen": -200.12879943847656, + "logps/rejected": -270.4982604980469, + "loss": 0.5893, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.21998968720436096, + "rewards/margins": 0.367271363735199, + "rewards/rejected": -0.5872610807418823, + "step": 665 + }, + { + "epoch": 0.87, + "learning_rate": 4.930803164980496e-05, + "logits/chosen": -2.324084520339966, + "logits/rejected": -2.4046638011932373, + "logps/chosen": -199.2420196533203, + "logps/rejected": -188.57501220703125, + "loss": 0.651, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.24593767523765564, + "rewards/margins": 0.17680732905864716, + "rewards/rejected": -0.4227449893951416, + "step": 666 + }, + { + "epoch": 0.87, + "learning_rate": 4.930559576462473e-05, + "logits/chosen": -2.4203665256500244, + "logits/rejected": -2.5350735187530518, + "logps/chosen": -203.4146728515625, + "logps/rejected": -220.17127990722656, + "loss": 0.6676, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.36869651079177856, + "rewards/margins": 0.1714363694190979, + "rewards/rejected": -0.5401328802108765, + "step": 667 + }, + { + "epoch": 0.87, + "learning_rate": 4.930315565992189e-05, + "logits/chosen": -2.5649983882904053, + "logits/rejected": -2.587256669998169, + "logps/chosen": -211.79006958007812, + "logps/rejected": -197.28616333007812, + "loss": 0.6443, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.41877490282058716, + "rewards/margins": 0.1472373902797699, + "rewards/rejected": -0.5660123229026794, + "step": 668 + }, + { + "epoch": 0.88, + "learning_rate": 4.930071133612005e-05, + "logits/chosen": -2.4042892456054688, + "logits/rejected": -2.4121689796447754, + "logps/chosen": -163.4969940185547, + "logps/rejected": -214.19903564453125, + "loss": 0.5355, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12853318452835083, + "rewards/margins": 0.4976612329483032, + "rewards/rejected": -0.626194417476654, + "step": 669 + }, + { + "epoch": 0.88, + "learning_rate": 4.929826279364357e-05, + "logits/chosen": -2.6725361347198486, + "logits/rejected": -2.6962034702301025, + "logps/chosen": -206.29966735839844, + "logps/rejected": -225.02146911621094, + "loss": 0.7439, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.4318912625312805, + "rewards/margins": -0.017002558335661888, + "rewards/rejected": -0.4148887097835541, + "step": 670 + }, + { + "epoch": 0.88, + "learning_rate": 4.9295810032917486e-05, + "logits/chosen": -2.6312649250030518, + "logits/rejected": -2.677178144454956, + "logps/chosen": -233.12554931640625, + "logps/rejected": -249.84649658203125, + "loss": 0.6412, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3292105495929718, + "rewards/margins": 0.18580739200115204, + "rewards/rejected": -0.5150179266929626, + "step": 671 + }, + { + "epoch": 0.88, + "learning_rate": 4.929335305436764e-05, + "logits/chosen": -2.350733995437622, + "logits/rejected": -2.4738504886627197, + "logps/chosen": -159.27565002441406, + "logps/rejected": -197.69430541992188, + "loss": 0.5996, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.3497101664543152, + "rewards/margins": 0.35428282618522644, + "rewards/rejected": -0.7039930820465088, + "step": 672 + }, + { + "epoch": 0.88, + "learning_rate": 4.9290891858420554e-05, + "logits/chosen": -2.341266632080078, + "logits/rejected": -2.412942409515381, + "logps/chosen": -221.41571044921875, + "logps/rejected": -226.8486328125, + "loss": 0.5985, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.34598851203918457, + "rewards/margins": 0.2551480233669281, + "rewards/rejected": -0.6011365652084351, + "step": 673 + }, + { + "epoch": 0.88, + "learning_rate": 4.92884264455035e-05, + "logits/chosen": -2.302943706512451, + "logits/rejected": -2.3145551681518555, + "logps/chosen": -149.34109497070312, + "logps/rejected": -138.53506469726562, + "loss": 0.644, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.1495213508605957, + "rewards/margins": 0.15739065408706665, + "rewards/rejected": -0.30691200494766235, + "step": 674 + }, + { + "epoch": 0.88, + "learning_rate": 4.9285956816044486e-05, + "logits/chosen": -2.228523015975952, + "logits/rejected": -2.173222064971924, + "logps/chosen": -177.70559692382812, + "logps/rejected": -210.69927978515625, + "loss": 0.7657, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.3950153589248657, + "rewards/margins": -0.07809535413980484, + "rewards/rejected": -0.3169199824333191, + "step": 675 + }, + { + "epoch": 0.88, + "learning_rate": 4.9283482970472245e-05, + "logits/chosen": -2.3849382400512695, + "logits/rejected": -2.513273000717163, + "logps/chosen": -185.79124450683594, + "logps/rejected": -201.27066040039062, + "loss": 0.6207, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.13350968062877655, + "rewards/margins": 0.2174827605485916, + "rewards/rejected": -0.35099244117736816, + "step": 676 + }, + { + "epoch": 0.89, + "learning_rate": 4.928100490921625e-05, + "logits/chosen": -2.458615779876709, + "logits/rejected": -2.4962515830993652, + "logps/chosen": -204.7447509765625, + "logps/rejected": -199.8260955810547, + "loss": 0.7519, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6041467189788818, + "rewards/margins": 0.038420408964157104, + "rewards/rejected": -0.6425671577453613, + "step": 677 + }, + { + "epoch": 0.89, + "learning_rate": 4.927852263270667e-05, + "logits/chosen": -2.5404415130615234, + "logits/rejected": -2.5182065963745117, + "logps/chosen": -208.4690399169922, + "logps/rejected": -227.29690551757812, + "loss": 0.6729, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.28778883814811707, + "rewards/margins": 0.08314257860183716, + "rewards/rejected": -0.3709314167499542, + "step": 678 + }, + { + "epoch": 0.89, + "learning_rate": 4.9276036141374473e-05, + "logits/chosen": -2.528585910797119, + "logits/rejected": -2.5861098766326904, + "logps/chosen": -211.7762908935547, + "logps/rejected": -236.7048797607422, + "loss": 0.7562, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4092783033847809, + "rewards/margins": -0.03306754678487778, + "rewards/rejected": -0.3762107491493225, + "step": 679 + }, + { + "epoch": 0.89, + "learning_rate": 4.92735454356513e-05, + "logits/chosen": -2.172372341156006, + "logits/rejected": -2.1387619972229004, + "logps/chosen": -230.44168090820312, + "logps/rejected": -221.9505615234375, + "loss": 0.6126, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.0995088517665863, + "rewards/margins": 0.24330440163612366, + "rewards/rejected": -0.34281325340270996, + "step": 680 + }, + { + "epoch": 0.89, + "learning_rate": 4.927105051596956e-05, + "logits/chosen": -2.5018787384033203, + "logits/rejected": -2.6133289337158203, + "logps/chosen": -159.59141540527344, + "logps/rejected": -165.89492797851562, + "loss": 0.6031, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0614471361041069, + "rewards/margins": 0.24630111455917358, + "rewards/rejected": -0.3077481985092163, + "step": 681 + }, + { + "epoch": 0.89, + "learning_rate": 4.926855138276236e-05, + "logits/chosen": -2.532803773880005, + "logits/rejected": -2.6221201419830322, + "logps/chosen": -192.40725708007812, + "logps/rejected": -176.5653839111328, + "loss": 0.7419, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4705914258956909, + "rewards/margins": 0.014578044414520264, + "rewards/rejected": -0.48516952991485596, + "step": 682 + }, + { + "epoch": 0.89, + "learning_rate": 4.9266048036463566e-05, + "logits/chosen": -2.5754356384277344, + "logits/rejected": -2.668449878692627, + "logps/chosen": -223.462646484375, + "logps/rejected": -249.08425903320312, + "loss": 0.5664, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4348244071006775, + "rewards/margins": 0.38006237149238586, + "rewards/rejected": -0.8148868680000305, + "step": 683 + }, + { + "epoch": 0.9, + "learning_rate": 4.926354047750777e-05, + "logits/chosen": -2.148771047592163, + "logits/rejected": -2.174349546432495, + "logps/chosen": -191.0968780517578, + "logps/rejected": -201.15272521972656, + "loss": 0.7219, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.25711679458618164, + "rewards/margins": 0.0128050297498703, + "rewards/rejected": -0.26992183923721313, + "step": 684 + }, + { + "epoch": 0.9, + "learning_rate": 4.926102870633029e-05, + "logits/chosen": -2.5497798919677734, + "logits/rejected": -2.5441908836364746, + "logps/chosen": -236.3572998046875, + "logps/rejected": -223.45065307617188, + "loss": 0.734, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.43852365016937256, + "rewards/margins": 0.028032146394252777, + "rewards/rejected": -0.46655580401420593, + "step": 685 + }, + { + "epoch": 0.9, + "learning_rate": 4.925851272336717e-05, + "logits/chosen": -2.49284029006958, + "logits/rejected": -2.4962539672851562, + "logps/chosen": -187.33041381835938, + "logps/rejected": -161.27676391601562, + "loss": 0.7157, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4357530474662781, + "rewards/margins": 0.01825621724128723, + "rewards/rejected": -0.4540092647075653, + "step": 686 + }, + { + "epoch": 0.9, + "learning_rate": 4.9255992529055195e-05, + "logits/chosen": -2.4232540130615234, + "logits/rejected": -2.4808590412139893, + "logps/chosen": -177.44873046875, + "logps/rejected": -173.36376953125, + "loss": 0.5234, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.000440012663602829, + "rewards/margins": 0.42676159739494324, + "rewards/rejected": -0.4272015690803528, + "step": 687 + }, + { + "epoch": 0.9, + "learning_rate": 4.925346812383188e-05, + "logits/chosen": -2.5829415321350098, + "logits/rejected": -2.539314031600952, + "logps/chosen": -201.11561584472656, + "logps/rejected": -157.01829528808594, + "loss": 0.659, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.18970540165901184, + "rewards/margins": 0.1848244071006775, + "rewards/rejected": -0.37452977895736694, + "step": 688 + }, + { + "epoch": 0.9, + "learning_rate": 4.925093950813547e-05, + "logits/chosen": -2.1912455558776855, + "logits/rejected": -2.235294818878174, + "logps/chosen": -158.46246337890625, + "logps/rejected": -182.1004180908203, + "loss": 0.7104, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.2667257487773895, + "rewards/margins": 0.0735367089509964, + "rewards/rejected": -0.3402624726295471, + "step": 689 + }, + { + "epoch": 0.9, + "learning_rate": 4.924840668240495e-05, + "logits/chosen": -2.2375993728637695, + "logits/rejected": -2.270219564437866, + "logps/chosen": -206.09689331054688, + "logps/rejected": -185.82130432128906, + "loss": 0.8109, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5142911672592163, + "rewards/margins": -0.055089227855205536, + "rewards/rejected": -0.4592019319534302, + "step": 690 + }, + { + "epoch": 0.9, + "learning_rate": 4.924586964708e-05, + "logits/chosen": -2.683393716812134, + "logits/rejected": -2.745443344116211, + "logps/chosen": -211.4540252685547, + "logps/rejected": -204.97247314453125, + "loss": 0.6848, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.388435035943985, + "rewards/margins": 0.09198732674121857, + "rewards/rejected": -0.48042237758636475, + "step": 691 + }, + { + "epoch": 0.91, + "learning_rate": 4.9243328402601075e-05, + "logits/chosen": -2.4001381397247314, + "logits/rejected": -2.446291208267212, + "logps/chosen": -173.99537658691406, + "logps/rejected": -170.11166381835938, + "loss": 0.6884, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3871830105781555, + "rewards/margins": 0.05776102468371391, + "rewards/rejected": -0.44494396448135376, + "step": 692 + }, + { + "epoch": 0.91, + "learning_rate": 4.924078294940935e-05, + "logits/chosen": -2.3540093898773193, + "logits/rejected": -2.4053566455841064, + "logps/chosen": -238.71336364746094, + "logps/rejected": -232.11026000976562, + "loss": 0.6272, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.38184699416160583, + "rewards/margins": 0.2840040624141693, + "rewards/rejected": -0.6658511161804199, + "step": 693 + }, + { + "epoch": 0.91, + "learning_rate": 4.92382332879467e-05, + "logits/chosen": -2.5318589210510254, + "logits/rejected": -2.570647954940796, + "logps/chosen": -182.59814453125, + "logps/rejected": -190.98745727539062, + "loss": 0.6351, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4041218161582947, + "rewards/margins": 0.2801179587841034, + "rewards/rejected": -0.6842397451400757, + "step": 694 + }, + { + "epoch": 0.91, + "learning_rate": 4.923567941865577e-05, + "logits/chosen": -2.440756320953369, + "logits/rejected": -2.4445791244506836, + "logps/chosen": -174.62588500976562, + "logps/rejected": -186.00222778320312, + "loss": 0.6708, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3223167061805725, + "rewards/margins": 0.1542733609676361, + "rewards/rejected": -0.4765900671482086, + "step": 695 + }, + { + "epoch": 0.91, + "learning_rate": 4.923312134197991e-05, + "logits/chosen": -2.3524816036224365, + "logits/rejected": -2.2868611812591553, + "logps/chosen": -174.77328491210938, + "logps/rejected": -193.611083984375, + "loss": 0.8216, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4883018434047699, + "rewards/margins": -0.1851535439491272, + "rewards/rejected": -0.3031483292579651, + "step": 696 + }, + { + "epoch": 0.91, + "learning_rate": 4.9230559058363223e-05, + "logits/chosen": -2.2453904151916504, + "logits/rejected": -2.2618868350982666, + "logps/chosen": -204.50955200195312, + "logps/rejected": -213.33969116210938, + "loss": 0.6688, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.2155238389968872, + "rewards/margins": 0.14622408151626587, + "rewards/rejected": -0.3617479205131531, + "step": 697 + }, + { + "epoch": 0.91, + "learning_rate": 4.922799256825051e-05, + "logits/chosen": -2.2087559700012207, + "logits/rejected": -2.289468765258789, + "logps/chosen": -223.2991943359375, + "logps/rejected": -193.14785766601562, + "loss": 0.8451, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.45806586742401123, + "rewards/margins": -0.23903487622737885, + "rewards/rejected": -0.21903102099895477, + "step": 698 + }, + { + "epoch": 0.91, + "learning_rate": 4.9225421872087344e-05, + "logits/chosen": -2.6118435859680176, + "logits/rejected": -2.5954737663269043, + "logps/chosen": -218.72201538085938, + "logps/rejected": -223.23770141601562, + "loss": 0.6797, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.33901646733283997, + "rewards/margins": 0.09220410138368607, + "rewards/rejected": -0.43122053146362305, + "step": 699 + }, + { + "epoch": 0.92, + "learning_rate": 4.922284697031999e-05, + "logits/chosen": -2.532050371170044, + "logits/rejected": -2.5437393188476562, + "logps/chosen": -205.47463989257812, + "logps/rejected": -194.44210815429688, + "loss": 0.6153, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.293450266122818, + "rewards/margins": 0.32601144909858704, + "rewards/rejected": -0.619461715221405, + "step": 700 + }, + { + "epoch": 0.92, + "learning_rate": 4.922026786339545e-05, + "logits/chosen": -2.4742062091827393, + "logits/rejected": -2.507223606109619, + "logps/chosen": -249.90118408203125, + "logps/rejected": -243.16616821289062, + "loss": 0.6483, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.3557482063770294, + "rewards/margins": 0.16462744772434235, + "rewards/rejected": -0.5203756093978882, + "step": 701 + }, + { + "epoch": 0.92, + "learning_rate": 4.921768455176149e-05, + "logits/chosen": -2.313598155975342, + "logits/rejected": -2.38657546043396, + "logps/chosen": -218.81178283691406, + "logps/rejected": -273.9117126464844, + "loss": 0.5901, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16350752115249634, + "rewards/margins": 0.2863968014717102, + "rewards/rejected": -0.44990429282188416, + "step": 702 + }, + { + "epoch": 0.92, + "learning_rate": 4.921509703586657e-05, + "logits/chosen": -2.4725236892700195, + "logits/rejected": -2.46490216255188, + "logps/chosen": -204.388916015625, + "logps/rejected": -202.4940948486328, + "loss": 0.6274, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11353873461484909, + "rewards/margins": 0.18249502778053284, + "rewards/rejected": -0.2960337698459625, + "step": 703 + }, + { + "epoch": 0.92, + "learning_rate": 4.9212505316159876e-05, + "logits/chosen": -2.4417524337768555, + "logits/rejected": -2.4327855110168457, + "logps/chosen": -185.76559448242188, + "logps/rejected": -185.82675170898438, + "loss": 0.6754, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.16719910502433777, + "rewards/margins": 0.131285160779953, + "rewards/rejected": -0.29848429560661316, + "step": 704 + }, + { + "epoch": 0.92, + "learning_rate": 4.920990939309135e-05, + "logits/chosen": -2.405703544616699, + "logits/rejected": -2.4366328716278076, + "logps/chosen": -182.18052673339844, + "logps/rejected": -177.89529418945312, + "loss": 0.5939, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0767878070473671, + "rewards/margins": 0.26920846104621887, + "rewards/rejected": -0.34599626064300537, + "step": 705 + }, + { + "epoch": 0.92, + "learning_rate": 4.9207309267111654e-05, + "logits/chosen": -2.267641067504883, + "logits/rejected": -2.359164237976074, + "logps/chosen": -167.9193115234375, + "logps/rejected": -198.30853271484375, + "loss": 0.5458, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.17353487014770508, + "rewards/margins": 0.4786756932735443, + "rewards/rejected": -0.6522104144096375, + "step": 706 + }, + { + "epoch": 0.93, + "learning_rate": 4.920470493867217e-05, + "logits/chosen": -2.3301897048950195, + "logits/rejected": -2.287158489227295, + "logps/chosen": -179.1868133544922, + "logps/rejected": -173.25205993652344, + "loss": 0.8096, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.3885132968425751, + "rewards/margins": -0.15448781847953796, + "rewards/rejected": -0.2340254783630371, + "step": 707 + }, + { + "epoch": 0.93, + "learning_rate": 4.920209640822503e-05, + "logits/chosen": -2.3286001682281494, + "logits/rejected": -2.372384786605835, + "logps/chosen": -183.89163208007812, + "logps/rejected": -171.33837890625, + "loss": 0.6243, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.200511634349823, + "rewards/margins": 0.1888035386800766, + "rewards/rejected": -0.3893151581287384, + "step": 708 + }, + { + "epoch": 0.93, + "learning_rate": 4.919948367622307e-05, + "logits/chosen": -2.049145221710205, + "logits/rejected": -2.1091737747192383, + "logps/chosen": -165.6485595703125, + "logps/rejected": -196.3398895263672, + "loss": 0.6972, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.21507728099822998, + "rewards/margins": 0.1645798683166504, + "rewards/rejected": -0.37965714931488037, + "step": 709 + }, + { + "epoch": 0.93, + "learning_rate": 4.919686674311987e-05, + "logits/chosen": -2.2241785526275635, + "logits/rejected": -2.1966073513031006, + "logps/chosen": -197.89120483398438, + "logps/rejected": -222.09132385253906, + "loss": 0.6355, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.12498195469379425, + "rewards/margins": 0.2979465425014496, + "rewards/rejected": -0.42292848229408264, + "step": 710 + }, + { + "epoch": 0.93, + "learning_rate": 4.9194245609369746e-05, + "logits/chosen": -2.3444929122924805, + "logits/rejected": -2.3332207202911377, + "logps/chosen": -240.3154296875, + "logps/rejected": -229.0690460205078, + "loss": 0.6236, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17825375497341156, + "rewards/margins": 0.23597893118858337, + "rewards/rejected": -0.41423267126083374, + "step": 711 + }, + { + "epoch": 0.93, + "learning_rate": 4.9191620275427725e-05, + "logits/chosen": -2.472026824951172, + "logits/rejected": -2.424983501434326, + "logps/chosen": -212.58364868164062, + "logps/rejected": -186.08990478515625, + "loss": 0.684, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.1622922122478485, + "rewards/margins": 0.10522370040416718, + "rewards/rejected": -0.2675159275531769, + "step": 712 + }, + { + "epoch": 0.93, + "learning_rate": 4.9188990741749576e-05, + "logits/chosen": -2.481387138366699, + "logits/rejected": -2.4912960529327393, + "logps/chosen": -259.65753173828125, + "logps/rejected": -245.88864135742188, + "loss": 0.6385, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3589114546775818, + "rewards/margins": 0.13590216636657715, + "rewards/rejected": -0.49481362104415894, + "step": 713 + }, + { + "epoch": 0.93, + "learning_rate": 4.918635700879179e-05, + "logits/chosen": -2.4197511672973633, + "logits/rejected": -2.442139148712158, + "logps/chosen": -180.3101806640625, + "logps/rejected": -201.78469848632812, + "loss": 0.696, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.27619680762290955, + "rewards/margins": 0.04660003259778023, + "rewards/rejected": -0.32279688119888306, + "step": 714 + }, + { + "epoch": 0.94, + "learning_rate": 4.918371907701159e-05, + "logits/chosen": -2.333615303039551, + "logits/rejected": -2.3523952960968018, + "logps/chosen": -176.4866485595703, + "logps/rejected": -190.32089233398438, + "loss": 0.755, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.353998064994812, + "rewards/margins": -0.033618077635765076, + "rewards/rejected": -0.32037997245788574, + "step": 715 + }, + { + "epoch": 0.94, + "learning_rate": 4.9181076946866944e-05, + "logits/chosen": -2.45198655128479, + "logits/rejected": -2.432290554046631, + "logps/chosen": -178.83279418945312, + "logps/rejected": -188.7766876220703, + "loss": 0.7776, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.4141733944416046, + "rewards/margins": -0.06833554804325104, + "rewards/rejected": -0.3458378314971924, + "step": 716 + }, + { + "epoch": 0.94, + "learning_rate": 4.917843061881652e-05, + "logits/chosen": -2.4485764503479004, + "logits/rejected": -2.412081241607666, + "logps/chosen": -229.4910430908203, + "logps/rejected": -225.20999145507812, + "loss": 0.7713, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4986688494682312, + "rewards/margins": -0.1049906462430954, + "rewards/rejected": -0.3936781883239746, + "step": 717 + }, + { + "epoch": 0.94, + "learning_rate": 4.9175780093319723e-05, + "logits/chosen": -2.619525194168091, + "logits/rejected": -2.5976409912109375, + "logps/chosen": -152.8652801513672, + "logps/rejected": -190.03179931640625, + "loss": 0.6384, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.044289495795965195, + "rewards/margins": 0.1848900467157364, + "rewards/rejected": -0.2291795164346695, + "step": 718 + }, + { + "epoch": 0.94, + "learning_rate": 4.91731253708367e-05, + "logits/chosen": -2.2807188034057617, + "logits/rejected": -2.338017463684082, + "logps/chosen": -182.47259521484375, + "logps/rejected": -196.39678955078125, + "loss": 0.6927, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.26287615299224854, + "rewards/margins": 0.050617627799510956, + "rewards/rejected": -0.3134937882423401, + "step": 719 + }, + { + "epoch": 0.94, + "learning_rate": 4.9170466451828326e-05, + "logits/chosen": -2.2561631202697754, + "logits/rejected": -2.3567490577697754, + "logps/chosen": -242.33407592773438, + "logps/rejected": -263.4303283691406, + "loss": 0.7338, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.2756967544555664, + "rewards/margins": 0.13067740201950073, + "rewards/rejected": -0.40637409687042236, + "step": 720 + }, + { + "epoch": 0.94, + "learning_rate": 4.916780333675618e-05, + "logits/chosen": -2.5038459300994873, + "logits/rejected": -2.500086545944214, + "logps/chosen": -187.3552703857422, + "logps/rejected": -187.51268005371094, + "loss": 0.6628, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.2900952398777008, + "rewards/margins": 0.16025833785533905, + "rewards/rejected": -0.4503535330295563, + "step": 721 + }, + { + "epoch": 0.94, + "learning_rate": 4.9165136026082604e-05, + "logits/chosen": -2.2002265453338623, + "logits/rejected": -2.226296901702881, + "logps/chosen": -200.2354278564453, + "logps/rejected": -195.80328369140625, + "loss": 0.6676, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4443664848804474, + "rewards/margins": 0.14572381973266602, + "rewards/rejected": -0.5900903344154358, + "step": 722 + }, + { + "epoch": 0.95, + "learning_rate": 4.916246452027063e-05, + "logits/chosen": -2.1249847412109375, + "logits/rejected": -2.116081953048706, + "logps/chosen": -214.81161499023438, + "logps/rejected": -220.05429077148438, + "loss": 0.664, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.25689569115638733, + "rewards/margins": 0.1331162005662918, + "rewards/rejected": -0.39001190662384033, + "step": 723 + }, + { + "epoch": 0.95, + "learning_rate": 4.9159788819784064e-05, + "logits/chosen": -2.3239493370056152, + "logits/rejected": -2.3413610458374023, + "logps/chosen": -209.00692749023438, + "logps/rejected": -198.1204376220703, + "loss": 0.627, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.2650049328804016, + "rewards/margins": 0.20938965678215027, + "rewards/rejected": -0.47439464926719666, + "step": 724 + }, + { + "epoch": 0.95, + "learning_rate": 4.9157108925087405e-05, + "logits/chosen": -2.420534610748291, + "logits/rejected": -2.442646026611328, + "logps/chosen": -211.38424682617188, + "logps/rejected": -207.95278930664062, + "loss": 0.5418, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.22766509652137756, + "rewards/margins": 0.3782108724117279, + "rewards/rejected": -0.6058759689331055, + "step": 725 + }, + { + "epoch": 0.95, + "learning_rate": 4.915442483664588e-05, + "logits/chosen": -2.6422438621520996, + "logits/rejected": -2.4694361686706543, + "logps/chosen": -173.2271728515625, + "logps/rejected": -145.02798461914062, + "loss": 0.8003, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.25257498025894165, + "rewards/margins": -0.11290948837995529, + "rewards/rejected": -0.13966546952724457, + "step": 726 + }, + { + "epoch": 0.95, + "learning_rate": 4.915173655492547e-05, + "logits/chosen": -2.5392398834228516, + "logits/rejected": -2.4417877197265625, + "logps/chosen": -228.2000732421875, + "logps/rejected": -199.39288330078125, + "loss": 0.609, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.23776192963123322, + "rewards/margins": 0.23510348796844482, + "rewards/rejected": -0.47286540269851685, + "step": 727 + }, + { + "epoch": 0.95, + "learning_rate": 4.914904408039286e-05, + "logits/chosen": -2.4822921752929688, + "logits/rejected": -2.5780792236328125, + "logps/chosen": -194.31045532226562, + "logps/rejected": -235.24607849121094, + "loss": 0.5732, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3551061451435089, + "rewards/margins": 0.39131835103034973, + "rewards/rejected": -0.7464244961738586, + "step": 728 + }, + { + "epoch": 0.95, + "learning_rate": 4.914634741351547e-05, + "logits/chosen": -2.4005560874938965, + "logits/rejected": -2.4636473655700684, + "logps/chosen": -222.8491668701172, + "logps/rejected": -229.69638061523438, + "loss": 0.7059, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.444613516330719, + "rewards/margins": 0.05394814535975456, + "rewards/rejected": -0.49856165051460266, + "step": 729 + }, + { + "epoch": 0.96, + "learning_rate": 4.914364655476146e-05, + "logits/chosen": -2.541283369064331, + "logits/rejected": -2.4821250438690186, + "logps/chosen": -199.74075317382812, + "logps/rejected": -244.3222198486328, + "loss": 0.5754, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.24406713247299194, + "rewards/margins": 0.44718241691589355, + "rewards/rejected": -0.6912496089935303, + "step": 730 + }, + { + "epoch": 0.96, + "learning_rate": 4.914094150459969e-05, + "logits/chosen": -2.3721487522125244, + "logits/rejected": -2.4787163734436035, + "logps/chosen": -185.42625427246094, + "logps/rejected": -214.8418426513672, + "loss": 0.741, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5116318464279175, + "rewards/margins": -0.03801153600215912, + "rewards/rejected": -0.47362035512924194, + "step": 731 + }, + { + "epoch": 0.96, + "learning_rate": 4.9138232263499784e-05, + "logits/chosen": -2.5538036823272705, + "logits/rejected": -2.467003583908081, + "logps/chosen": -197.03834533691406, + "logps/rejected": -214.0052947998047, + "loss": 0.812, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.5602855682373047, + "rewards/margins": -0.166466623544693, + "rewards/rejected": -0.3938189446926117, + "step": 732 + }, + { + "epoch": 0.96, + "learning_rate": 4.913551883193206e-05, + "logits/chosen": -2.4558639526367188, + "logits/rejected": -2.5085341930389404, + "logps/chosen": -225.220947265625, + "logps/rejected": -227.97213745117188, + "loss": 0.6471, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2800041437149048, + "rewards/margins": 0.34519922733306885, + "rewards/rejected": -0.6252033114433289, + "step": 733 + }, + { + "epoch": 0.96, + "learning_rate": 4.9132801210367586e-05, + "logits/chosen": -2.405069351196289, + "logits/rejected": -2.4901866912841797, + "logps/chosen": -152.56527709960938, + "logps/rejected": -152.64671325683594, + "loss": 0.6334, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.3601643145084381, + "rewards/margins": 0.2201770395040512, + "rewards/rejected": -0.5803413987159729, + "step": 734 + }, + { + "epoch": 0.96, + "learning_rate": 4.913007939927814e-05, + "logits/chosen": -2.3362531661987305, + "logits/rejected": -2.3917181491851807, + "logps/chosen": -193.826904296875, + "logps/rejected": -231.73898315429688, + "loss": 0.6171, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5505282282829285, + "rewards/margins": 0.3396502137184143, + "rewards/rejected": -0.890178382396698, + "step": 735 + }, + { + "epoch": 0.96, + "learning_rate": 4.912735339913625e-05, + "logits/chosen": -2.132725715637207, + "logits/rejected": -2.2681026458740234, + "logps/chosen": -198.4554443359375, + "logps/rejected": -205.1267852783203, + "loss": 0.6419, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.34412312507629395, + "rewards/margins": 0.1572694331407547, + "rewards/rejected": -0.5013926029205322, + "step": 736 + }, + { + "epoch": 0.96, + "learning_rate": 4.912462321041513e-05, + "logits/chosen": -2.352013349533081, + "logits/rejected": -2.474015235900879, + "logps/chosen": -216.76731872558594, + "logps/rejected": -208.0941162109375, + "loss": 0.7258, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.49095839262008667, + "rewards/margins": -0.004371330142021179, + "rewards/rejected": -0.4865871071815491, + "step": 737 + }, + { + "epoch": 0.97, + "learning_rate": 4.9121888833588795e-05, + "logits/chosen": -2.395151138305664, + "logits/rejected": -2.40535306930542, + "logps/chosen": -163.3991241455078, + "logps/rejected": -184.1988525390625, + "loss": 0.6083, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.35962533950805664, + "rewards/margins": 0.20414502918720245, + "rewards/rejected": -0.5637703537940979, + "step": 738 + }, + { + "epoch": 0.97, + "learning_rate": 4.91191502691319e-05, + "logits/chosen": -2.3076629638671875, + "logits/rejected": -2.460707426071167, + "logps/chosen": -163.59388732910156, + "logps/rejected": -190.1605987548828, + "loss": 0.6092, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.2011553794145584, + "rewards/margins": 0.24963317811489105, + "rewards/rejected": -0.45078855752944946, + "step": 739 + }, + { + "epoch": 0.97, + "learning_rate": 4.911640751751988e-05, + "logits/chosen": -2.419591188430786, + "logits/rejected": -2.537069797515869, + "logps/chosen": -250.05996704101562, + "logps/rejected": -225.0284881591797, + "loss": 0.7242, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6081552505493164, + "rewards/margins": 0.06761530041694641, + "rewards/rejected": -0.6757705807685852, + "step": 740 + }, + { + "epoch": 0.97, + "learning_rate": 4.9113660579228886e-05, + "logits/chosen": -2.4080655574798584, + "logits/rejected": -2.4866342544555664, + "logps/chosen": -168.5004119873047, + "logps/rejected": -216.32675170898438, + "loss": 0.6338, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5959919691085815, + "rewards/margins": 0.22804662585258484, + "rewards/rejected": -0.8240386247634888, + "step": 741 + }, + { + "epoch": 0.97, + "learning_rate": 4.91109094547358e-05, + "logits/chosen": -2.2880287170410156, + "logits/rejected": -2.4074809551239014, + "logps/chosen": -184.07696533203125, + "logps/rejected": -215.08717346191406, + "loss": 0.6027, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.49041178822517395, + "rewards/margins": 0.23776140809059143, + "rewards/rejected": -0.7281732559204102, + "step": 742 + }, + { + "epoch": 0.97, + "learning_rate": 4.910815414451821e-05, + "logits/chosen": -2.4181840419769287, + "logits/rejected": -2.4892990589141846, + "logps/chosen": -215.46221923828125, + "logps/rejected": -229.75799560546875, + "loss": 0.608, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7831647992134094, + "rewards/margins": 0.38174957036972046, + "rewards/rejected": -1.1649143695831299, + "step": 743 + }, + { + "epoch": 0.97, + "learning_rate": 4.910539464905447e-05, + "logits/chosen": -2.2858383655548096, + "logits/rejected": -2.3867931365966797, + "logps/chosen": -257.6205749511719, + "logps/rejected": -225.37481689453125, + "loss": 0.6948, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5961012840270996, + "rewards/margins": 0.15920449793338776, + "rewards/rejected": -0.7553057670593262, + "step": 744 + }, + { + "epoch": 0.97, + "learning_rate": 4.910263096882362e-05, + "logits/chosen": -2.2314653396606445, + "logits/rejected": -2.242586851119995, + "logps/chosen": -157.20448303222656, + "logps/rejected": -189.92567443847656, + "loss": 0.7938, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.5152949690818787, + "rewards/margins": -0.07010670006275177, + "rewards/rejected": -0.4451882541179657, + "step": 745 + }, + { + "epoch": 0.98, + "learning_rate": 4.9099863104305446e-05, + "logits/chosen": -2.34870982170105, + "logits/rejected": -2.3678641319274902, + "logps/chosen": -157.9035186767578, + "logps/rejected": -194.49575805664062, + "loss": 0.6431, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.39278292655944824, + "rewards/margins": 0.1728055626153946, + "rewards/rejected": -0.5655884742736816, + "step": 746 + }, + { + "epoch": 0.98, + "learning_rate": 4.909709105598045e-05, + "logits/chosen": -2.290536403656006, + "logits/rejected": -2.316704034805298, + "logps/chosen": -173.55892944335938, + "logps/rejected": -190.94125366210938, + "loss": 0.6078, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3803100883960724, + "rewards/margins": 0.2535950839519501, + "rewards/rejected": -0.6339051723480225, + "step": 747 + }, + { + "epoch": 0.98, + "learning_rate": 4.9094314824329876e-05, + "logits/chosen": -2.5316176414489746, + "logits/rejected": -2.4976158142089844, + "logps/chosen": -206.0629119873047, + "logps/rejected": -219.75418090820312, + "loss": 0.8139, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.5901838541030884, + "rewards/margins": -0.1481025069952011, + "rewards/rejected": -0.4420813322067261, + "step": 748 + }, + { + "epoch": 0.98, + "learning_rate": 4.9091534409835694e-05, + "logits/chosen": -2.500584363937378, + "logits/rejected": -2.5805132389068604, + "logps/chosen": -174.99212646484375, + "logps/rejected": -207.73011779785156, + "loss": 0.574, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4852156937122345, + "rewards/margins": 0.3404516577720642, + "rewards/rejected": -0.8256674408912659, + "step": 749 + }, + { + "epoch": 0.98, + "learning_rate": 4.908874981298057e-05, + "logits/chosen": -2.5304508209228516, + "logits/rejected": -2.6482512950897217, + "logps/chosen": -160.60971069335938, + "logps/rejected": -178.6311492919922, + "loss": 0.7716, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4607134461402893, + "rewards/margins": -0.038665421307086945, + "rewards/rejected": -0.42204800248146057, + "step": 750 + }, + { + "epoch": 0.98, + "learning_rate": 4.908596103424794e-05, + "logits/chosen": -2.247166633605957, + "logits/rejected": -2.2175076007843018, + "logps/chosen": -185.73583984375, + "logps/rejected": -191.45596313476562, + "loss": 0.8569, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.6400306820869446, + "rewards/margins": -0.2516973912715912, + "rewards/rejected": -0.3883333206176758, + "step": 751 + }, + { + "epoch": 0.98, + "learning_rate": 4.908316807412194e-05, + "logits/chosen": -2.492708206176758, + "logits/rejected": -2.585394859313965, + "logps/chosen": -213.05731201171875, + "logps/rejected": -260.92462158203125, + "loss": 0.6273, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5991235971450806, + "rewards/margins": 0.2069501429796219, + "rewards/rejected": -0.806073784828186, + "step": 752 + }, + { + "epoch": 0.99, + "learning_rate": 4.908037093308742e-05, + "logits/chosen": -2.5198185443878174, + "logits/rejected": -2.5527148246765137, + "logps/chosen": -238.94021606445312, + "logps/rejected": -262.2608337402344, + "loss": 0.699, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.38716763257980347, + "rewards/margins": 0.033374398946762085, + "rewards/rejected": -0.42054203152656555, + "step": 753 + }, + { + "epoch": 0.99, + "learning_rate": 4.907756961162999e-05, + "logits/chosen": -2.26259708404541, + "logits/rejected": -2.2545177936553955, + "logps/chosen": -150.26882934570312, + "logps/rejected": -227.8256072998047, + "loss": 0.579, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3753708302974701, + "rewards/margins": 0.4033656418323517, + "rewards/rejected": -0.7787365317344666, + "step": 754 + }, + { + "epoch": 0.99, + "learning_rate": 4.907476411023596e-05, + "logits/chosen": -2.3281123638153076, + "logits/rejected": -2.330186128616333, + "logps/chosen": -149.7628936767578, + "logps/rejected": -193.10093688964844, + "loss": 0.7339, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6727653741836548, + "rewards/margins": 0.06607560813426971, + "rewards/rejected": -0.7388409376144409, + "step": 755 + }, + { + "epoch": 0.99, + "learning_rate": 4.907195442939237e-05, + "logits/chosen": -2.4469387531280518, + "logits/rejected": -2.509931802749634, + "logps/chosen": -174.05670166015625, + "logps/rejected": -189.08465576171875, + "loss": 0.701, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5176682472229004, + "rewards/margins": 0.04994421452283859, + "rewards/rejected": -0.5676124095916748, + "step": 756 + }, + { + "epoch": 0.99, + "learning_rate": 4.906914056958699e-05, + "logits/chosen": -2.6258902549743652, + "logits/rejected": -2.6027467250823975, + "logps/chosen": -208.80203247070312, + "logps/rejected": -200.4151611328125, + "loss": 0.7588, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6569511294364929, + "rewards/margins": -0.044889748096466064, + "rewards/rejected": -0.6120614409446716, + "step": 757 + }, + { + "epoch": 0.99, + "learning_rate": 4.906632253130833e-05, + "logits/chosen": -2.5387301445007324, + "logits/rejected": -2.496183156967163, + "logps/chosen": -193.3511199951172, + "logps/rejected": -199.80130004882812, + "loss": 0.7945, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6182142496109009, + "rewards/margins": -0.13901755213737488, + "rewards/rejected": -0.4791966676712036, + "step": 758 + }, + { + "epoch": 0.99, + "learning_rate": 4.9063500315045586e-05, + "logits/chosen": -2.5884640216827393, + "logits/rejected": -2.641512393951416, + "logps/chosen": -186.569091796875, + "logps/rejected": -200.59988403320312, + "loss": 0.7447, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6641149520874023, + "rewards/margins": -0.048228248953819275, + "rewards/rejected": -0.6158866882324219, + "step": 759 + }, + { + "epoch": 0.99, + "learning_rate": 4.9060673921288716e-05, + "logits/chosen": -2.633413791656494, + "logits/rejected": -2.598926305770874, + "logps/chosen": -161.99928283691406, + "logps/rejected": -154.25979614257812, + "loss": 0.7324, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.37947404384613037, + "rewards/margins": -0.021064819768071175, + "rewards/rejected": -0.35840916633605957, + "step": 760 + }, + { + "epoch": 1.0, + "learning_rate": 4.905784335052839e-05, + "logits/chosen": -2.534295082092285, + "logits/rejected": -2.5714004039764404, + "logps/chosen": -213.0132598876953, + "logps/rejected": -223.10958862304688, + "loss": 0.685, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.7191438674926758, + "rewards/margins": 0.12883953750133514, + "rewards/rejected": -0.8479833602905273, + "step": 761 + }, + { + "epoch": 1.0, + "learning_rate": 4.905500860325599e-05, + "logits/chosen": -2.3805532455444336, + "logits/rejected": -2.502260208129883, + "logps/chosen": -187.92947387695312, + "logps/rejected": -244.21273803710938, + "loss": 0.5568, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.30298084020614624, + "rewards/margins": 0.5082517862319946, + "rewards/rejected": -0.8112326264381409, + "step": 762 + }, + { + "epoch": 1.0, + "learning_rate": 4.905216967996367e-05, + "logits/chosen": -2.494596242904663, + "logits/rejected": -2.5522210597991943, + "logps/chosen": -193.31663513183594, + "logps/rejected": -253.28793334960938, + "loss": 0.5326, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.31040558218955994, + "rewards/margins": 0.41559797525405884, + "rewards/rejected": -0.7260035872459412, + "step": 763 + }, + { + "epoch": 1.0, + "learning_rate": 4.904932658114423e-05, + "logits/chosen": -2.4357709884643555, + "logits/rejected": -2.402021884918213, + "logps/chosen": -207.14736938476562, + "logps/rejected": -224.48757934570312, + "loss": 0.7383, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6394506692886353, + "rewards/margins": 0.015030726790428162, + "rewards/rejected": -0.6544813513755798, + "step": 764 + }, + { + "epoch": 1.0, + "learning_rate": 4.904647930729128e-05, + "logits/chosen": -2.492413282394409, + "logits/rejected": -2.5191187858581543, + "logps/chosen": -209.2412872314453, + "logps/rejected": -204.59518432617188, + "loss": 0.2419, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.42869845032691956, + "rewards/margins": 2.3007590770721436, + "rewards/rejected": -1.872060775756836, + "step": 765 + }, + { + "epoch": 1.0, + "learning_rate": 4.9043627858899086e-05, + "logits/chosen": -2.616361618041992, + "logits/rejected": -2.640256404876709, + "logps/chosen": -180.7271270751953, + "logps/rejected": -231.54254150390625, + "loss": 0.1455, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1313096433877945, + "rewards/margins": 2.220848321914673, + "rewards/rejected": -2.08953857421875, + "step": 766 + }, + { + "epoch": 1.0, + "learning_rate": 4.9040772236462695e-05, + "logits/chosen": -2.3850975036621094, + "logits/rejected": -2.354506254196167, + "logps/chosen": -176.04693603515625, + "logps/rejected": -191.62985229492188, + "loss": 0.2206, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09096850454807281, + "rewards/margins": 1.8953343629837036, + "rewards/rejected": -1.8043657541275024, + "step": 767 + }, + { + "epoch": 1.01, + "learning_rate": 4.903791244047783e-05, + "logits/chosen": -2.448336362838745, + "logits/rejected": -2.4442625045776367, + "logps/chosen": -159.71731567382812, + "logps/rejected": -198.8458251953125, + "loss": 0.2542, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14124417304992676, + "rewards/margins": 1.9044493436813354, + "rewards/rejected": -1.7632049322128296, + "step": 768 + }, + { + "epoch": 1.01, + "learning_rate": 4.9035048471440966e-05, + "logits/chosen": -2.39510178565979, + "logits/rejected": -2.3210549354553223, + "logps/chosen": -146.8027801513672, + "logps/rejected": -182.59580993652344, + "loss": 0.2789, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.44124722480773926, + "rewards/margins": 2.291222095489502, + "rewards/rejected": -1.8499748706817627, + "step": 769 + }, + { + "epoch": 1.01, + "learning_rate": 4.90321803298493e-05, + "logits/chosen": -2.610016107559204, + "logits/rejected": -2.5617635250091553, + "logps/chosen": -191.46194458007812, + "logps/rejected": -232.915283203125, + "loss": 0.1781, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.716643214225769, + "rewards/margins": 2.4686214923858643, + "rewards/rejected": -1.7519782781600952, + "step": 770 + }, + { + "epoch": 1.01, + "learning_rate": 4.9029308016200735e-05, + "logits/chosen": -2.512070655822754, + "logits/rejected": -2.522047758102417, + "logps/chosen": -228.5362548828125, + "logps/rejected": -267.5350341796875, + "loss": 0.1084, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.7526074051856995, + "rewards/margins": 3.0486981868743896, + "rewards/rejected": -2.296090602874756, + "step": 771 + }, + { + "epoch": 1.01, + "learning_rate": 4.902643153099394e-05, + "logits/chosen": -2.7084665298461914, + "logits/rejected": -2.702622413635254, + "logps/chosen": -214.2321319580078, + "logps/rejected": -234.52236938476562, + "loss": 0.2319, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.38715142011642456, + "rewards/margins": 2.1679859161376953, + "rewards/rejected": -1.780834436416626, + "step": 772 + }, + { + "epoch": 1.01, + "learning_rate": 4.902355087472826e-05, + "logits/chosen": -2.4830422401428223, + "logits/rejected": -2.449504852294922, + "logps/chosen": -157.030517578125, + "logps/rejected": -182.37347412109375, + "loss": 0.2089, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3868522644042969, + "rewards/margins": 2.223161220550537, + "rewards/rejected": -1.8363089561462402, + "step": 773 + }, + { + "epoch": 1.01, + "learning_rate": 4.902066604790379e-05, + "logits/chosen": -2.387758731842041, + "logits/rejected": -2.3118813037872314, + "logps/chosen": -216.08274841308594, + "logps/rejected": -232.153076171875, + "loss": 0.3762, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.1578804850578308, + "rewards/margins": 1.8973913192749023, + "rewards/rejected": -1.7395107746124268, + "step": 774 + }, + { + "epoch": 1.01, + "learning_rate": 4.901777705102135e-05, + "logits/chosen": -2.4494662284851074, + "logits/rejected": -2.5127110481262207, + "logps/chosen": -173.77740478515625, + "logps/rejected": -214.7391357421875, + "loss": 0.1939, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14900563657283783, + "rewards/margins": 2.133729934692383, + "rewards/rejected": -1.9847242832183838, + "step": 775 + }, + { + "epoch": 1.02, + "learning_rate": 4.901488388458247e-05, + "logits/chosen": -2.4321305751800537, + "logits/rejected": -2.449230432510376, + "logps/chosen": -173.65748596191406, + "logps/rejected": -176.95252990722656, + "loss": 0.218, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.25331249833106995, + "rewards/margins": 2.11264967918396, + "rewards/rejected": -1.8593370914459229, + "step": 776 + }, + { + "epoch": 1.02, + "learning_rate": 4.901198654908942e-05, + "logits/chosen": -2.3953897953033447, + "logits/rejected": -2.4313783645629883, + "logps/chosen": -207.082275390625, + "logps/rejected": -261.56427001953125, + "loss": 0.1348, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4747293293476105, + "rewards/margins": 3.3515007495880127, + "rewards/rejected": -2.8767709732055664, + "step": 777 + }, + { + "epoch": 1.02, + "learning_rate": 4.9009085045045175e-05, + "logits/chosen": -2.613166093826294, + "logits/rejected": -2.6309146881103516, + "logps/chosen": -218.4897003173828, + "logps/rejected": -231.17636108398438, + "loss": 0.2987, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.09049064666032791, + "rewards/margins": 1.798804521560669, + "rewards/rejected": -1.8892953395843506, + "step": 778 + }, + { + "epoch": 1.02, + "learning_rate": 4.9006179372953466e-05, + "logits/chosen": -2.4088118076324463, + "logits/rejected": -2.3893849849700928, + "logps/chosen": -231.118408203125, + "logps/rejected": -272.7298278808594, + "loss": 0.1272, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.22063946723937988, + "rewards/margins": 3.302457332611084, + "rewards/rejected": -3.0818183422088623, + "step": 779 + }, + { + "epoch": 1.02, + "learning_rate": 4.9003269533318704e-05, + "logits/chosen": -2.716017961502075, + "logits/rejected": -2.5699970722198486, + "logps/chosen": -216.4443817138672, + "logps/rejected": -191.3568115234375, + "loss": 0.1168, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5278897285461426, + "rewards/margins": 2.6199638843536377, + "rewards/rejected": -2.092073917388916, + "step": 780 + }, + { + "epoch": 1.02, + "learning_rate": 4.900035552664605e-05, + "logits/chosen": -2.5695183277130127, + "logits/rejected": -2.6082301139831543, + "logps/chosen": -200.3180389404297, + "logps/rejected": -233.2794189453125, + "loss": 0.1978, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.10721993446350098, + "rewards/margins": 2.7741177082061768, + "rewards/rejected": -2.666897773742676, + "step": 781 + }, + { + "epoch": 1.02, + "learning_rate": 4.899743735344139e-05, + "logits/chosen": -2.1258223056793213, + "logits/rejected": -2.2032220363616943, + "logps/chosen": -201.60614013671875, + "logps/rejected": -282.1663818359375, + "loss": 0.1137, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.04111824184656143, + "rewards/margins": 3.605041027069092, + "rewards/rejected": -3.56392240524292, + "step": 782 + }, + { + "epoch": 1.02, + "learning_rate": 4.899451501421133e-05, + "logits/chosen": -2.280001163482666, + "logits/rejected": -2.454638957977295, + "logps/chosen": -185.27694702148438, + "logps/rejected": -216.79037475585938, + "loss": 0.181, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.409013569355011, + "rewards/margins": 2.352353572845459, + "rewards/rejected": -2.7613673210144043, + "step": 783 + }, + { + "epoch": 1.03, + "learning_rate": 4.899158850946319e-05, + "logits/chosen": -2.4744930267333984, + "logits/rejected": -2.5907466411590576, + "logps/chosen": -216.58287048339844, + "logps/rejected": -261.81353759765625, + "loss": 0.1848, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.11592866480350494, + "rewards/margins": 2.6159138679504395, + "rewards/rejected": -2.4999852180480957, + "step": 784 + }, + { + "epoch": 1.03, + "learning_rate": 4.8988657839705024e-05, + "logits/chosen": -2.6295738220214844, + "logits/rejected": -2.565519332885742, + "logps/chosen": -175.5509033203125, + "logps/rejected": -192.17001342773438, + "loss": 0.2344, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.13090485334396362, + "rewards/margins": 2.3108062744140625, + "rewards/rejected": -2.44171142578125, + "step": 785 + }, + { + "epoch": 1.03, + "learning_rate": 4.89857230054456e-05, + "logits/chosen": -2.4025466442108154, + "logits/rejected": -2.478341817855835, + "logps/chosen": -160.97872924804688, + "logps/rejected": -221.78729248046875, + "loss": 0.2185, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18785348534584045, + "rewards/margins": 2.667365074157715, + "rewards/rejected": -2.8552186489105225, + "step": 786 + }, + { + "epoch": 1.03, + "learning_rate": 4.898278400719442e-05, + "logits/chosen": -2.309408664703369, + "logits/rejected": -2.336895227432251, + "logps/chosen": -194.12857055664062, + "logps/rejected": -229.64393615722656, + "loss": 0.1647, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.5826263427734375, + "rewards/margins": 2.673534870147705, + "rewards/rejected": -3.2561612129211426, + "step": 787 + }, + { + "epoch": 1.03, + "learning_rate": 4.897984084546169e-05, + "logits/chosen": -2.5822911262512207, + "logits/rejected": -2.594147205352783, + "logps/chosen": -189.2248992919922, + "logps/rejected": -217.4370880126953, + "loss": 0.1947, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18159398436546326, + "rewards/margins": 2.9417152404785156, + "rewards/rejected": -3.123309373855591, + "step": 788 + }, + { + "epoch": 1.03, + "learning_rate": 4.897689352075837e-05, + "logits/chosen": -2.395965576171875, + "logits/rejected": -2.3766961097717285, + "logps/chosen": -234.74583435058594, + "logps/rejected": -283.2368469238281, + "loss": 0.2304, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.37718701362609863, + "rewards/margins": 3.130066394805908, + "rewards/rejected": -3.507253408432007, + "step": 789 + }, + { + "epoch": 1.03, + "learning_rate": 4.897394203359611e-05, + "logits/chosen": -2.223119020462036, + "logits/rejected": -2.1850533485412598, + "logps/chosen": -163.25404357910156, + "logps/rejected": -190.954833984375, + "loss": 0.1986, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.48350968956947327, + "rewards/margins": 2.4246490001678467, + "rewards/rejected": -2.908158779144287, + "step": 790 + }, + { + "epoch": 1.04, + "learning_rate": 4.897098638448731e-05, + "logits/chosen": -2.617220878601074, + "logits/rejected": -2.6898252964019775, + "logps/chosen": -191.36790466308594, + "logps/rejected": -270.26287841796875, + "loss": 0.1789, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.7169384360313416, + "rewards/margins": 2.659780979156494, + "rewards/rejected": -3.3767194747924805, + "step": 791 + }, + { + "epoch": 1.04, + "learning_rate": 4.896802657394506e-05, + "logits/chosen": -2.3435750007629395, + "logits/rejected": -2.2688729763031006, + "logps/chosen": -154.95065307617188, + "logps/rejected": -174.447021484375, + "loss": 0.2578, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.7484995722770691, + "rewards/margins": 2.27825927734375, + "rewards/rejected": -3.0267586708068848, + "step": 792 + }, + { + "epoch": 1.04, + "learning_rate": 4.8965062602483205e-05, + "logits/chosen": -2.4353139400482178, + "logits/rejected": -2.381152629852295, + "logps/chosen": -270.47259521484375, + "logps/rejected": -281.2034606933594, + "loss": 0.16, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4860559105873108, + "rewards/margins": 3.1953186988830566, + "rewards/rejected": -3.6813745498657227, + "step": 793 + }, + { + "epoch": 1.04, + "learning_rate": 4.89620944706163e-05, + "logits/chosen": -2.391406536102295, + "logits/rejected": -2.426130533218384, + "logps/chosen": -167.80072021484375, + "logps/rejected": -207.9481964111328, + "loss": 0.1768, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.36457765102386475, + "rewards/margins": 3.225442409515381, + "rewards/rejected": -3.590019941329956, + "step": 794 + }, + { + "epoch": 1.04, + "learning_rate": 4.8959122178859616e-05, + "logits/chosen": -2.5002002716064453, + "logits/rejected": -2.584379196166992, + "logps/chosen": -194.40185546875, + "logps/rejected": -240.38218688964844, + "loss": 0.1416, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.7205836176872253, + "rewards/margins": 3.2785701751708984, + "rewards/rejected": -3.9991536140441895, + "step": 795 + }, + { + "epoch": 1.04, + "learning_rate": 4.8956145727729156e-05, + "logits/chosen": -2.7242343425750732, + "logits/rejected": -2.7420852184295654, + "logps/chosen": -193.07855224609375, + "logps/rejected": -213.01675415039062, + "loss": 0.2132, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.5593734979629517, + "rewards/margins": 2.7595760822296143, + "rewards/rejected": -3.3189496994018555, + "step": 796 + }, + { + "epoch": 1.04, + "learning_rate": 4.895316511774164e-05, + "logits/chosen": -2.5653488636016846, + "logits/rejected": -2.6561145782470703, + "logps/chosen": -187.71011352539062, + "logps/rejected": -234.84353637695312, + "loss": 0.2078, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.685555100440979, + "rewards/margins": 2.5933103561401367, + "rewards/rejected": -3.278865337371826, + "step": 797 + }, + { + "epoch": 1.04, + "learning_rate": 4.895018034941451e-05, + "logits/chosen": -2.6024954319000244, + "logits/rejected": -2.519937753677368, + "logps/chosen": -168.91847229003906, + "logps/rejected": -224.6294708251953, + "loss": 0.1913, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.535056471824646, + "rewards/margins": 3.0812528133392334, + "rewards/rejected": -3.61630916595459, + "step": 798 + }, + { + "epoch": 1.05, + "learning_rate": 4.894719142326593e-05, + "logits/chosen": -2.21370005607605, + "logits/rejected": -2.3001461029052734, + "logps/chosen": -166.51718139648438, + "logps/rejected": -224.1083526611328, + "loss": 0.3661, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.7483687996864319, + "rewards/margins": 2.128541946411133, + "rewards/rejected": -2.8769102096557617, + "step": 799 + }, + { + "epoch": 1.05, + "learning_rate": 4.894419833981478e-05, + "logits/chosen": -2.4892656803131104, + "logits/rejected": -2.568337917327881, + "logps/chosen": -247.6711883544922, + "logps/rejected": -264.4457092285156, + "loss": 0.1815, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.5068062543869019, + "rewards/margins": 3.0614163875579834, + "rewards/rejected": -3.5682225227355957, + "step": 800 + }, + { + "epoch": 1.05, + "learning_rate": 4.894120109958068e-05, + "logits/chosen": -2.5446887016296387, + "logits/rejected": -2.5595788955688477, + "logps/chosen": -148.94400024414062, + "logps/rejected": -197.58807373046875, + "loss": 0.2073, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.3782866895198822, + "rewards/margins": 2.5527689456939697, + "rewards/rejected": -2.931055784225464, + "step": 801 + }, + { + "epoch": 1.05, + "learning_rate": 4.893819970308394e-05, + "logits/chosen": -2.6223721504211426, + "logits/rejected": -2.6008565425872803, + "logps/chosen": -223.73875427246094, + "logps/rejected": -270.4544982910156, + "loss": 0.131, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.582880973815918, + "rewards/margins": 2.6789181232452393, + "rewards/rejected": -3.2617993354797363, + "step": 802 + }, + { + "epoch": 1.05, + "learning_rate": 4.893519415084564e-05, + "logits/chosen": -2.576484203338623, + "logits/rejected": -2.459780216217041, + "logps/chosen": -176.4852294921875, + "logps/rejected": -229.3186492919922, + "loss": 0.1817, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5410618782043457, + "rewards/margins": 2.697970390319824, + "rewards/rejected": -3.239032030105591, + "step": 803 + }, + { + "epoch": 1.05, + "learning_rate": 4.8932184443387536e-05, + "logits/chosen": -2.32121205329895, + "logits/rejected": -2.379164695739746, + "logps/chosen": -186.08949279785156, + "logps/rejected": -203.67755126953125, + "loss": 0.3161, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.43141627311706543, + "rewards/margins": 2.3219566345214844, + "rewards/rejected": -2.7533726692199707, + "step": 804 + }, + { + "epoch": 1.05, + "learning_rate": 4.892917058123212e-05, + "logits/chosen": -2.4327099323272705, + "logits/rejected": -2.4539988040924072, + "logps/chosen": -223.7473602294922, + "logps/rejected": -243.0752410888672, + "loss": 0.2913, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.5450294017791748, + "rewards/margins": 1.9214046001434326, + "rewards/rejected": -2.4664340019226074, + "step": 805 + }, + { + "epoch": 1.05, + "learning_rate": 4.892615256490262e-05, + "logits/chosen": -2.549992084503174, + "logits/rejected": -2.67326283454895, + "logps/chosen": -192.02581787109375, + "logps/rejected": -212.2666778564453, + "loss": 0.2081, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.1347290426492691, + "rewards/margins": 2.5369441509246826, + "rewards/rejected": -2.671673059463501, + "step": 806 + }, + { + "epoch": 1.06, + "learning_rate": 4.8923130394922947e-05, + "logits/chosen": -2.503438949584961, + "logits/rejected": -2.5232760906219482, + "logps/chosen": -202.92604064941406, + "logps/rejected": -230.6805877685547, + "loss": 0.2341, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.3751166760921478, + "rewards/margins": 2.6738672256469727, + "rewards/rejected": -3.0489840507507324, + "step": 807 + }, + { + "epoch": 1.06, + "learning_rate": 4.8920104071817786e-05, + "logits/chosen": -2.3755786418914795, + "logits/rejected": -2.5557618141174316, + "logps/chosen": -137.71795654296875, + "logps/rejected": -216.0382080078125, + "loss": 0.1602, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.13777978718280792, + "rewards/margins": 3.1972692012786865, + "rewards/rejected": -3.3350489139556885, + "step": 808 + }, + { + "epoch": 1.06, + "learning_rate": 4.891707359611251e-05, + "logits/chosen": -2.4458518028259277, + "logits/rejected": -2.485722303390503, + "logps/chosen": -232.0552520751953, + "logps/rejected": -290.8924255371094, + "loss": 0.112, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5124866962432861, + "rewards/margins": 3.1787357330322266, + "rewards/rejected": -3.6912224292755127, + "step": 809 + }, + { + "epoch": 1.06, + "learning_rate": 4.89140389683332e-05, + "logits/chosen": -2.6487104892730713, + "logits/rejected": -2.691654920578003, + "logps/chosen": -200.54293823242188, + "logps/rejected": -267.951416015625, + "loss": 0.0792, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.20664259791374207, + "rewards/margins": 3.448357582092285, + "rewards/rejected": -3.6550002098083496, + "step": 810 + }, + { + "epoch": 1.06, + "learning_rate": 4.8911000189006694e-05, + "logits/chosen": -2.47886323928833, + "logits/rejected": -2.5354814529418945, + "logps/chosen": -155.83856201171875, + "logps/rejected": -193.6571502685547, + "loss": 0.1333, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1021997407078743, + "rewards/margins": 3.2138075828552246, + "rewards/rejected": -3.111607789993286, + "step": 811 + }, + { + "epoch": 1.06, + "learning_rate": 4.8907957258660534e-05, + "logits/chosen": -2.288719654083252, + "logits/rejected": -2.2621853351593018, + "logps/chosen": -219.19277954101562, + "logps/rejected": -233.79931640625, + "loss": 0.291, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.2972843647003174, + "rewards/margins": 2.0650277137756348, + "rewards/rejected": -2.3623123168945312, + "step": 812 + }, + { + "epoch": 1.06, + "learning_rate": 4.890491017782296e-05, + "logits/chosen": -2.5341038703918457, + "logits/rejected": -2.636235475540161, + "logps/chosen": -166.26109313964844, + "logps/rejected": -217.66732788085938, + "loss": 0.2438, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.7079929709434509, + "rewards/margins": 2.6824710369110107, + "rewards/rejected": -3.3904640674591064, + "step": 813 + }, + { + "epoch": 1.07, + "learning_rate": 4.890185894702298e-05, + "logits/chosen": -2.5911717414855957, + "logits/rejected": -2.6073203086853027, + "logps/chosen": -208.44723510742188, + "logps/rejected": -287.8471374511719, + "loss": 0.0952, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.16351568698883057, + "rewards/margins": 3.8280396461486816, + "rewards/rejected": -3.6645243167877197, + "step": 814 + }, + { + "epoch": 1.07, + "learning_rate": 4.8898803566790296e-05, + "logits/chosen": -2.3474974632263184, + "logits/rejected": -2.4600653648376465, + "logps/chosen": -225.6314697265625, + "logps/rejected": -247.96067810058594, + "loss": 0.1813, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11835913360118866, + "rewards/margins": 3.3629634380340576, + "rewards/rejected": -3.481323003768921, + "step": 815 + }, + { + "epoch": 1.07, + "learning_rate": 4.889574403765531e-05, + "logits/chosen": -2.314152717590332, + "logits/rejected": -2.2856571674346924, + "logps/chosen": -222.77545166015625, + "logps/rejected": -240.05615234375, + "loss": 0.2903, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.5903599858283997, + "rewards/margins": 2.8621745109558105, + "rewards/rejected": -3.4525346755981445, + "step": 816 + }, + { + "epoch": 1.07, + "learning_rate": 4.889268036014918e-05, + "logits/chosen": -2.2042934894561768, + "logits/rejected": -2.252229928970337, + "logps/chosen": -168.29946899414062, + "logps/rejected": -220.2432403564453, + "loss": 0.181, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.6510707139968872, + "rewards/margins": 2.5488698482513428, + "rewards/rejected": -3.1999404430389404, + "step": 817 + }, + { + "epoch": 1.07, + "learning_rate": 4.888961253480376e-05, + "logits/chosen": -2.330993890762329, + "logits/rejected": -2.439598321914673, + "logps/chosen": -191.20274353027344, + "logps/rejected": -240.98287963867188, + "loss": 0.0927, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.08680248260498047, + "rewards/margins": 3.754706382751465, + "rewards/rejected": -3.8415091037750244, + "step": 818 + }, + { + "epoch": 1.07, + "learning_rate": 4.8886540562151653e-05, + "logits/chosen": -2.60939884185791, + "logits/rejected": -2.64247989654541, + "logps/chosen": -186.75680541992188, + "logps/rejected": -217.1247100830078, + "loss": 0.1396, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13618728518486023, + "rewards/margins": 2.6276023387908936, + "rewards/rejected": -2.7637898921966553, + "step": 819 + }, + { + "epoch": 1.07, + "learning_rate": 4.8883464442726146e-05, + "logits/chosen": -2.451847791671753, + "logits/rejected": -2.550415277481079, + "logps/chosen": -194.68426513671875, + "logps/rejected": -221.12335205078125, + "loss": 0.2368, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.38820838928222656, + "rewards/margins": 2.107210159301758, + "rewards/rejected": -2.4954187870025635, + "step": 820 + }, + { + "epoch": 1.07, + "learning_rate": 4.888038417706126e-05, + "logits/chosen": -2.4475693702697754, + "logits/rejected": -2.4619081020355225, + "logps/chosen": -185.6713409423828, + "logps/rejected": -246.03526306152344, + "loss": 0.1785, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.18588271737098694, + "rewards/margins": 3.4940121173858643, + "rewards/rejected": -3.6798949241638184, + "step": 821 + }, + { + "epoch": 1.08, + "learning_rate": 4.887729976569174e-05, + "logits/chosen": -2.6383659839630127, + "logits/rejected": -2.641087293624878, + "logps/chosen": -220.7265625, + "logps/rejected": -246.0904083251953, + "loss": 0.1677, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.5302943587303162, + "rewards/margins": 3.091730833053589, + "rewards/rejected": -3.6220250129699707, + "step": 822 + }, + { + "epoch": 1.08, + "learning_rate": 4.8874211209153066e-05, + "logits/chosen": -2.7357144355773926, + "logits/rejected": -2.756209373474121, + "logps/chosen": -228.9771270751953, + "logps/rejected": -244.65692138671875, + "loss": 0.1871, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.2874932289123535, + "rewards/margins": 3.040170669555664, + "rewards/rejected": -3.3276634216308594, + "step": 823 + }, + { + "epoch": 1.08, + "learning_rate": 4.8871118507981396e-05, + "logits/chosen": -2.541903257369995, + "logits/rejected": -2.642256259918213, + "logps/chosen": -172.7350311279297, + "logps/rejected": -263.6834716796875, + "loss": 0.1574, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.31187328696250916, + "rewards/margins": 3.909829616546631, + "rewards/rejected": -4.221703052520752, + "step": 824 + }, + { + "epoch": 1.08, + "learning_rate": 4.886802166271364e-05, + "logits/chosen": -2.5496389865875244, + "logits/rejected": -2.632146120071411, + "logps/chosen": -189.05809020996094, + "logps/rejected": -210.13885498046875, + "loss": 0.15, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08079969137907028, + "rewards/margins": 3.155087947845459, + "rewards/rejected": -3.2358880043029785, + "step": 825 + }, + { + "epoch": 1.08, + "learning_rate": 4.8864920673887434e-05, + "logits/chosen": -2.508222818374634, + "logits/rejected": -2.495868682861328, + "logps/chosen": -178.13548278808594, + "logps/rejected": -204.01275634765625, + "loss": 0.1071, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.022821810096502304, + "rewards/margins": 3.6802027225494385, + "rewards/rejected": -3.703024387359619, + "step": 826 + }, + { + "epoch": 1.08, + "learning_rate": 4.88618155420411e-05, + "logits/chosen": -2.3088462352752686, + "logits/rejected": -2.442979574203491, + "logps/chosen": -207.34799194335938, + "logps/rejected": -256.6688232421875, + "loss": 0.1586, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.63365638256073, + "rewards/margins": 2.7970571517944336, + "rewards/rejected": -3.430713415145874, + "step": 827 + }, + { + "epoch": 1.08, + "learning_rate": 4.8858706267713704e-05, + "logits/chosen": -2.532752275466919, + "logits/rejected": -2.47641658782959, + "logps/chosen": -181.96983337402344, + "logps/rejected": -179.16371154785156, + "loss": 0.2413, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.7146714329719543, + "rewards/margins": 2.151493787765503, + "rewards/rejected": -2.8661651611328125, + "step": 828 + }, + { + "epoch": 1.08, + "learning_rate": 4.885559285144503e-05, + "logits/chosen": -2.4036006927490234, + "logits/rejected": -2.451720952987671, + "logps/chosen": -179.5876922607422, + "logps/rejected": -236.59486389160156, + "loss": 0.1633, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.3670279383659363, + "rewards/margins": 3.0567290782928467, + "rewards/rejected": -3.4237565994262695, + "step": 829 + }, + { + "epoch": 1.09, + "learning_rate": 4.885247529377557e-05, + "logits/chosen": -2.4562742710113525, + "logits/rejected": -2.516852855682373, + "logps/chosen": -139.80506896972656, + "logps/rejected": -169.3076171875, + "loss": 0.2494, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.586427628993988, + "rewards/margins": 1.9348812103271484, + "rewards/rejected": -2.521308660507202, + "step": 830 + }, + { + "epoch": 1.09, + "learning_rate": 4.884935359524655e-05, + "logits/chosen": -2.433465003967285, + "logits/rejected": -2.401857852935791, + "logps/chosen": -181.75381469726562, + "logps/rejected": -226.99884033203125, + "loss": 0.1503, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.24370934069156647, + "rewards/margins": 3.303581953048706, + "rewards/rejected": -3.5472910404205322, + "step": 831 + }, + { + "epoch": 1.09, + "learning_rate": 4.884622775639989e-05, + "logits/chosen": -2.4640562534332275, + "logits/rejected": -2.4529128074645996, + "logps/chosen": -228.76239013671875, + "logps/rejected": -277.97503662109375, + "loss": 0.1871, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.40687793493270874, + "rewards/margins": 2.9247701168060303, + "rewards/rejected": -3.331648111343384, + "step": 832 + }, + { + "epoch": 1.09, + "learning_rate": 4.884309777777826e-05, + "logits/chosen": -2.600520372390747, + "logits/rejected": -2.6682722568511963, + "logps/chosen": -239.098388671875, + "logps/rejected": -298.6829528808594, + "loss": 0.0985, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.40839752554893494, + "rewards/margins": 3.9475584030151367, + "rewards/rejected": -4.355956077575684, + "step": 833 + }, + { + "epoch": 1.09, + "learning_rate": 4.883996365992502e-05, + "logits/chosen": -2.470278024673462, + "logits/rejected": -2.446775197982788, + "logps/chosen": -225.15782165527344, + "logps/rejected": -273.9099426269531, + "loss": 0.1943, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.9289353489875793, + "rewards/margins": 3.2997019290924072, + "rewards/rejected": -4.228637218475342, + "step": 834 + }, + { + "epoch": 1.09, + "learning_rate": 4.883682540338428e-05, + "logits/chosen": -2.265939474105835, + "logits/rejected": -2.3391735553741455, + "logps/chosen": -203.70504760742188, + "logps/rejected": -275.56427001953125, + "loss": 0.1327, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.4642513394355774, + "rewards/margins": 4.042669296264648, + "rewards/rejected": -4.506920337677002, + "step": 835 + }, + { + "epoch": 1.09, + "learning_rate": 4.883368300870083e-05, + "logits/chosen": -2.431640148162842, + "logits/rejected": -2.453253984451294, + "logps/chosen": -199.0352020263672, + "logps/rejected": -255.3938751220703, + "loss": 0.1519, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.729556679725647, + "rewards/margins": 3.61323881149292, + "rewards/rejected": -4.3427958488464355, + "step": 836 + }, + { + "epoch": 1.1, + "learning_rate": 4.883053647642022e-05, + "logits/chosen": -2.531667947769165, + "logits/rejected": -2.473618507385254, + "logps/chosen": -258.14215087890625, + "logps/rejected": -253.9119110107422, + "loss": 0.05, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.14007891714572906, + "rewards/margins": 4.125094890594482, + "rewards/rejected": -4.26517391204834, + "step": 837 + }, + { + "epoch": 1.1, + "learning_rate": 4.8827385807088674e-05, + "logits/chosen": -2.368967056274414, + "logits/rejected": -2.3205454349517822, + "logps/chosen": -237.3386688232422, + "logps/rejected": -306.61737060546875, + "loss": 0.1212, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8273277282714844, + "rewards/margins": 3.641183614730835, + "rewards/rejected": -4.468511581420898, + "step": 838 + }, + { + "epoch": 1.1, + "learning_rate": 4.8824231001253186e-05, + "logits/chosen": -2.4554178714752197, + "logits/rejected": -2.457733631134033, + "logps/chosen": -219.16522216796875, + "logps/rejected": -298.85003662109375, + "loss": 0.2137, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.3690099716186523, + "rewards/margins": 2.852839231491089, + "rewards/rejected": -4.221848964691162, + "step": 839 + }, + { + "epoch": 1.1, + "learning_rate": 4.882107205946142e-05, + "logits/chosen": -2.19034743309021, + "logits/rejected": -2.1936280727386475, + "logps/chosen": -196.35073852539062, + "logps/rejected": -266.54638671875, + "loss": 0.2458, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.5001093149185181, + "rewards/margins": 3.14082670211792, + "rewards/rejected": -3.6409358978271484, + "step": 840 + }, + { + "epoch": 1.1, + "learning_rate": 4.8817908982261765e-05, + "logits/chosen": -2.3098065853118896, + "logits/rejected": -2.4067697525024414, + "logps/chosen": -197.4673614501953, + "logps/rejected": -242.01669311523438, + "loss": 0.1775, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7165354490280151, + "rewards/margins": 3.300830841064453, + "rewards/rejected": -4.017366409301758, + "step": 841 + }, + { + "epoch": 1.1, + "learning_rate": 4.881474177020336e-05, + "logits/chosen": -2.541229009628296, + "logits/rejected": -2.5617356300354004, + "logps/chosen": -219.83966064453125, + "logps/rejected": -247.85601806640625, + "loss": 0.0592, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8507525324821472, + "rewards/margins": 3.5395710468292236, + "rewards/rejected": -4.390323638916016, + "step": 842 + }, + { + "epoch": 1.1, + "learning_rate": 4.881157042383605e-05, + "logits/chosen": -2.2904212474823, + "logits/rejected": -2.3347108364105225, + "logps/chosen": -231.440673828125, + "logps/rejected": -303.37939453125, + "loss": 0.2497, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1560659408569336, + "rewards/margins": 2.958097457885742, + "rewards/rejected": -4.114163398742676, + "step": 843 + }, + { + "epoch": 1.1, + "learning_rate": 4.8808394943710376e-05, + "logits/chosen": -2.279738187789917, + "logits/rejected": -2.3323707580566406, + "logps/chosen": -213.64846801757812, + "logps/rejected": -239.2349395751953, + "loss": 0.243, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1719560623168945, + "rewards/margins": 2.3868768215179443, + "rewards/rejected": -3.558833122253418, + "step": 844 + }, + { + "epoch": 1.11, + "learning_rate": 4.880521533037762e-05, + "logits/chosen": -2.3850831985473633, + "logits/rejected": -2.4819254875183105, + "logps/chosen": -211.81105041503906, + "logps/rejected": -253.8992919921875, + "loss": 0.1005, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6735836267471313, + "rewards/margins": 3.870424509048462, + "rewards/rejected": -4.544007778167725, + "step": 845 + }, + { + "epoch": 1.11, + "learning_rate": 4.880203158438975e-05, + "logits/chosen": -2.4267005920410156, + "logits/rejected": -2.4583656787872314, + "logps/chosen": -185.1043243408203, + "logps/rejected": -223.85244750976562, + "loss": 0.1452, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.911338746547699, + "rewards/margins": 3.077333688735962, + "rewards/rejected": -3.9886722564697266, + "step": 846 + }, + { + "epoch": 1.11, + "learning_rate": 4.87988437062995e-05, + "logits/chosen": -2.0992445945739746, + "logits/rejected": -2.171211004257202, + "logps/chosen": -159.3900909423828, + "logps/rejected": -242.703369140625, + "loss": 0.25, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.7375649213790894, + "rewards/margins": 3.8496501445770264, + "rewards/rejected": -4.587214946746826, + "step": 847 + }, + { + "epoch": 1.11, + "learning_rate": 4.879565169666028e-05, + "logits/chosen": -2.2952377796173096, + "logits/rejected": -2.3588685989379883, + "logps/chosen": -201.49908447265625, + "logps/rejected": -241.18902587890625, + "loss": 0.2027, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.5940706729888916, + "rewards/margins": 3.3175528049468994, + "rewards/rejected": -3.91162371635437, + "step": 848 + }, + { + "epoch": 1.11, + "learning_rate": 4.879245555602624e-05, + "logits/chosen": -1.9072455167770386, + "logits/rejected": -2.0710573196411133, + "logps/chosen": -167.665283203125, + "logps/rejected": -231.74786376953125, + "loss": 0.2365, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0657278299331665, + "rewards/margins": 3.313453435897827, + "rewards/rejected": -4.379181385040283, + "step": 849 + }, + { + "epoch": 1.11, + "learning_rate": 4.878925528495223e-05, + "logits/chosen": -2.076167106628418, + "logits/rejected": -2.0585241317749023, + "logps/chosen": -160.9062957763672, + "logps/rejected": -198.1348419189453, + "loss": 0.2318, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.9388778209686279, + "rewards/margins": 2.9717299938201904, + "rewards/rejected": -3.9106080532073975, + "step": 850 + }, + { + "epoch": 1.11, + "learning_rate": 4.878605088399384e-05, + "logits/chosen": -2.1791696548461914, + "logits/rejected": -2.2479426860809326, + "logps/chosen": -191.9034881591797, + "logps/rejected": -257.7882995605469, + "loss": 0.0744, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5156456828117371, + "rewards/margins": 3.969588041305542, + "rewards/rejected": -4.485233783721924, + "step": 851 + }, + { + "epoch": 1.12, + "learning_rate": 4.878284235370735e-05, + "logits/chosen": -2.1952590942382812, + "logits/rejected": -2.3678793907165527, + "logps/chosen": -195.3539276123047, + "logps/rejected": -240.06231689453125, + "loss": 0.1083, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.9977157115936279, + "rewards/margins": 3.5119776725769043, + "rewards/rejected": -4.509693622589111, + "step": 852 + }, + { + "epoch": 1.12, + "learning_rate": 4.877962969464978e-05, + "logits/chosen": -2.0996241569519043, + "logits/rejected": -2.2190651893615723, + "logps/chosen": -184.02914428710938, + "logps/rejected": -213.2720489501953, + "loss": 0.2316, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.107424020767212, + "rewards/margins": 2.9243013858795166, + "rewards/rejected": -4.0317254066467285, + "step": 853 + }, + { + "epoch": 1.12, + "learning_rate": 4.877641290737884e-05, + "logits/chosen": -2.398710012435913, + "logits/rejected": -2.455815315246582, + "logps/chosen": -196.89927673339844, + "logps/rejected": -228.26943969726562, + "loss": 0.2229, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.7392411828041077, + "rewards/margins": 3.144235134124756, + "rewards/rejected": -3.8834760189056396, + "step": 854 + }, + { + "epoch": 1.12, + "learning_rate": 4.8773191992453e-05, + "logits/chosen": -2.585550546646118, + "logits/rejected": -2.6128385066986084, + "logps/chosen": -221.81942749023438, + "logps/rejected": -235.17910766601562, + "loss": 0.1117, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5257879495620728, + "rewards/margins": 3.225809335708618, + "rewards/rejected": -3.7515969276428223, + "step": 855 + }, + { + "epoch": 1.12, + "learning_rate": 4.87699669504314e-05, + "logits/chosen": -2.3643717765808105, + "logits/rejected": -2.397857427597046, + "logps/chosen": -220.61892700195312, + "logps/rejected": -270.6251220703125, + "loss": 0.1944, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2166273593902588, + "rewards/margins": 3.119513511657715, + "rewards/rejected": -4.3361406326293945, + "step": 856 + }, + { + "epoch": 1.12, + "learning_rate": 4.876673778187393e-05, + "logits/chosen": -2.182027578353882, + "logits/rejected": -2.232729434967041, + "logps/chosen": -172.77215576171875, + "logps/rejected": -192.00790405273438, + "loss": 0.2663, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.9920349717140198, + "rewards/margins": 3.2848072052001953, + "rewards/rejected": -4.2768425941467285, + "step": 857 + }, + { + "epoch": 1.12, + "learning_rate": 4.876350448734117e-05, + "logits/chosen": -2.317965507507324, + "logits/rejected": -2.4213790893554688, + "logps/chosen": -170.614990234375, + "logps/rejected": -179.1256866455078, + "loss": 0.3067, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.353677749633789, + "rewards/margins": 1.795522928237915, + "rewards/rejected": -3.149200916290283, + "step": 858 + }, + { + "epoch": 1.12, + "learning_rate": 4.876026706739444e-05, + "logits/chosen": -2.362269401550293, + "logits/rejected": -2.385591506958008, + "logps/chosen": -277.1317138671875, + "logps/rejected": -302.71575927734375, + "loss": 0.2337, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.7664923667907715, + "rewards/margins": 2.5697085857391357, + "rewards/rejected": -4.336201190948486, + "step": 859 + }, + { + "epoch": 1.13, + "learning_rate": 4.8757025522595756e-05, + "logits/chosen": -2.5295486450195312, + "logits/rejected": -2.5323615074157715, + "logps/chosen": -205.54013061523438, + "logps/rejected": -218.6345977783203, + "loss": 0.1368, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.2794644236564636, + "rewards/margins": 2.9989418983459473, + "rewards/rejected": -3.2784063816070557, + "step": 860 + }, + { + "epoch": 1.13, + "learning_rate": 4.8753779853507874e-05, + "logits/chosen": -2.117382764816284, + "logits/rejected": -2.120049238204956, + "logps/chosen": -196.75279235839844, + "logps/rejected": -217.56846618652344, + "loss": 0.1823, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.5300887227058411, + "rewards/margins": 3.763706922531128, + "rewards/rejected": -4.293795585632324, + "step": 861 + }, + { + "epoch": 1.13, + "learning_rate": 4.8750530060694236e-05, + "logits/chosen": -1.7907865047454834, + "logits/rejected": -1.8680351972579956, + "logps/chosen": -134.9779510498047, + "logps/rejected": -194.37901306152344, + "loss": 0.1639, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8513069152832031, + "rewards/margins": 2.7600948810577393, + "rewards/rejected": -3.6114020347595215, + "step": 862 + }, + { + "epoch": 1.13, + "learning_rate": 4.874727614471903e-05, + "logits/chosen": -2.110868453979492, + "logits/rejected": -2.1362719535827637, + "logps/chosen": -194.04483032226562, + "logps/rejected": -234.01123046875, + "loss": 0.2074, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.6307097673416138, + "rewards/margins": 3.406496047973633, + "rewards/rejected": -5.037205696105957, + "step": 863 + }, + { + "epoch": 1.13, + "learning_rate": 4.8744018106147135e-05, + "logits/chosen": -2.32252836227417, + "logits/rejected": -2.344344139099121, + "logps/chosen": -224.74435424804688, + "logps/rejected": -252.48287963867188, + "loss": 0.1992, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.4851446151733398, + "rewards/margins": 3.0550522804260254, + "rewards/rejected": -4.540196418762207, + "step": 864 + }, + { + "epoch": 1.13, + "learning_rate": 4.8740755945544156e-05, + "logits/chosen": -2.2035303115844727, + "logits/rejected": -2.198829174041748, + "logps/chosen": -147.6013641357422, + "logps/rejected": -178.75584411621094, + "loss": 0.218, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.2339822053909302, + "rewards/margins": 2.6723711490631104, + "rewards/rejected": -3.90635347366333, + "step": 865 + }, + { + "epoch": 1.13, + "learning_rate": 4.873748966347642e-05, + "logits/chosen": -2.1133739948272705, + "logits/rejected": -2.088212251663208, + "logps/chosen": -183.02053833007812, + "logps/rejected": -214.74398803710938, + "loss": 0.1746, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.064955234527588, + "rewards/margins": 3.2688629627227783, + "rewards/rejected": -4.333818435668945, + "step": 866 + }, + { + "epoch": 1.13, + "learning_rate": 4.873421926051095e-05, + "logits/chosen": -2.3783798217773438, + "logits/rejected": -2.4469428062438965, + "logps/chosen": -175.8081817626953, + "logps/rejected": -228.7090301513672, + "loss": 0.2164, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.9884763956069946, + "rewards/margins": 3.5400943756103516, + "rewards/rejected": -4.528570652008057, + "step": 867 + }, + { + "epoch": 1.14, + "learning_rate": 4.873094473721552e-05, + "logits/chosen": -2.122225284576416, + "logits/rejected": -2.229064464569092, + "logps/chosen": -163.4876708984375, + "logps/rejected": -226.0997314453125, + "loss": 0.1045, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.584133267402649, + "rewards/margins": 3.3204777240753174, + "rewards/rejected": -4.904611110687256, + "step": 868 + }, + { + "epoch": 1.14, + "learning_rate": 4.872766609415857e-05, + "logits/chosen": -2.342576265335083, + "logits/rejected": -2.389909029006958, + "logps/chosen": -161.72445678710938, + "logps/rejected": -186.76344299316406, + "loss": 0.2182, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.8539198040962219, + "rewards/margins": 2.9423582553863525, + "rewards/rejected": -3.7962779998779297, + "step": 869 + }, + { + "epoch": 1.14, + "learning_rate": 4.872438333190931e-05, + "logits/chosen": -2.137856960296631, + "logits/rejected": -2.179835081100464, + "logps/chosen": -191.9931182861328, + "logps/rejected": -224.08697509765625, + "loss": 0.1825, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.108015775680542, + "rewards/margins": 3.190267562866211, + "rewards/rejected": -4.298283100128174, + "step": 870 + }, + { + "epoch": 1.14, + "learning_rate": 4.872109645103762e-05, + "logits/chosen": -2.0209460258483887, + "logits/rejected": -2.05436372756958, + "logps/chosen": -242.2396697998047, + "logps/rejected": -256.2543640136719, + "loss": 0.233, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.9021006226539612, + "rewards/margins": 2.7395806312561035, + "rewards/rejected": -3.64168119430542, + "step": 871 + }, + { + "epoch": 1.14, + "learning_rate": 4.871780545211411e-05, + "logits/chosen": -2.396819829940796, + "logits/rejected": -2.3696892261505127, + "logps/chosen": -262.4898376464844, + "logps/rejected": -300.5354919433594, + "loss": 0.3388, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.824455976486206, + "rewards/margins": 3.6746246814727783, + "rewards/rejected": -4.499080657958984, + "step": 872 + }, + { + "epoch": 1.14, + "learning_rate": 4.8714510335710114e-05, + "logits/chosen": -2.2704551219940186, + "logits/rejected": -2.289527416229248, + "logps/chosen": -190.1211395263672, + "logps/rejected": -210.47433471679688, + "loss": 0.2625, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.8054894208908081, + "rewards/margins": 3.2580995559692383, + "rewards/rejected": -4.063589096069336, + "step": 873 + }, + { + "epoch": 1.14, + "learning_rate": 4.871121110239768e-05, + "logits/chosen": -2.0636115074157715, + "logits/rejected": -2.149376392364502, + "logps/chosen": -178.03355407714844, + "logps/rejected": -191.28359985351562, + "loss": 0.1928, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.7658480405807495, + "rewards/margins": 3.3343939781188965, + "rewards/rejected": -4.100241661071777, + "step": 874 + }, + { + "epoch": 1.15, + "learning_rate": 4.870790775274955e-05, + "logits/chosen": -2.3600223064422607, + "logits/rejected": -2.313652276992798, + "logps/chosen": -202.82212829589844, + "logps/rejected": -241.97523498535156, + "loss": 0.1404, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3490352630615234, + "rewards/margins": 3.479907512664795, + "rewards/rejected": -4.828942775726318, + "step": 875 + }, + { + "epoch": 1.15, + "learning_rate": 4.8704600287339205e-05, + "logits/chosen": -2.272904396057129, + "logits/rejected": -2.3197202682495117, + "logps/chosen": -195.68267822265625, + "logps/rejected": -230.81982421875, + "loss": 0.1807, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.507330060005188, + "rewards/margins": 3.294886827468872, + "rewards/rejected": -4.802216529846191, + "step": 876 + }, + { + "epoch": 1.15, + "learning_rate": 4.870128870674082e-05, + "logits/chosen": -2.062432289123535, + "logits/rejected": -2.095883846282959, + "logps/chosen": -177.28150939941406, + "logps/rejected": -238.36048889160156, + "loss": 0.0998, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.5119123458862305, + "rewards/margins": 3.5114619731903076, + "rewards/rejected": -5.023374080657959, + "step": 877 + }, + { + "epoch": 1.15, + "learning_rate": 4.869797301152931e-05, + "logits/chosen": -2.322573184967041, + "logits/rejected": -2.352694511413574, + "logps/chosen": -171.1867218017578, + "logps/rejected": -209.19857788085938, + "loss": 0.1745, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4840812683105469, + "rewards/margins": 3.0589983463287354, + "rewards/rejected": -4.543079376220703, + "step": 878 + }, + { + "epoch": 1.15, + "learning_rate": 4.869465320228028e-05, + "logits/chosen": -1.5731091499328613, + "logits/rejected": -1.5635374784469604, + "logps/chosen": -162.3439483642578, + "logps/rejected": -218.9515380859375, + "loss": 0.1648, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3453707695007324, + "rewards/margins": 3.3265371322631836, + "rewards/rejected": -4.671907901763916, + "step": 879 + }, + { + "epoch": 1.15, + "learning_rate": 4.869132927957007e-05, + "logits/chosen": -2.057164192199707, + "logits/rejected": -2.169400930404663, + "logps/chosen": -146.08116149902344, + "logps/rejected": -217.79507446289062, + "loss": 0.1879, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3128044605255127, + "rewards/margins": 4.181766033172607, + "rewards/rejected": -5.494570732116699, + "step": 880 + }, + { + "epoch": 1.15, + "learning_rate": 4.86880012439757e-05, + "logits/chosen": -2.09201979637146, + "logits/rejected": -2.123706817626953, + "logps/chosen": -205.8859100341797, + "logps/rejected": -238.71531677246094, + "loss": 0.2388, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.5409486293792725, + "rewards/margins": 2.8045778274536133, + "rewards/rejected": -4.345526695251465, + "step": 881 + }, + { + "epoch": 1.15, + "learning_rate": 4.868466909607494e-05, + "logits/chosen": -1.9597307443618774, + "logits/rejected": -2.0664074420928955, + "logps/chosen": -208.81736755371094, + "logps/rejected": -236.41000366210938, + "loss": 0.0614, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3827075958251953, + "rewards/margins": 3.953075647354126, + "rewards/rejected": -5.3357834815979, + "step": 882 + }, + { + "epoch": 1.16, + "learning_rate": 4.868133283644627e-05, + "logits/chosen": -2.1879220008850098, + "logits/rejected": -2.2641048431396484, + "logps/chosen": -195.26947021484375, + "logps/rejected": -233.18475341796875, + "loss": 0.1049, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.485203742980957, + "rewards/margins": 3.5149152278900146, + "rewards/rejected": -5.000119209289551, + "step": 883 + }, + { + "epoch": 1.16, + "learning_rate": 4.867799246566887e-05, + "logits/chosen": -1.873902440071106, + "logits/rejected": -1.9271022081375122, + "logps/chosen": -218.30856323242188, + "logps/rejected": -250.31251525878906, + "loss": 0.1605, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.570319652557373, + "rewards/margins": 4.087669372558594, + "rewards/rejected": -5.657989025115967, + "step": 884 + }, + { + "epoch": 1.16, + "learning_rate": 4.867464798432262e-05, + "logits/chosen": -2.293184995651245, + "logits/rejected": -2.348799705505371, + "logps/chosen": -204.55638122558594, + "logps/rejected": -239.99803161621094, + "loss": 0.1635, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.4229328632354736, + "rewards/margins": 4.076297283172607, + "rewards/rejected": -5.499229907989502, + "step": 885 + }, + { + "epoch": 1.16, + "learning_rate": 4.8671299392988146e-05, + "logits/chosen": -1.9167914390563965, + "logits/rejected": -1.835463285446167, + "logps/chosen": -228.05908203125, + "logps/rejected": -258.6400146484375, + "loss": 0.2577, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.6862237453460693, + "rewards/margins": 2.3948185443878174, + "rewards/rejected": -4.081042289733887, + "step": 886 + }, + { + "epoch": 1.16, + "learning_rate": 4.866794669224678e-05, + "logits/chosen": -2.263591766357422, + "logits/rejected": -2.2648370265960693, + "logps/chosen": -323.7992248535156, + "logps/rejected": -340.5594177246094, + "loss": 0.0843, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0585577487945557, + "rewards/margins": 3.989744186401367, + "rewards/rejected": -5.048301696777344, + "step": 887 + }, + { + "epoch": 1.16, + "learning_rate": 4.8664589882680544e-05, + "logits/chosen": -2.3918349742889404, + "logits/rejected": -2.4805490970611572, + "logps/chosen": -153.11036682128906, + "logps/rejected": -231.31317138671875, + "loss": 0.2822, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.8393661975860596, + "rewards/margins": 3.1571884155273438, + "rewards/rejected": -3.9965548515319824, + "step": 888 + }, + { + "epoch": 1.16, + "learning_rate": 4.866122896487221e-05, + "logits/chosen": -2.1840920448303223, + "logits/rejected": -2.304276466369629, + "logps/chosen": -167.58294677734375, + "logps/rejected": -200.28634643554688, + "loss": 0.1857, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.8503859043121338, + "rewards/margins": 3.1673898696899414, + "rewards/rejected": -4.017776012420654, + "step": 889 + }, + { + "epoch": 1.16, + "learning_rate": 4.865786393940522e-05, + "logits/chosen": -2.069397211074829, + "logits/rejected": -2.1860339641571045, + "logps/chosen": -179.6785125732422, + "logps/rejected": -228.13262939453125, + "loss": 0.0932, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1931277513504028, + "rewards/margins": 3.2066426277160645, + "rewards/rejected": -4.399770736694336, + "step": 890 + }, + { + "epoch": 1.17, + "learning_rate": 4.8654494806863775e-05, + "logits/chosen": -2.1783037185668945, + "logits/rejected": -2.2106220722198486, + "logps/chosen": -193.16891479492188, + "logps/rejected": -243.27835083007812, + "loss": 0.2, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0907084941864014, + "rewards/margins": 3.496738910675049, + "rewards/rejected": -4.587447643280029, + "step": 891 + }, + { + "epoch": 1.17, + "learning_rate": 4.865112156783275e-05, + "logits/chosen": -2.294567584991455, + "logits/rejected": -2.1514720916748047, + "logps/chosen": -214.3270263671875, + "logps/rejected": -187.45822143554688, + "loss": 0.214, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0721594095230103, + "rewards/margins": 2.2023167610168457, + "rewards/rejected": -3.2744765281677246, + "step": 892 + }, + { + "epoch": 1.17, + "learning_rate": 4.864774422289776e-05, + "logits/chosen": -2.0853824615478516, + "logits/rejected": -2.1526715755462646, + "logps/chosen": -204.90089416503906, + "logps/rejected": -246.01055908203125, + "loss": 0.112, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.11933372169733047, + "rewards/margins": 4.104992866516113, + "rewards/rejected": -4.2243266105651855, + "step": 893 + }, + { + "epoch": 1.17, + "learning_rate": 4.864436277264511e-05, + "logits/chosen": -2.1137301921844482, + "logits/rejected": -2.1245357990264893, + "logps/chosen": -159.5576171875, + "logps/rejected": -167.99693298339844, + "loss": 0.3612, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1939194202423096, + "rewards/margins": 2.4540228843688965, + "rewards/rejected": -3.647942543029785, + "step": 894 + }, + { + "epoch": 1.17, + "learning_rate": 4.864097721766184e-05, + "logits/chosen": -2.0749716758728027, + "logits/rejected": -2.199066162109375, + "logps/chosen": -196.0050048828125, + "logps/rejected": -234.677001953125, + "loss": 0.1027, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.997223973274231, + "rewards/margins": 3.582221269607544, + "rewards/rejected": -4.5794453620910645, + "step": 895 + }, + { + "epoch": 1.17, + "learning_rate": 4.8637587558535695e-05, + "logits/chosen": -2.2920432090759277, + "logits/rejected": -2.3417181968688965, + "logps/chosen": -236.25619506835938, + "logps/rejected": -282.4396057128906, + "loss": 0.2044, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.38354048132896423, + "rewards/margins": 2.915595531463623, + "rewards/rejected": -3.29913592338562, + "step": 896 + }, + { + "epoch": 1.17, + "learning_rate": 4.863419379585512e-05, + "logits/chosen": -2.2515666484832764, + "logits/rejected": -2.2877609729766846, + "logps/chosen": -189.43197631835938, + "logps/rejected": -230.85414123535156, + "loss": 0.1691, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.3132818937301636, + "rewards/margins": 3.6810238361358643, + "rewards/rejected": -3.9943056106567383, + "step": 897 + }, + { + "epoch": 1.18, + "learning_rate": 4.863079593020928e-05, + "logits/chosen": -2.3358492851257324, + "logits/rejected": -2.2418711185455322, + "logps/chosen": -200.99757385253906, + "logps/rejected": -208.28480529785156, + "loss": 0.1435, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17593544721603394, + "rewards/margins": 3.3050811290740967, + "rewards/rejected": -3.4810166358947754, + "step": 898 + }, + { + "epoch": 1.18, + "learning_rate": 4.8627393962188075e-05, + "logits/chosen": -2.2615466117858887, + "logits/rejected": -2.26615047454834, + "logps/chosen": -265.6878662109375, + "logps/rejected": -265.2508239746094, + "loss": 0.2, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6857917904853821, + "rewards/margins": 3.3371822834014893, + "rewards/rejected": -4.022973537445068, + "step": 899 + }, + { + "epoch": 1.18, + "learning_rate": 4.8623987892382066e-05, + "logits/chosen": -2.3570547103881836, + "logits/rejected": -2.3362598419189453, + "logps/chosen": -199.10662841796875, + "logps/rejected": -217.95907592773438, + "loss": 0.111, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.27905502915382385, + "rewards/margins": 3.4710071086883545, + "rewards/rejected": -3.7500622272491455, + "step": 900 + }, + { + "epoch": 1.18, + "learning_rate": 4.862057772138258e-05, + "logits/chosen": -2.2032244205474854, + "logits/rejected": -2.2657198905944824, + "logps/chosen": -213.52206420898438, + "logps/rejected": -225.04454040527344, + "loss": 0.1846, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8243767023086548, + "rewards/margins": 3.046542167663574, + "rewards/rejected": -3.8709182739257812, + "step": 901 + }, + { + "epoch": 1.18, + "learning_rate": 4.861716344978162e-05, + "logits/chosen": -2.394089698791504, + "logits/rejected": -2.292032241821289, + "logps/chosen": -234.7174835205078, + "logps/rejected": -255.44989013671875, + "loss": 0.1544, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.5697602033615112, + "rewards/margins": 3.2541327476501465, + "rewards/rejected": -3.8238930702209473, + "step": 902 + }, + { + "epoch": 1.18, + "learning_rate": 4.861374507817193e-05, + "logits/chosen": -2.3428890705108643, + "logits/rejected": -2.3085713386535645, + "logps/chosen": -221.3970489501953, + "logps/rejected": -264.0530090332031, + "loss": 0.0761, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.48718422651290894, + "rewards/margins": 3.7521519660949707, + "rewards/rejected": -4.239336013793945, + "step": 903 + }, + { + "epoch": 1.18, + "learning_rate": 4.8610322607146926e-05, + "logits/chosen": -2.417067766189575, + "logits/rejected": -2.393913745880127, + "logps/chosen": -178.41981506347656, + "logps/rejected": -186.7463836669922, + "loss": 0.277, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.6322096586227417, + "rewards/margins": 1.9191558361053467, + "rewards/rejected": -2.551365375518799, + "step": 904 + }, + { + "epoch": 1.18, + "learning_rate": 4.8606896037300776e-05, + "logits/chosen": -2.2859747409820557, + "logits/rejected": -2.3659892082214355, + "logps/chosen": -210.5970458984375, + "logps/rejected": -241.645263671875, + "loss": 0.0721, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2545009255409241, + "rewards/margins": 3.632350444793701, + "rewards/rejected": -3.8868515491485596, + "step": 905 + }, + { + "epoch": 1.19, + "learning_rate": 4.860346536922834e-05, + "logits/chosen": -2.369377374649048, + "logits/rejected": -2.4897830486297607, + "logps/chosen": -254.58274841308594, + "logps/rejected": -319.5621643066406, + "loss": 0.1598, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.3183284401893616, + "rewards/margins": 3.658812999725342, + "rewards/rejected": -3.9771416187286377, + "step": 906 + }, + { + "epoch": 1.19, + "learning_rate": 4.860003060352518e-05, + "logits/chosen": -2.3488306999206543, + "logits/rejected": -2.3815548419952393, + "logps/chosen": -171.2317352294922, + "logps/rejected": -204.7757568359375, + "loss": 0.1461, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.7481203675270081, + "rewards/margins": 2.8177552223205566, + "rewards/rejected": -3.56587553024292, + "step": 907 + }, + { + "epoch": 1.19, + "learning_rate": 4.85965917407876e-05, + "logits/chosen": -2.2379674911499023, + "logits/rejected": -2.3881618976593018, + "logps/chosen": -165.37860107421875, + "logps/rejected": -203.4194793701172, + "loss": 0.2971, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.6805337071418762, + "rewards/margins": 2.6565561294555664, + "rewards/rejected": -3.337089776992798, + "step": 908 + }, + { + "epoch": 1.19, + "learning_rate": 4.8593148781612587e-05, + "logits/chosen": -2.630882740020752, + "logits/rejected": -2.544029951095581, + "logps/chosen": -201.74972534179688, + "logps/rejected": -221.31356811523438, + "loss": 0.1897, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.4801185429096222, + "rewards/margins": 3.1227259635925293, + "rewards/rejected": -3.602844715118408, + "step": 909 + }, + { + "epoch": 1.19, + "learning_rate": 4.858970172659785e-05, + "logits/chosen": -2.405344247817993, + "logits/rejected": -2.471182107925415, + "logps/chosen": -232.18885803222656, + "logps/rejected": -307.023681640625, + "loss": 0.0831, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4412127435207367, + "rewards/margins": 3.6795432567596436, + "rewards/rejected": -4.12075662612915, + "step": 910 + }, + { + "epoch": 1.19, + "learning_rate": 4.85862505763418e-05, + "logits/chosen": -1.9770020246505737, + "logits/rejected": -1.8537812232971191, + "logps/chosen": -236.51913452148438, + "logps/rejected": -222.99220275878906, + "loss": 0.2417, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.4903448224067688, + "rewards/margins": 2.971850872039795, + "rewards/rejected": -3.46219539642334, + "step": 911 + }, + { + "epoch": 1.19, + "learning_rate": 4.858279533144358e-05, + "logits/chosen": -2.5417568683624268, + "logits/rejected": -2.6035280227661133, + "logps/chosen": -172.0741729736328, + "logps/rejected": -214.9192657470703, + "loss": 0.1949, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.9094291925430298, + "rewards/margins": 2.6431045532226562, + "rewards/rejected": -3.5525336265563965, + "step": 912 + }, + { + "epoch": 1.19, + "learning_rate": 4.857933599250303e-05, + "logits/chosen": -2.342681407928467, + "logits/rejected": -2.256180763244629, + "logps/chosen": -196.65240478515625, + "logps/rejected": -195.11648559570312, + "loss": 0.1316, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5412741899490356, + "rewards/margins": 3.1819822788238525, + "rewards/rejected": -3.7232565879821777, + "step": 913 + }, + { + "epoch": 1.2, + "learning_rate": 4.857587256012068e-05, + "logits/chosen": -2.2213358879089355, + "logits/rejected": -2.223132371902466, + "logps/chosen": -217.80227661132812, + "logps/rejected": -258.8191833496094, + "loss": 0.2183, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.33522820472717285, + "rewards/margins": 3.4243385791778564, + "rewards/rejected": -3.759566307067871, + "step": 914 + }, + { + "epoch": 1.2, + "learning_rate": 4.857240503489782e-05, + "logits/chosen": -2.633331060409546, + "logits/rejected": -2.5645124912261963, + "logps/chosen": -200.99314880371094, + "logps/rejected": -216.61944580078125, + "loss": 0.0838, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.03604121133685112, + "rewards/margins": 3.366603136062622, + "rewards/rejected": -3.402644157409668, + "step": 915 + }, + { + "epoch": 1.2, + "learning_rate": 4.8568933417436416e-05, + "logits/chosen": -2.60661244392395, + "logits/rejected": -2.568995475769043, + "logps/chosen": -229.74319458007812, + "logps/rejected": -272.7928161621094, + "loss": 0.0913, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.10647392272949219, + "rewards/margins": 4.324003219604492, + "rewards/rejected": -4.430477142333984, + "step": 916 + }, + { + "epoch": 1.2, + "learning_rate": 4.856545770833913e-05, + "logits/chosen": -2.1530046463012695, + "logits/rejected": -2.075482130050659, + "logps/chosen": -218.10240173339844, + "logps/rejected": -234.6465301513672, + "loss": 0.166, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.3659510612487793, + "rewards/margins": 4.201175212860107, + "rewards/rejected": -3.8352243900299072, + "step": 917 + }, + { + "epoch": 1.2, + "learning_rate": 4.8561977908209384e-05, + "logits/chosen": -2.322742462158203, + "logits/rejected": -2.2862117290496826, + "logps/chosen": -188.68533325195312, + "logps/rejected": -230.53916931152344, + "loss": 0.1539, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.2586498260498047, + "rewards/margins": 3.89760684967041, + "rewards/rejected": -4.156256198883057, + "step": 918 + }, + { + "epoch": 1.2, + "learning_rate": 4.855849401765127e-05, + "logits/chosen": -2.245067834854126, + "logits/rejected": -2.264173984527588, + "logps/chosen": -163.19442749023438, + "logps/rejected": -244.39859008789062, + "loss": 0.2118, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.5528127551078796, + "rewards/margins": 2.983473539352417, + "rewards/rejected": -3.5362865924835205, + "step": 919 + }, + { + "epoch": 1.2, + "learning_rate": 4.8555006037269594e-05, + "logits/chosen": -2.4369189739227295, + "logits/rejected": -2.4961891174316406, + "logps/chosen": -199.59515380859375, + "logps/rejected": -218.68148803710938, + "loss": 0.2001, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.507201611995697, + "rewards/margins": 2.782557487487793, + "rewards/rejected": -3.2897589206695557, + "step": 920 + }, + { + "epoch": 1.21, + "learning_rate": 4.855151396766988e-05, + "logits/chosen": -2.557804584503174, + "logits/rejected": -2.554647922515869, + "logps/chosen": -195.2904815673828, + "logps/rejected": -226.03387451171875, + "loss": 0.1858, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.013599276542663574, + "rewards/margins": 4.105103492736816, + "rewards/rejected": -4.091504096984863, + "step": 921 + }, + { + "epoch": 1.21, + "learning_rate": 4.854801780945837e-05, + "logits/chosen": -1.9899704456329346, + "logits/rejected": -1.9653894901275635, + "logps/chosen": -201.0773468017578, + "logps/rejected": -210.57366943359375, + "loss": 0.247, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.5728083252906799, + "rewards/margins": 2.8485476970672607, + "rewards/rejected": -3.421356201171875, + "step": 922 + }, + { + "epoch": 1.21, + "learning_rate": 4.854451756324201e-05, + "logits/chosen": -2.334390878677368, + "logits/rejected": -2.2994754314422607, + "logps/chosen": -231.74093627929688, + "logps/rejected": -252.7148895263672, + "loss": 0.2625, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.6329984664916992, + "rewards/margins": 2.753272533416748, + "rewards/rejected": -3.386270523071289, + "step": 923 + }, + { + "epoch": 1.21, + "learning_rate": 4.854101322962845e-05, + "logits/chosen": -2.4350640773773193, + "logits/rejected": -2.4160242080688477, + "logps/chosen": -202.79510498046875, + "logps/rejected": -196.56761169433594, + "loss": 0.1783, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.7683102488517761, + "rewards/margins": 2.6973066329956055, + "rewards/rejected": -3.4656169414520264, + "step": 924 + }, + { + "epoch": 1.21, + "learning_rate": 4.853750480922604e-05, + "logits/chosen": -2.4578471183776855, + "logits/rejected": -2.4201016426086426, + "logps/chosen": -236.7185516357422, + "logps/rejected": -225.40760803222656, + "loss": 0.1667, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.3481481373310089, + "rewards/margins": 3.050295352935791, + "rewards/rejected": -3.3984434604644775, + "step": 925 + }, + { + "epoch": 1.21, + "learning_rate": 4.853399230264387e-05, + "logits/chosen": -2.5239510536193848, + "logits/rejected": -2.5775818824768066, + "logps/chosen": -139.86842346191406, + "logps/rejected": -190.04673767089844, + "loss": 0.3665, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.7791432738304138, + "rewards/margins": 1.9640579223632812, + "rewards/rejected": -2.74320125579834, + "step": 926 + }, + { + "epoch": 1.21, + "learning_rate": 4.853047571049171e-05, + "logits/chosen": -2.3366544246673584, + "logits/rejected": -2.2847678661346436, + "logps/chosen": -184.1388702392578, + "logps/rejected": -197.55262756347656, + "loss": 0.1811, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.3248026967048645, + "rewards/margins": 2.802187919616699, + "rewards/rejected": -3.126990795135498, + "step": 927 + }, + { + "epoch": 1.21, + "learning_rate": 4.852695503338005e-05, + "logits/chosen": -2.2322957515716553, + "logits/rejected": -2.232264518737793, + "logps/chosen": -218.7916259765625, + "logps/rejected": -239.33642578125, + "loss": 0.2817, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.7828483581542969, + "rewards/margins": 2.7246387004852295, + "rewards/rejected": -3.5074868202209473, + "step": 928 + }, + { + "epoch": 1.22, + "learning_rate": 4.85234302719201e-05, + "logits/chosen": -2.3104605674743652, + "logits/rejected": -2.292299509048462, + "logps/chosen": -223.07472229003906, + "logps/rejected": -220.38296508789062, + "loss": 0.1399, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.13662558794021606, + "rewards/margins": 3.366118907928467, + "rewards/rejected": -3.502744436264038, + "step": 929 + }, + { + "epoch": 1.22, + "learning_rate": 4.851990142672376e-05, + "logits/chosen": -2.27694034576416, + "logits/rejected": -2.416929244995117, + "logps/chosen": -163.55599975585938, + "logps/rejected": -187.94540405273438, + "loss": 0.1647, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.6064982414245605, + "rewards/margins": 2.823979616165161, + "rewards/rejected": -3.4304776191711426, + "step": 930 + }, + { + "epoch": 1.22, + "learning_rate": 4.8516368498403654e-05, + "logits/chosen": -2.1894800662994385, + "logits/rejected": -2.198641777038574, + "logps/chosen": -192.32850646972656, + "logps/rejected": -232.83926391601562, + "loss": 0.2586, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.7106565237045288, + "rewards/margins": 2.401294231414795, + "rewards/rejected": -3.1119508743286133, + "step": 931 + }, + { + "epoch": 1.22, + "learning_rate": 4.85128314875731e-05, + "logits/chosen": -2.2995123863220215, + "logits/rejected": -2.458733320236206, + "logps/chosen": -184.31101989746094, + "logps/rejected": -263.00048828125, + "loss": 0.0955, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.1651136875152588, + "rewards/margins": 3.4938466548919678, + "rewards/rejected": -3.6589601039886475, + "step": 932 + }, + { + "epoch": 1.22, + "learning_rate": 4.850929039484614e-05, + "logits/chosen": -2.3589913845062256, + "logits/rejected": -2.4086785316467285, + "logps/chosen": -147.282470703125, + "logps/rejected": -214.50318908691406, + "loss": 0.1216, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.45500433444976807, + "rewards/margins": 3.236088275909424, + "rewards/rejected": -3.6910927295684814, + "step": 933 + }, + { + "epoch": 1.22, + "learning_rate": 4.850574522083753e-05, + "logits/chosen": -2.222720146179199, + "logits/rejected": -2.3172354698181152, + "logps/chosen": -180.91482543945312, + "logps/rejected": -231.47238159179688, + "loss": 0.1583, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.7558724284172058, + "rewards/margins": 2.776618480682373, + "rewards/rejected": -3.5324912071228027, + "step": 934 + }, + { + "epoch": 1.22, + "learning_rate": 4.8502195966162694e-05, + "logits/chosen": -2.328801393508911, + "logits/rejected": -2.4272334575653076, + "logps/chosen": -154.9918212890625, + "logps/rejected": -246.27830505371094, + "loss": 0.0796, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6289770007133484, + "rewards/margins": 3.6553382873535156, + "rewards/rejected": -4.28431510925293, + "step": 935 + }, + { + "epoch": 1.22, + "learning_rate": 4.849864263143781e-05, + "logits/chosen": -2.335493564605713, + "logits/rejected": -2.3662097454071045, + "logps/chosen": -163.90878295898438, + "logps/rejected": -192.3372802734375, + "loss": 0.2689, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.5199804306030273, + "rewards/margins": 2.6915531158447266, + "rewards/rejected": -3.211533546447754, + "step": 936 + }, + { + "epoch": 1.23, + "learning_rate": 4.8495085217279755e-05, + "logits/chosen": -2.3108789920806885, + "logits/rejected": -2.411635637283325, + "logps/chosen": -178.46237182617188, + "logps/rejected": -252.6321563720703, + "loss": 0.0912, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.3370632529258728, + "rewards/margins": 3.8149256706237793, + "rewards/rejected": -4.151988506317139, + "step": 937 + }, + { + "epoch": 1.23, + "learning_rate": 4.849152372430609e-05, + "logits/chosen": -2.3515238761901855, + "logits/rejected": -2.4710798263549805, + "logps/chosen": -186.52601623535156, + "logps/rejected": -236.93490600585938, + "loss": 0.1544, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.4036705791950226, + "rewards/margins": 3.555159091949463, + "rewards/rejected": -3.958829402923584, + "step": 938 + }, + { + "epoch": 1.23, + "learning_rate": 4.848795815313511e-05, + "logits/chosen": -2.0485763549804688, + "logits/rejected": -2.0782063007354736, + "logps/chosen": -190.405517578125, + "logps/rejected": -239.02328491210938, + "loss": 0.1048, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.12277688086032867, + "rewards/margins": 3.6550817489624023, + "rewards/rejected": -3.7778589725494385, + "step": 939 + }, + { + "epoch": 1.23, + "learning_rate": 4.8484388504385806e-05, + "logits/chosen": -2.2379536628723145, + "logits/rejected": -2.215625762939453, + "logps/chosen": -169.50054931640625, + "logps/rejected": -222.1637420654297, + "loss": 0.2537, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.6230670809745789, + "rewards/margins": 2.660698413848877, + "rewards/rejected": -3.2837653160095215, + "step": 940 + }, + { + "epoch": 1.23, + "learning_rate": 4.8480814778677886e-05, + "logits/chosen": -2.2735724449157715, + "logits/rejected": -2.31823468208313, + "logps/chosen": -181.47183227539062, + "logps/rejected": -204.40936279296875, + "loss": 0.2299, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.45316997170448303, + "rewards/margins": 3.5822412967681885, + "rewards/rejected": -4.035411357879639, + "step": 941 + }, + { + "epoch": 1.23, + "learning_rate": 4.847723697663175e-05, + "logits/chosen": -2.504059076309204, + "logits/rejected": -2.55910325050354, + "logps/chosen": -228.25503540039062, + "logps/rejected": -302.8750305175781, + "loss": 0.0823, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8190544843673706, + "rewards/margins": 3.961775541305542, + "rewards/rejected": -4.780829906463623, + "step": 942 + }, + { + "epoch": 1.23, + "learning_rate": 4.847365509886851e-05, + "logits/chosen": -2.3396451473236084, + "logits/rejected": -2.4200096130371094, + "logps/chosen": -202.72042846679688, + "logps/rejected": -239.9745635986328, + "loss": 0.0795, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4111427068710327, + "rewards/margins": 3.3438093662261963, + "rewards/rejected": -3.7549519538879395, + "step": 943 + }, + { + "epoch": 1.24, + "learning_rate": 4.847006914601001e-05, + "logits/chosen": -2.4212615489959717, + "logits/rejected": -2.48210072517395, + "logps/chosen": -213.84280395507812, + "logps/rejected": -260.6656188964844, + "loss": 0.1249, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8076397776603699, + "rewards/margins": 3.792834997177124, + "rewards/rejected": -4.600475311279297, + "step": 944 + }, + { + "epoch": 1.24, + "learning_rate": 4.8466479118678766e-05, + "logits/chosen": -2.2285969257354736, + "logits/rejected": -2.315225124359131, + "logps/chosen": -187.538330078125, + "logps/rejected": -268.7898254394531, + "loss": 0.2034, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.2221934795379639, + "rewards/margins": 3.516084671020508, + "rewards/rejected": -4.738278388977051, + "step": 945 + }, + { + "epoch": 1.24, + "learning_rate": 4.846288501749802e-05, + "logits/chosen": -2.5156943798065186, + "logits/rejected": -2.5410516262054443, + "logps/chosen": -222.49359130859375, + "logps/rejected": -282.3955993652344, + "loss": 0.1176, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.7577163577079773, + "rewards/margins": 4.017005920410156, + "rewards/rejected": -4.774722576141357, + "step": 946 + }, + { + "epoch": 1.24, + "learning_rate": 4.8459286843091724e-05, + "logits/chosen": -2.2465834617614746, + "logits/rejected": -2.3958494663238525, + "logps/chosen": -205.92015075683594, + "logps/rejected": -250.27149963378906, + "loss": 0.1785, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.3019455671310425, + "rewards/margins": 3.5620269775390625, + "rewards/rejected": -4.8639726638793945, + "step": 947 + }, + { + "epoch": 1.24, + "learning_rate": 4.845568459608453e-05, + "logits/chosen": -2.2198760509490967, + "logits/rejected": -2.3069236278533936, + "logps/chosen": -164.4407196044922, + "logps/rejected": -227.1414794921875, + "loss": 0.19, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.6457600593566895, + "rewards/margins": 3.604132652282715, + "rewards/rejected": -4.249892711639404, + "step": 948 + }, + { + "epoch": 1.24, + "learning_rate": 4.8452078277101783e-05, + "logits/chosen": -2.1330924034118652, + "logits/rejected": -2.2624382972717285, + "logps/chosen": -141.55337524414062, + "logps/rejected": -193.1168670654297, + "loss": 0.18, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.4278246760368347, + "rewards/margins": 3.105529308319092, + "rewards/rejected": -3.5333540439605713, + "step": 949 + }, + { + "epoch": 1.24, + "learning_rate": 4.8448467886769585e-05, + "logits/chosen": -1.9669086933135986, + "logits/rejected": -2.0849437713623047, + "logps/chosen": -181.0906982421875, + "logps/rejected": -237.8538055419922, + "loss": 0.1504, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0563127994537354, + "rewards/margins": 3.4143903255462646, + "rewards/rejected": -4.470703125, + "step": 950 + }, + { + "epoch": 1.24, + "learning_rate": 4.844485342571468e-05, + "logits/chosen": -1.7133499383926392, + "logits/rejected": -1.701385259628296, + "logps/chosen": -169.70852661132812, + "logps/rejected": -198.37237548828125, + "loss": 0.4077, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.4771201610565186, + "rewards/margins": 2.8859434127807617, + "rewards/rejected": -4.363063335418701, + "step": 951 + }, + { + "epoch": 1.25, + "learning_rate": 4.844123489456456e-05, + "logits/chosen": -2.262361526489258, + "logits/rejected": -2.4248523712158203, + "logps/chosen": -198.08021545410156, + "logps/rejected": -303.7059631347656, + "loss": 0.1379, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.432203769683838, + "rewards/margins": 3.657777786254883, + "rewards/rejected": -5.089981555938721, + "step": 952 + }, + { + "epoch": 1.25, + "learning_rate": 4.843761229394741e-05, + "logits/chosen": -2.3017332553863525, + "logits/rejected": -2.291661500930786, + "logps/chosen": -189.6435089111328, + "logps/rejected": -246.8826141357422, + "loss": 0.0758, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8683138489723206, + "rewards/margins": 4.264334678649902, + "rewards/rejected": -5.132648468017578, + "step": 953 + }, + { + "epoch": 1.25, + "learning_rate": 4.843398562449212e-05, + "logits/chosen": -1.9980626106262207, + "logits/rejected": -1.996732473373413, + "logps/chosen": -190.13116455078125, + "logps/rejected": -222.26028442382812, + "loss": 0.2367, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.423780918121338, + "rewards/margins": 2.6461732387542725, + "rewards/rejected": -4.069953918457031, + "step": 954 + }, + { + "epoch": 1.25, + "learning_rate": 4.84303548868283e-05, + "logits/chosen": -2.3841123580932617, + "logits/rejected": -2.439812660217285, + "logps/chosen": -231.7444610595703, + "logps/rejected": -276.234619140625, + "loss": 0.1215, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.5399219989776611, + "rewards/margins": 3.503429651260376, + "rewards/rejected": -5.043351650238037, + "step": 955 + }, + { + "epoch": 1.25, + "learning_rate": 4.842672008158625e-05, + "logits/chosen": -2.028545379638672, + "logits/rejected": -2.0214412212371826, + "logps/chosen": -144.1702880859375, + "logps/rejected": -193.11929321289062, + "loss": 0.3515, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.5417052507400513, + "rewards/margins": 2.0481746196746826, + "rewards/rejected": -3.5898799896240234, + "step": 956 + }, + { + "epoch": 1.25, + "learning_rate": 4.842308120939699e-05, + "logits/chosen": -2.440342426300049, + "logits/rejected": -2.4507193565368652, + "logps/chosen": -215.12884521484375, + "logps/rejected": -226.1699981689453, + "loss": 0.1973, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6147558689117432, + "rewards/margins": 3.0818982124328613, + "rewards/rejected": -4.696654319763184, + "step": 957 + }, + { + "epoch": 1.25, + "learning_rate": 4.8419438270892226e-05, + "logits/chosen": -2.179013252258301, + "logits/rejected": -2.0757102966308594, + "logps/chosen": -157.2231903076172, + "logps/rejected": -176.61666870117188, + "loss": 0.2506, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5769833326339722, + "rewards/margins": 2.5216851234436035, + "rewards/rejected": -4.098668098449707, + "step": 958 + }, + { + "epoch": 1.26, + "learning_rate": 4.8415791266704404e-05, + "logits/chosen": -2.1697206497192383, + "logits/rejected": -2.1985995769500732, + "logps/chosen": -197.36570739746094, + "logps/rejected": -242.4941864013672, + "loss": 0.217, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0622727870941162, + "rewards/margins": 3.3413608074188232, + "rewards/rejected": -4.4036335945129395, + "step": 959 + }, + { + "epoch": 1.26, + "learning_rate": 4.841214019746663e-05, + "logits/chosen": -2.132883310317993, + "logits/rejected": -2.2006301879882812, + "logps/chosen": -195.13247680664062, + "logps/rejected": -226.83316040039062, + "loss": 0.2752, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.512275218963623, + "rewards/margins": 2.76702880859375, + "rewards/rejected": -4.279304027557373, + "step": 960 + }, + { + "epoch": 1.26, + "learning_rate": 4.840848506381276e-05, + "logits/chosen": -2.286079168319702, + "logits/rejected": -2.3662781715393066, + "logps/chosen": -201.76133728027344, + "logps/rejected": -226.46603393554688, + "loss": 0.1871, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.301385521888733, + "rewards/margins": 3.0335183143615723, + "rewards/rejected": -4.334904193878174, + "step": 961 + }, + { + "epoch": 1.26, + "learning_rate": 4.840482586637732e-05, + "logits/chosen": -2.354762554168701, + "logits/rejected": -2.3351876735687256, + "logps/chosen": -213.7999267578125, + "logps/rejected": -268.6205139160156, + "loss": 0.3769, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.8948153257369995, + "rewards/margins": 2.8223719596862793, + "rewards/rejected": -4.71718692779541, + "step": 962 + }, + { + "epoch": 1.26, + "learning_rate": 4.840116260579557e-05, + "logits/chosen": -2.3253631591796875, + "logits/rejected": -2.4139952659606934, + "logps/chosen": -196.48281860351562, + "logps/rejected": -258.4675598144531, + "loss": 0.0903, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2855195999145508, + "rewards/margins": 4.241164684295654, + "rewards/rejected": -5.526684284210205, + "step": 963 + }, + { + "epoch": 1.26, + "learning_rate": 4.839749528270345e-05, + "logits/chosen": -2.1117494106292725, + "logits/rejected": -2.233508586883545, + "logps/chosen": -162.23635864257812, + "logps/rejected": -216.20143127441406, + "loss": 0.1694, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.412933349609375, + "rewards/margins": 3.0781798362731934, + "rewards/rejected": -4.49111270904541, + "step": 964 + }, + { + "epoch": 1.26, + "learning_rate": 4.8393823897737634e-05, + "logits/chosen": -1.8761091232299805, + "logits/rejected": -1.8636384010314941, + "logps/chosen": -202.1988525390625, + "logps/rejected": -255.47442626953125, + "loss": 0.1544, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0524611473083496, + "rewards/margins": 3.7740285396575928, + "rewards/rejected": -4.8264899253845215, + "step": 965 + }, + { + "epoch": 1.26, + "learning_rate": 4.839014845153548e-05, + "logits/chosen": -2.455805778503418, + "logits/rejected": -2.4408648014068604, + "logps/chosen": -265.5133056640625, + "logps/rejected": -322.55694580078125, + "loss": 0.1826, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1125422716140747, + "rewards/margins": 3.8850979804992676, + "rewards/rejected": -4.997640132904053, + "step": 966 + }, + { + "epoch": 1.27, + "learning_rate": 4.838646894473505e-05, + "logits/chosen": -1.7058061361312866, + "logits/rejected": -1.7635576725006104, + "logps/chosen": -178.57357788085938, + "logps/rejected": -224.11141967773438, + "loss": 0.2088, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.7308789491653442, + "rewards/margins": 2.972151279449463, + "rewards/rejected": -4.703030586242676, + "step": 967 + }, + { + "epoch": 1.27, + "learning_rate": 4.8382785377975116e-05, + "logits/chosen": -2.187974452972412, + "logits/rejected": -2.2810630798339844, + "logps/chosen": -189.5592803955078, + "logps/rejected": -282.3250732421875, + "loss": 0.2646, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.5101513862609863, + "rewards/margins": 3.0083746910095215, + "rewards/rejected": -4.518526077270508, + "step": 968 + }, + { + "epoch": 1.27, + "learning_rate": 4.8379097751895166e-05, + "logits/chosen": -2.3393805027008057, + "logits/rejected": -2.3446669578552246, + "logps/chosen": -280.5021667480469, + "logps/rejected": -287.1352233886719, + "loss": 0.1423, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8774653077125549, + "rewards/margins": 3.9873814582824707, + "rewards/rejected": -4.864847183227539, + "step": 969 + }, + { + "epoch": 1.27, + "learning_rate": 4.837540606713538e-05, + "logits/chosen": -2.3770768642425537, + "logits/rejected": -2.3758389949798584, + "logps/chosen": -198.17864990234375, + "logps/rejected": -225.19105529785156, + "loss": 0.1787, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.3526650667190552, + "rewards/margins": 2.9090161323547363, + "rewards/rejected": -4.26168155670166, + "step": 970 + }, + { + "epoch": 1.27, + "learning_rate": 4.837171032433663e-05, + "logits/chosen": -2.226538896560669, + "logits/rejected": -2.3716554641723633, + "logps/chosen": -181.35006713867188, + "logps/rejected": -279.17645263671875, + "loss": 0.0983, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6791074275970459, + "rewards/margins": 4.5800275802612305, + "rewards/rejected": -5.259134292602539, + "step": 971 + }, + { + "epoch": 1.27, + "learning_rate": 4.836801052414053e-05, + "logits/chosen": -1.950883388519287, + "logits/rejected": -2.080726146697998, + "logps/chosen": -170.36874389648438, + "logps/rejected": -215.67941284179688, + "loss": 0.2373, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.109346866607666, + "rewards/margins": 2.8225221633911133, + "rewards/rejected": -3.9318690299987793, + "step": 972 + }, + { + "epoch": 1.27, + "learning_rate": 4.8364306667189364e-05, + "logits/chosen": -2.23186993598938, + "logits/rejected": -2.1386542320251465, + "logps/chosen": -180.2755126953125, + "logps/rejected": -230.13412475585938, + "loss": 0.112, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8742038011550903, + "rewards/margins": 3.692150115966797, + "rewards/rejected": -4.566353797912598, + "step": 973 + }, + { + "epoch": 1.27, + "learning_rate": 4.836059875412613e-05, + "logits/chosen": -2.152693748474121, + "logits/rejected": -2.1664419174194336, + "logps/chosen": -197.43896484375, + "logps/rejected": -206.69210815429688, + "loss": 0.2823, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.0961377620697021, + "rewards/margins": 3.136141538619995, + "rewards/rejected": -4.2322797775268555, + "step": 974 + }, + { + "epoch": 1.28, + "learning_rate": 4.8356886785594544e-05, + "logits/chosen": -2.3830044269561768, + "logits/rejected": -2.392028331756592, + "logps/chosen": -191.5318603515625, + "logps/rejected": -222.13458251953125, + "loss": 0.082, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9227029085159302, + "rewards/margins": 4.220012664794922, + "rewards/rejected": -5.142715930938721, + "step": 975 + }, + { + "epoch": 1.28, + "learning_rate": 4.835317076223901e-05, + "logits/chosen": -2.298764228820801, + "logits/rejected": -2.360914468765259, + "logps/chosen": -238.4506378173828, + "logps/rejected": -276.4215087890625, + "loss": 0.1926, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.0809506177902222, + "rewards/margins": 3.495990037918091, + "rewards/rejected": -4.576941013336182, + "step": 976 + }, + { + "epoch": 1.28, + "learning_rate": 4.834945068470463e-05, + "logits/chosen": -2.3851685523986816, + "logits/rejected": -2.402043342590332, + "logps/chosen": -215.42071533203125, + "logps/rejected": -260.2996520996094, + "loss": 0.1705, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.8322378396987915, + "rewards/margins": 3.276536703109741, + "rewards/rejected": -4.1087751388549805, + "step": 977 + }, + { + "epoch": 1.28, + "learning_rate": 4.834572655363723e-05, + "logits/chosen": -2.3521459102630615, + "logits/rejected": -2.359070301055908, + "logps/chosen": -172.63397216796875, + "logps/rejected": -249.02813720703125, + "loss": 0.1009, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.6295977830886841, + "rewards/margins": 4.736910343170166, + "rewards/rejected": -5.3665080070495605, + "step": 978 + }, + { + "epoch": 1.28, + "learning_rate": 4.834199836968334e-05, + "logits/chosen": -2.232576847076416, + "logits/rejected": -2.2787835597991943, + "logps/chosen": -212.65316772460938, + "logps/rejected": -277.7938537597656, + "loss": 0.1771, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.9311662912368774, + "rewards/margins": 4.264695644378662, + "rewards/rejected": -5.19586181640625, + "step": 979 + }, + { + "epoch": 1.28, + "learning_rate": 4.833826613349016e-05, + "logits/chosen": -2.2517361640930176, + "logits/rejected": -2.284072160720825, + "logps/chosen": -220.1957550048828, + "logps/rejected": -214.6461181640625, + "loss": 0.2048, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.8580664396286011, + "rewards/margins": 3.004542589187622, + "rewards/rejected": -3.8626086711883545, + "step": 980 + }, + { + "epoch": 1.28, + "learning_rate": 4.8334529845705635e-05, + "logits/chosen": -2.3734512329101562, + "logits/rejected": -2.3852882385253906, + "logps/chosen": -185.3246612548828, + "logps/rejected": -235.97036743164062, + "loss": 0.0987, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8764241337776184, + "rewards/margins": 3.519378900527954, + "rewards/rejected": -4.395802974700928, + "step": 981 + }, + { + "epoch": 1.29, + "learning_rate": 4.833078950697839e-05, + "logits/chosen": -2.318061113357544, + "logits/rejected": -2.3952810764312744, + "logps/chosen": -203.9101104736328, + "logps/rejected": -264.8881530761719, + "loss": 0.1093, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5848926305770874, + "rewards/margins": 3.398397207260132, + "rewards/rejected": -4.98328971862793, + "step": 982 + }, + { + "epoch": 1.29, + "learning_rate": 4.832704511795776e-05, + "logits/chosen": -2.257204294204712, + "logits/rejected": -2.243340015411377, + "logps/chosen": -220.27609252929688, + "logps/rejected": -302.30224609375, + "loss": 0.0464, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7187144756317139, + "rewards/margins": 5.5636396408081055, + "rewards/rejected": -6.282354354858398, + "step": 983 + }, + { + "epoch": 1.29, + "learning_rate": 4.8323296679293774e-05, + "logits/chosen": -2.0718610286712646, + "logits/rejected": -2.163663148880005, + "logps/chosen": -201.58529663085938, + "logps/rejected": -284.6639099121094, + "loss": 0.186, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.4365129470825195, + "rewards/margins": 3.119493007659912, + "rewards/rejected": -4.55600643157959, + "step": 984 + }, + { + "epoch": 1.29, + "learning_rate": 4.831954419163719e-05, + "logits/chosen": -2.378993034362793, + "logits/rejected": -2.4145779609680176, + "logps/chosen": -194.3405303955078, + "logps/rejected": -258.2578430175781, + "loss": 0.1569, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0772855281829834, + "rewards/margins": 3.5224766731262207, + "rewards/rejected": -4.599761962890625, + "step": 985 + }, + { + "epoch": 1.29, + "learning_rate": 4.8315787655639436e-05, + "logits/chosen": -2.158259391784668, + "logits/rejected": -2.242321729660034, + "logps/chosen": -172.10662841796875, + "logps/rejected": -238.54212951660156, + "loss": 0.1116, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0470584630966187, + "rewards/margins": 3.594759225845337, + "rewards/rejected": -4.641817569732666, + "step": 986 + }, + { + "epoch": 1.29, + "learning_rate": 4.8312027071952645e-05, + "logits/chosen": -2.3166654109954834, + "logits/rejected": -2.371119499206543, + "logps/chosen": -201.59706115722656, + "logps/rejected": -208.94171142578125, + "loss": 0.1019, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.161280632019043, + "rewards/margins": 3.300464630126953, + "rewards/rejected": -4.461745262145996, + "step": 987 + }, + { + "epoch": 1.29, + "learning_rate": 4.8308262441229704e-05, + "logits/chosen": -2.368316888809204, + "logits/rejected": -2.464874029159546, + "logps/chosen": -227.5435333251953, + "logps/rejected": -269.635009765625, + "loss": 0.1186, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2794337272644043, + "rewards/margins": 3.7162654399871826, + "rewards/rejected": -4.995699882507324, + "step": 988 + }, + { + "epoch": 1.29, + "learning_rate": 4.830449376412413e-05, + "logits/chosen": -2.4263885021209717, + "logits/rejected": -2.330914258956909, + "logps/chosen": -174.01873779296875, + "logps/rejected": -176.23687744140625, + "loss": 0.2376, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1324949264526367, + "rewards/margins": 2.7193217277526855, + "rewards/rejected": -3.8518168926239014, + "step": 989 + }, + { + "epoch": 1.3, + "learning_rate": 4.8300721041290194e-05, + "logits/chosen": -2.482351303100586, + "logits/rejected": -2.511679172515869, + "logps/chosen": -258.9258117675781, + "logps/rejected": -289.4918212890625, + "loss": 0.0891, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.4783955812454224, + "rewards/margins": 3.5242300033569336, + "rewards/rejected": -5.002625465393066, + "step": 990 + }, + { + "epoch": 1.3, + "learning_rate": 4.829694427338284e-05, + "logits/chosen": -2.210787773132324, + "logits/rejected": -2.2695932388305664, + "logps/chosen": -242.7936553955078, + "logps/rejected": -285.85845947265625, + "loss": 0.0737, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1135588884353638, + "rewards/margins": 4.644796371459961, + "rewards/rejected": -5.758354663848877, + "step": 991 + }, + { + "epoch": 1.3, + "learning_rate": 4.829316346105773e-05, + "logits/chosen": -2.5152206420898438, + "logits/rejected": -2.598931312561035, + "logps/chosen": -233.4760284423828, + "logps/rejected": -257.0856628417969, + "loss": 0.1722, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4175981283187866, + "rewards/margins": 3.751023530960083, + "rewards/rejected": -5.168622016906738, + "step": 992 + }, + { + "epoch": 1.3, + "learning_rate": 4.828937860497124e-05, + "logits/chosen": -1.9849010705947876, + "logits/rejected": -2.02589750289917, + "logps/chosen": -202.5492706298828, + "logps/rejected": -240.67250061035156, + "loss": 0.2435, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1498560905456543, + "rewards/margins": 2.9630770683288574, + "rewards/rejected": -4.112933158874512, + "step": 993 + }, + { + "epoch": 1.3, + "learning_rate": 4.82855897057804e-05, + "logits/chosen": -2.2394447326660156, + "logits/rejected": -2.2644007205963135, + "logps/chosen": -200.177978515625, + "logps/rejected": -232.31613159179688, + "loss": 0.1656, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2112644910812378, + "rewards/margins": 3.5165750980377197, + "rewards/rejected": -4.727839469909668, + "step": 994 + }, + { + "epoch": 1.3, + "learning_rate": 4.8281796764143e-05, + "logits/chosen": -2.1232497692108154, + "logits/rejected": -2.1741788387298584, + "logps/chosen": -187.11965942382812, + "logps/rejected": -211.57391357421875, + "loss": 0.0946, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9103628396987915, + "rewards/margins": 3.7156426906585693, + "rewards/rejected": -4.62600564956665, + "step": 995 + }, + { + "epoch": 1.3, + "learning_rate": 4.8277999780717496e-05, + "logits/chosen": -2.3636045455932617, + "logits/rejected": -2.3993918895721436, + "logps/chosen": -209.7058868408203, + "logps/rejected": -273.6556396484375, + "loss": 0.1253, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.197162389755249, + "rewards/margins": 4.630073547363281, + "rewards/rejected": -5.827235698699951, + "step": 996 + }, + { + "epoch": 1.3, + "learning_rate": 4.827419875616306e-05, + "logits/chosen": -2.4287328720092773, + "logits/rejected": -2.444946765899658, + "logps/chosen": -187.06781005859375, + "logps/rejected": -227.0997314453125, + "loss": 0.3696, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.5417091846466064, + "rewards/margins": 3.166154623031616, + "rewards/rejected": -4.707863807678223, + "step": 997 + }, + { + "epoch": 1.31, + "learning_rate": 4.827039369113956e-05, + "logits/chosen": -2.3056039810180664, + "logits/rejected": -2.3643782138824463, + "logps/chosen": -218.290283203125, + "logps/rejected": -289.4818420410156, + "loss": 0.0622, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1654748916625977, + "rewards/margins": 4.145327091217041, + "rewards/rejected": -5.310801982879639, + "step": 998 + }, + { + "epoch": 1.31, + "learning_rate": 4.826658458630756e-05, + "logits/chosen": -2.3559858798980713, + "logits/rejected": -2.4298417568206787, + "logps/chosen": -279.0400695800781, + "logps/rejected": -276.64044189453125, + "loss": 0.1911, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6909762620925903, + "rewards/margins": 3.3049442768096924, + "rewards/rejected": -4.995920658111572, + "step": 999 + }, + { + "epoch": 1.31, + "learning_rate": 4.826277144232834e-05, + "logits/chosen": -2.3466007709503174, + "logits/rejected": -2.376455068588257, + "logps/chosen": -215.84930419921875, + "logps/rejected": -249.32769775390625, + "loss": 0.1875, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2421891689300537, + "rewards/margins": 4.225651264190674, + "rewards/rejected": -5.467840194702148, + "step": 1000 + }, + { + "epoch": 1.31, + "learning_rate": 4.825895425986386e-05, + "logits/chosen": -2.164769172668457, + "logits/rejected": -2.1837494373321533, + "logps/chosen": -250.56695556640625, + "logps/rejected": -270.6837463378906, + "loss": 0.1681, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.4974141120910645, + "rewards/margins": 3.5974225997924805, + "rewards/rejected": -5.094837188720703, + "step": 1001 + }, + { + "epoch": 1.31, + "learning_rate": 4.82551330395768e-05, + "logits/chosen": -2.248732328414917, + "logits/rejected": -2.308931589126587, + "logps/chosen": -226.01544189453125, + "logps/rejected": -276.2228698730469, + "loss": 0.1258, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.646399736404419, + "rewards/margins": 3.5327749252319336, + "rewards/rejected": -5.179174423217773, + "step": 1002 + }, + { + "epoch": 1.31, + "learning_rate": 4.825130778213055e-05, + "logits/chosen": -1.9366941452026367, + "logits/rejected": -1.9974042177200317, + "logps/chosen": -248.90155029296875, + "logps/rejected": -284.5340576171875, + "loss": 0.1246, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5386114120483398, + "rewards/margins": 4.703797340393066, + "rewards/rejected": -6.242408752441406, + "step": 1003 + }, + { + "epoch": 1.31, + "learning_rate": 4.8247478488189176e-05, + "logits/chosen": -2.381505012512207, + "logits/rejected": -2.374134063720703, + "logps/chosen": -215.34832763671875, + "logps/rejected": -241.09263610839844, + "loss": 0.1462, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2310172319412231, + "rewards/margins": 3.3694674968719482, + "rewards/rejected": -4.600484848022461, + "step": 1004 + }, + { + "epoch": 1.32, + "learning_rate": 4.824364515841745e-05, + "logits/chosen": -2.39587664604187, + "logits/rejected": -2.445971727371216, + "logps/chosen": -199.75390625, + "logps/rejected": -250.50318908691406, + "loss": 0.1496, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2829456329345703, + "rewards/margins": 3.5297467708587646, + "rewards/rejected": -4.812692642211914, + "step": 1005 + }, + { + "epoch": 1.32, + "learning_rate": 4.8239807793480845e-05, + "logits/chosen": -2.064811944961548, + "logits/rejected": -2.2144877910614014, + "logps/chosen": -206.80836486816406, + "logps/rejected": -274.880859375, + "loss": 0.0629, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6409518718719482, + "rewards/margins": 4.403507709503174, + "rewards/rejected": -6.044458866119385, + "step": 1006 + }, + { + "epoch": 1.32, + "learning_rate": 4.823596639404555e-05, + "logits/chosen": -1.9515126943588257, + "logits/rejected": -1.9221199750900269, + "logps/chosen": -183.7147216796875, + "logps/rejected": -199.12330627441406, + "loss": 0.2176, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.4174538850784302, + "rewards/margins": 2.61428165435791, + "rewards/rejected": -4.031735420227051, + "step": 1007 + }, + { + "epoch": 1.32, + "learning_rate": 4.8232120960778444e-05, + "logits/chosen": -2.1660711765289307, + "logits/rejected": -2.1986541748046875, + "logps/chosen": -219.20086669921875, + "logps/rejected": -308.8476867675781, + "loss": 0.1746, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5317251682281494, + "rewards/margins": 4.233796119689941, + "rewards/rejected": -5.765522003173828, + "step": 1008 + }, + { + "epoch": 1.32, + "learning_rate": 4.822827149434709e-05, + "logits/chosen": -2.3264455795288086, + "logits/rejected": -2.340402126312256, + "logps/chosen": -206.29624938964844, + "logps/rejected": -227.3938446044922, + "loss": 0.2069, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2022267580032349, + "rewards/margins": 3.670454263687134, + "rewards/rejected": -4.872681140899658, + "step": 1009 + }, + { + "epoch": 1.32, + "learning_rate": 4.822441799541979e-05, + "logits/chosen": -1.881190538406372, + "logits/rejected": -1.9560871124267578, + "logps/chosen": -176.16571044921875, + "logps/rejected": -228.5875244140625, + "loss": 0.3731, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.7666243314743042, + "rewards/margins": 2.6788949966430664, + "rewards/rejected": -4.44551944732666, + "step": 1010 + }, + { + "epoch": 1.32, + "learning_rate": 4.82205604646655e-05, + "logits/chosen": -1.936165690422058, + "logits/rejected": -1.9999370574951172, + "logps/chosen": -175.9329833984375, + "logps/rejected": -251.57110595703125, + "loss": 0.1323, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2743474245071411, + "rewards/margins": 4.712624549865723, + "rewards/rejected": -5.986972332000732, + "step": 1011 + }, + { + "epoch": 1.32, + "learning_rate": 4.8216698902753915e-05, + "logits/chosen": -2.4014575481414795, + "logits/rejected": -2.3882806301116943, + "logps/chosen": -235.85289001464844, + "logps/rejected": -276.8750915527344, + "loss": 0.1432, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5504268407821655, + "rewards/margins": 4.225334644317627, + "rewards/rejected": -5.775761127471924, + "step": 1012 + }, + { + "epoch": 1.33, + "learning_rate": 4.8212833310355397e-05, + "logits/chosen": -2.2567152976989746, + "logits/rejected": -2.307617425918579, + "logps/chosen": -250.41036987304688, + "logps/rejected": -298.62841796875, + "loss": 0.1228, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.641356110572815, + "rewards/margins": 3.9862754344940186, + "rewards/rejected": -5.627632141113281, + "step": 1013 + }, + { + "epoch": 1.33, + "learning_rate": 4.820896368814104e-05, + "logits/chosen": -2.222958564758301, + "logits/rejected": -2.243933916091919, + "logps/chosen": -144.96896362304688, + "logps/rejected": -212.0235595703125, + "loss": 0.115, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9734640121459961, + "rewards/margins": 3.973369598388672, + "rewards/rejected": -4.946833610534668, + "step": 1014 + }, + { + "epoch": 1.33, + "learning_rate": 4.8205090036782626e-05, + "logits/chosen": -2.0626208782196045, + "logits/rejected": -2.0410847663879395, + "logps/chosen": -171.70816040039062, + "logps/rejected": -231.00027465820312, + "loss": 0.1491, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.3891879320144653, + "rewards/margins": 3.029548406600952, + "rewards/rejected": -4.418736457824707, + "step": 1015 + }, + { + "epoch": 1.33, + "learning_rate": 4.820121235695261e-05, + "logits/chosen": -2.0706686973571777, + "logits/rejected": -2.0150628089904785, + "logps/chosen": -160.81874084472656, + "logps/rejected": -187.8358917236328, + "loss": 0.2422, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.6058461666107178, + "rewards/margins": 2.6923718452453613, + "rewards/rejected": -4.298218250274658, + "step": 1016 + }, + { + "epoch": 1.33, + "learning_rate": 4.8197330649324184e-05, + "logits/chosen": -2.411139965057373, + "logits/rejected": -2.2218780517578125, + "logps/chosen": -143.00469970703125, + "logps/rejected": -175.34141540527344, + "loss": 0.2557, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.3263788223266602, + "rewards/margins": 3.335325241088867, + "rewards/rejected": -4.661704063415527, + "step": 1017 + }, + { + "epoch": 1.33, + "learning_rate": 4.819344491457122e-05, + "logits/chosen": -2.058603048324585, + "logits/rejected": -2.1647167205810547, + "logps/chosen": -168.77676391601562, + "logps/rejected": -233.14918518066406, + "loss": 0.1883, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.0997314453125, + "rewards/margins": 3.1378016471862793, + "rewards/rejected": -5.2375335693359375, + "step": 1018 + }, + { + "epoch": 1.33, + "learning_rate": 4.818955515336829e-05, + "logits/chosen": -2.3335368633270264, + "logits/rejected": -2.343855381011963, + "logps/chosen": -220.887939453125, + "logps/rejected": -275.09100341796875, + "loss": 0.2941, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.5219594240188599, + "rewards/margins": 4.077702045440674, + "rewards/rejected": -5.599661350250244, + "step": 1019 + }, + { + "epoch": 1.33, + "learning_rate": 4.8185661366390676e-05, + "logits/chosen": -2.0001189708709717, + "logits/rejected": -2.10128116607666, + "logps/chosen": -237.78350830078125, + "logps/rejected": -302.6692810058594, + "loss": 0.2217, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.2568397521972656, + "rewards/margins": 3.168138027191162, + "rewards/rejected": -6.4249773025512695, + "step": 1020 + }, + { + "epoch": 1.34, + "learning_rate": 4.8181763554314345e-05, + "logits/chosen": -2.1002883911132812, + "logits/rejected": -2.1318533420562744, + "logps/chosen": -215.0624237060547, + "logps/rejected": -272.5454406738281, + "loss": 0.0886, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1878482103347778, + "rewards/margins": 5.048128128051758, + "rewards/rejected": -6.2359771728515625, + "step": 1021 + }, + { + "epoch": 1.34, + "learning_rate": 4.8177861717815976e-05, + "logits/chosen": -2.43477463722229, + "logits/rejected": -2.367626905441284, + "logps/chosen": -170.18011474609375, + "logps/rejected": -194.78160095214844, + "loss": 0.2638, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.694594383239746, + "rewards/margins": 2.889676094055176, + "rewards/rejected": -4.584270000457764, + "step": 1022 + }, + { + "epoch": 1.34, + "learning_rate": 4.8173955857572924e-05, + "logits/chosen": -2.2585108280181885, + "logits/rejected": -2.2841644287109375, + "logps/chosen": -221.76345825195312, + "logps/rejected": -282.0390930175781, + "loss": 0.123, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.552736759185791, + "rewards/margins": 4.385633945465088, + "rewards/rejected": -5.938370704650879, + "step": 1023 + }, + { + "epoch": 1.34, + "learning_rate": 4.817004597426327e-05, + "logits/chosen": -2.252422332763672, + "logits/rejected": -2.369584083557129, + "logps/chosen": -198.19515991210938, + "logps/rejected": -256.17254638671875, + "loss": 0.0797, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.3669312000274658, + "rewards/margins": 4.838632583618164, + "rewards/rejected": -6.205563545227051, + "step": 1024 + }, + { + "epoch": 1.34, + "learning_rate": 4.816613206856577e-05, + "logits/chosen": -1.7100903987884521, + "logits/rejected": -1.7708444595336914, + "logps/chosen": -224.86050415039062, + "logps/rejected": -296.8453674316406, + "loss": 0.1638, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.458321452140808, + "rewards/margins": 4.833914756774902, + "rewards/rejected": -6.292235851287842, + "step": 1025 + }, + { + "epoch": 1.34, + "learning_rate": 4.8162214141159914e-05, + "logits/chosen": -2.167717933654785, + "logits/rejected": -2.2409541606903076, + "logps/chosen": -137.0130615234375, + "logps/rejected": -195.80404663085938, + "loss": 0.2602, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.234320878982544, + "rewards/margins": 3.5150794982910156, + "rewards/rejected": -4.749400615692139, + "step": 1026 + }, + { + "epoch": 1.34, + "learning_rate": 4.815829219272584e-05, + "logits/chosen": -2.1251039505004883, + "logits/rejected": -2.1560146808624268, + "logps/chosen": -217.72801208496094, + "logps/rejected": -259.56439208984375, + "loss": 0.1545, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.7046087980270386, + "rewards/margins": 3.3975937366485596, + "rewards/rejected": -5.102202415466309, + "step": 1027 + }, + { + "epoch": 1.35, + "learning_rate": 4.815436622394441e-05, + "logits/chosen": -2.1002719402313232, + "logits/rejected": -2.108283042907715, + "logps/chosen": -123.21249389648438, + "logps/rejected": -177.59942626953125, + "loss": 0.334, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.118762493133545, + "rewards/margins": 2.3827548027038574, + "rewards/rejected": -3.5015172958374023, + "step": 1028 + }, + { + "epoch": 1.35, + "learning_rate": 4.815043623549721e-05, + "logits/chosen": -1.9848045110702515, + "logits/rejected": -2.0297982692718506, + "logps/chosen": -191.30886840820312, + "logps/rejected": -238.38487243652344, + "loss": 0.1696, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3014191389083862, + "rewards/margins": 3.1689629554748535, + "rewards/rejected": -4.470382213592529, + "step": 1029 + }, + { + "epoch": 1.35, + "learning_rate": 4.814650222806647e-05, + "logits/chosen": -2.1499392986297607, + "logits/rejected": -2.2093987464904785, + "logps/chosen": -200.4322509765625, + "logps/rejected": -266.98614501953125, + "loss": 0.1951, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3251994848251343, + "rewards/margins": 3.7016544342041016, + "rewards/rejected": -5.026854038238525, + "step": 1030 + }, + { + "epoch": 1.35, + "learning_rate": 4.8142564202335155e-05, + "logits/chosen": -2.294783115386963, + "logits/rejected": -2.1916940212249756, + "logps/chosen": -168.14263916015625, + "logps/rejected": -178.0428466796875, + "loss": 0.2519, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.3096660375595093, + "rewards/margins": 2.79194974899292, + "rewards/rejected": -4.101615905761719, + "step": 1031 + }, + { + "epoch": 1.35, + "learning_rate": 4.813862215898692e-05, + "logits/chosen": -2.161398410797119, + "logits/rejected": -2.232086181640625, + "logps/chosen": -197.67620849609375, + "logps/rejected": -290.12420654296875, + "loss": 0.1613, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.239977478981018, + "rewards/margins": 3.1511924266815186, + "rewards/rejected": -4.391169548034668, + "step": 1032 + }, + { + "epoch": 1.35, + "learning_rate": 4.813467609870611e-05, + "logits/chosen": -2.2221438884735107, + "logits/rejected": -2.1995956897735596, + "logps/chosen": -224.52113342285156, + "logps/rejected": -237.9773712158203, + "loss": 0.0758, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0239717960357666, + "rewards/margins": 4.245767116546631, + "rewards/rejected": -5.269738674163818, + "step": 1033 + }, + { + "epoch": 1.35, + "learning_rate": 4.813072602217778e-05, + "logits/chosen": -2.0670764446258545, + "logits/rejected": -2.113513469696045, + "logps/chosen": -233.91201782226562, + "logps/rejected": -259.1944885253906, + "loss": 0.2214, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.0860904455184937, + "rewards/margins": 4.289690971374512, + "rewards/rejected": -5.375781059265137, + "step": 1034 + }, + { + "epoch": 1.35, + "learning_rate": 4.8126771930087674e-05, + "logits/chosen": -2.3435795307159424, + "logits/rejected": -2.508849620819092, + "logps/chosen": -174.04713439941406, + "logps/rejected": -249.47862243652344, + "loss": 0.2356, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.164699673652649, + "rewards/margins": 4.200926303863525, + "rewards/rejected": -5.365626335144043, + "step": 1035 + }, + { + "epoch": 1.36, + "learning_rate": 4.8122813823122225e-05, + "logits/chosen": -2.4261474609375, + "logits/rejected": -2.459974527359009, + "logps/chosen": -207.60198974609375, + "logps/rejected": -263.3089294433594, + "loss": 0.1271, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1393440961837769, + "rewards/margins": 4.021913528442383, + "rewards/rejected": -5.161257266998291, + "step": 1036 + }, + { + "epoch": 1.36, + "learning_rate": 4.8118851701968584e-05, + "logits/chosen": -2.2980728149414062, + "logits/rejected": -2.3554952144622803, + "logps/chosen": -187.2412872314453, + "logps/rejected": -224.6619873046875, + "loss": 0.126, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.9625990390777588, + "rewards/margins": 3.2137532234191895, + "rewards/rejected": -4.176352500915527, + "step": 1037 + }, + { + "epoch": 1.36, + "learning_rate": 4.811488556731457e-05, + "logits/chosen": -1.995209813117981, + "logits/rejected": -2.0629470348358154, + "logps/chosen": -194.26153564453125, + "logps/rejected": -252.84649658203125, + "loss": 0.0712, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7154987454414368, + "rewards/margins": 4.5765485763549805, + "rewards/rejected": -5.292047023773193, + "step": 1038 + }, + { + "epoch": 1.36, + "learning_rate": 4.8110915419848734e-05, + "logits/chosen": -2.278034210205078, + "logits/rejected": -2.3754935264587402, + "logps/chosen": -180.0498809814453, + "logps/rejected": -240.5858154296875, + "loss": 0.1379, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1113700866699219, + "rewards/margins": 3.6900339126586914, + "rewards/rejected": -4.801403999328613, + "step": 1039 + }, + { + "epoch": 1.36, + "learning_rate": 4.8106941260260296e-05, + "logits/chosen": -2.460465908050537, + "logits/rejected": -2.4936022758483887, + "logps/chosen": -239.6426544189453, + "logps/rejected": -292.16949462890625, + "loss": 0.0555, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9402986764907837, + "rewards/margins": 4.606878280639648, + "rewards/rejected": -5.547176837921143, + "step": 1040 + }, + { + "epoch": 1.36, + "learning_rate": 4.8102963089239185e-05, + "logits/chosen": -2.076064348220825, + "logits/rejected": -2.1251323223114014, + "logps/chosen": -140.34951782226562, + "logps/rejected": -206.3195343017578, + "loss": 0.3204, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.3504629135131836, + "rewards/margins": 3.5889742374420166, + "rewards/rejected": -4.939437389373779, + "step": 1041 + }, + { + "epoch": 1.36, + "learning_rate": 4.809898090747601e-05, + "logits/chosen": -2.289942502975464, + "logits/rejected": -2.3287506103515625, + "logps/chosen": -213.01470947265625, + "logps/rejected": -265.6434020996094, + "loss": 0.1479, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.9047419428825378, + "rewards/margins": 3.5847079753875732, + "rewards/rejected": -4.489449977874756, + "step": 1042 + }, + { + "epoch": 1.36, + "learning_rate": 4.809499471566211e-05, + "logits/chosen": -2.0999417304992676, + "logits/rejected": -2.199672222137451, + "logps/chosen": -178.01663208007812, + "logps/rejected": -248.11026000976562, + "loss": 0.105, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8936255574226379, + "rewards/margins": 4.213724613189697, + "rewards/rejected": -5.107350826263428, + "step": 1043 + }, + { + "epoch": 1.37, + "learning_rate": 4.809100451448949e-05, + "logits/chosen": -2.2089450359344482, + "logits/rejected": -2.1612417697906494, + "logps/chosen": -157.15957641601562, + "logps/rejected": -182.26841735839844, + "loss": 0.2212, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.1464567184448242, + "rewards/margins": 3.457570791244507, + "rewards/rejected": -4.604027271270752, + "step": 1044 + }, + { + "epoch": 1.37, + "learning_rate": 4.8087010304650866e-05, + "logits/chosen": -2.1104516983032227, + "logits/rejected": -2.153797149658203, + "logps/chosen": -222.3661651611328, + "logps/rejected": -295.9107666015625, + "loss": 0.0823, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4407971203327179, + "rewards/margins": 4.490683078765869, + "rewards/rejected": -4.9314799308776855, + "step": 1045 + }, + { + "epoch": 1.37, + "learning_rate": 4.808301208683963e-05, + "logits/chosen": -2.568639039993286, + "logits/rejected": -2.5642662048339844, + "logps/chosen": -239.6685333251953, + "logps/rejected": -248.938720703125, + "loss": 0.1162, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3235012292861938, + "rewards/margins": 4.261366844177246, + "rewards/rejected": -5.58486795425415, + "step": 1046 + }, + { + "epoch": 1.37, + "learning_rate": 4.8079009861749904e-05, + "logits/chosen": -2.3855719566345215, + "logits/rejected": -2.369443893432617, + "logps/chosen": -236.78912353515625, + "logps/rejected": -273.231689453125, + "loss": 0.0566, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4863171577453613, + "rewards/margins": 4.7871551513671875, + "rewards/rejected": -6.273472785949707, + "step": 1047 + }, + { + "epoch": 1.37, + "learning_rate": 4.807500363007647e-05, + "logits/chosen": -2.229086399078369, + "logits/rejected": -2.2423181533813477, + "logps/chosen": -165.6907958984375, + "logps/rejected": -186.762451171875, + "loss": 0.1962, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.7180208563804626, + "rewards/margins": 2.7720799446105957, + "rewards/rejected": -3.490100860595703, + "step": 1048 + }, + { + "epoch": 1.37, + "learning_rate": 4.8070993392514826e-05, + "logits/chosen": -2.305532693862915, + "logits/rejected": -2.322361469268799, + "logps/chosen": -202.0149688720703, + "logps/rejected": -243.12847900390625, + "loss": 0.1414, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1048834323883057, + "rewards/margins": 4.179681301116943, + "rewards/rejected": -5.284564971923828, + "step": 1049 + }, + { + "epoch": 1.37, + "learning_rate": 4.806697914976116e-05, + "logits/chosen": -2.4315245151519775, + "logits/rejected": -2.420684337615967, + "logps/chosen": -206.34152221679688, + "logps/rejected": -244.23727416992188, + "loss": 0.171, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.3189445734024048, + "rewards/margins": 3.540530204772949, + "rewards/rejected": -4.859475135803223, + "step": 1050 + }, + { + "epoch": 1.38, + "learning_rate": 4.806296090251236e-05, + "logits/chosen": -2.0065677165985107, + "logits/rejected": -2.1579909324645996, + "logps/chosen": -167.84841918945312, + "logps/rejected": -246.53677368164062, + "loss": 0.1576, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1663308143615723, + "rewards/margins": 3.5337471961975098, + "rewards/rejected": -4.70007848739624, + "step": 1051 + }, + { + "epoch": 1.38, + "learning_rate": 4.805893865146601e-05, + "logits/chosen": -2.1749401092529297, + "logits/rejected": -2.2115371227264404, + "logps/chosen": -235.0357208251953, + "logps/rejected": -288.3175964355469, + "loss": 0.1302, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5622204542160034, + "rewards/margins": 3.5101852416992188, + "rewards/rejected": -5.0724053382873535, + "step": 1052 + }, + { + "epoch": 1.38, + "learning_rate": 4.805491239732037e-05, + "logits/chosen": -2.3291127681732178, + "logits/rejected": -2.414362668991089, + "logps/chosen": -173.30560302734375, + "logps/rejected": -301.0763854980469, + "loss": 0.0217, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7221314907073975, + "rewards/margins": 6.185711860656738, + "rewards/rejected": -7.907843589782715, + "step": 1053 + }, + { + "epoch": 1.38, + "learning_rate": 4.8050882140774425e-05, + "logits/chosen": -2.316331624984741, + "logits/rejected": -2.383364677429199, + "logps/chosen": -177.23760986328125, + "logps/rejected": -202.95751953125, + "loss": 0.2219, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.775866985321045, + "rewards/margins": 2.098893404006958, + "rewards/rejected": -3.874760150909424, + "step": 1054 + }, + { + "epoch": 1.38, + "learning_rate": 4.8046847882527826e-05, + "logits/chosen": -2.2674598693847656, + "logits/rejected": -2.2072408199310303, + "logps/chosen": -173.46591186523438, + "logps/rejected": -221.97886657714844, + "loss": 0.1288, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.116098403930664, + "rewards/margins": 3.485488176345825, + "rewards/rejected": -4.60158634185791, + "step": 1055 + }, + { + "epoch": 1.38, + "learning_rate": 4.8042809623280946e-05, + "logits/chosen": -2.2563393115997314, + "logits/rejected": -2.3027093410491943, + "logps/chosen": -178.65032958984375, + "logps/rejected": -210.56443786621094, + "loss": 0.2929, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6832330226898193, + "rewards/margins": 3.0855979919433594, + "rewards/rejected": -4.768831253051758, + "step": 1056 + }, + { + "epoch": 1.38, + "learning_rate": 4.803876736373483e-05, + "logits/chosen": -2.36661958694458, + "logits/rejected": -2.373730421066284, + "logps/chosen": -216.2982940673828, + "logps/rejected": -270.1596374511719, + "loss": 0.2312, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5078752040863037, + "rewards/margins": 3.428542137145996, + "rewards/rejected": -4.936417579650879, + "step": 1057 + }, + { + "epoch": 1.38, + "learning_rate": 4.8034721104591215e-05, + "logits/chosen": -2.025160074234009, + "logits/rejected": -2.0265681743621826, + "logps/chosen": -213.1846160888672, + "logps/rejected": -244.3919677734375, + "loss": 0.3887, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.756600022315979, + "rewards/margins": 3.3049325942993164, + "rewards/rejected": -5.061532497406006, + "step": 1058 + }, + { + "epoch": 1.39, + "learning_rate": 4.803067084655257e-05, + "logits/chosen": -2.212038278579712, + "logits/rejected": -2.2199130058288574, + "logps/chosen": -200.26730346679688, + "logps/rejected": -226.02320861816406, + "loss": 0.1888, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1899981498718262, + "rewards/margins": 3.551909923553467, + "rewards/rejected": -4.741908073425293, + "step": 1059 + }, + { + "epoch": 1.39, + "learning_rate": 4.802661659032202e-05, + "logits/chosen": -2.1756439208984375, + "logits/rejected": -2.2646827697753906, + "logps/chosen": -170.65829467773438, + "logps/rejected": -227.00344848632812, + "loss": 0.2532, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.4983477592468262, + "rewards/margins": 2.899458408355713, + "rewards/rejected": -4.397806167602539, + "step": 1060 + }, + { + "epoch": 1.39, + "learning_rate": 4.802255833660338e-05, + "logits/chosen": -2.0890917778015137, + "logits/rejected": -2.20941424369812, + "logps/chosen": -194.01486206054688, + "logps/rejected": -247.7682342529297, + "loss": 0.1442, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1267577409744263, + "rewards/margins": 4.959884166717529, + "rewards/rejected": -6.086641788482666, + "step": 1061 + }, + { + "epoch": 1.39, + "learning_rate": 4.8018496086101194e-05, + "logits/chosen": -2.4077579975128174, + "logits/rejected": -2.5162241458892822, + "logps/chosen": -242.92034912109375, + "logps/rejected": -266.2312316894531, + "loss": 0.1722, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1819337606430054, + "rewards/margins": 3.8653903007507324, + "rewards/rejected": -5.047324180603027, + "step": 1062 + }, + { + "epoch": 1.39, + "learning_rate": 4.801442983952067e-05, + "logits/chosen": -2.206264019012451, + "logits/rejected": -2.329880952835083, + "logps/chosen": -178.778076171875, + "logps/rejected": -243.94203186035156, + "loss": 0.2205, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.6328147649765015, + "rewards/margins": 3.6644177436828613, + "rewards/rejected": -5.297232627868652, + "step": 1063 + }, + { + "epoch": 1.39, + "learning_rate": 4.8010359597567736e-05, + "logits/chosen": -2.256436347961426, + "logits/rejected": -2.3086161613464355, + "logps/chosen": -194.04498291015625, + "logps/rejected": -268.311279296875, + "loss": 0.1495, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.80282062292099, + "rewards/margins": 4.107694625854492, + "rewards/rejected": -4.910515308380127, + "step": 1064 + }, + { + "epoch": 1.39, + "learning_rate": 4.8006285360948976e-05, + "logits/chosen": -2.2415833473205566, + "logits/rejected": -2.3424930572509766, + "logps/chosen": -214.00341796875, + "logps/rejected": -264.5972900390625, + "loss": 0.1117, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2024343013763428, + "rewards/margins": 3.891465425491333, + "rewards/rejected": -5.093900203704834, + "step": 1065 + }, + { + "epoch": 1.4, + "learning_rate": 4.8002207130371705e-05, + "logits/chosen": -2.282310724258423, + "logits/rejected": -2.3069472312927246, + "logps/chosen": -175.22569274902344, + "logps/rejected": -244.1259307861328, + "loss": 0.1882, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2420991659164429, + "rewards/margins": 3.709995985031128, + "rewards/rejected": -4.9520955085754395, + "step": 1066 + }, + { + "epoch": 1.4, + "learning_rate": 4.7998124906543906e-05, + "logits/chosen": -2.2816646099090576, + "logits/rejected": -2.3789501190185547, + "logps/chosen": -186.17849731445312, + "logps/rejected": -252.48399353027344, + "loss": 0.226, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.2462586164474487, + "rewards/margins": 3.481846332550049, + "rewards/rejected": -4.728104591369629, + "step": 1067 + }, + { + "epoch": 1.4, + "learning_rate": 4.799403869017427e-05, + "logits/chosen": -2.060042142868042, + "logits/rejected": -2.231752395629883, + "logps/chosen": -158.0439910888672, + "logps/rejected": -194.29510498046875, + "loss": 0.1857, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.7000864744186401, + "rewards/margins": 3.765867233276367, + "rewards/rejected": -4.465953826904297, + "step": 1068 + }, + { + "epoch": 1.4, + "learning_rate": 4.798994848197218e-05, + "logits/chosen": -2.6301236152648926, + "logits/rejected": -2.6716156005859375, + "logps/chosen": -189.2099151611328, + "logps/rejected": -236.8180694580078, + "loss": 0.3166, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7804349660873413, + "rewards/margins": 2.868950366973877, + "rewards/rejected": -4.649385929107666, + "step": 1069 + }, + { + "epoch": 1.4, + "learning_rate": 4.79858542826477e-05, + "logits/chosen": -2.251466751098633, + "logits/rejected": -2.363938808441162, + "logps/chosen": -189.79736328125, + "logps/rejected": -241.90504455566406, + "loss": 0.2008, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2981109619140625, + "rewards/margins": 3.1607394218444824, + "rewards/rejected": -4.458850860595703, + "step": 1070 + }, + { + "epoch": 1.4, + "learning_rate": 4.798175609291161e-05, + "logits/chosen": -2.4197938442230225, + "logits/rejected": -2.3917999267578125, + "logps/chosen": -212.80386352539062, + "logps/rejected": -250.47430419921875, + "loss": 0.2364, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.001808524131775, + "rewards/margins": 3.6702208518981934, + "rewards/rejected": -4.6720290184021, + "step": 1071 + }, + { + "epoch": 1.4, + "learning_rate": 4.797765391347534e-05, + "logits/chosen": -2.4398419857025146, + "logits/rejected": -2.46700119972229, + "logps/chosen": -183.11013793945312, + "logps/rejected": -220.2843780517578, + "loss": 0.1893, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.9826763272285461, + "rewards/margins": 3.509796142578125, + "rewards/rejected": -4.4924726486206055, + "step": 1072 + }, + { + "epoch": 1.4, + "learning_rate": 4.7973547745051074e-05, + "logits/chosen": -2.4978957176208496, + "logits/rejected": -2.5766334533691406, + "logps/chosen": -219.82437133789062, + "logps/rejected": -293.62359619140625, + "loss": 0.146, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2996751070022583, + "rewards/margins": 3.58370304107666, + "rewards/rejected": -4.883378028869629, + "step": 1073 + }, + { + "epoch": 1.41, + "learning_rate": 4.796943758835163e-05, + "logits/chosen": -2.3775582313537598, + "logits/rejected": -2.4787142276763916, + "logps/chosen": -211.21597290039062, + "logps/rejected": -254.6476287841797, + "loss": 0.0379, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7991434335708618, + "rewards/margins": 4.452718734741211, + "rewards/rejected": -5.251862049102783, + "step": 1074 + }, + { + "epoch": 1.41, + "learning_rate": 4.796532344409055e-05, + "logits/chosen": -2.4883899688720703, + "logits/rejected": -2.5110130310058594, + "logps/chosen": -183.7747802734375, + "logps/rejected": -197.6341552734375, + "loss": 0.1991, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0323280096054077, + "rewards/margins": 2.9341001510620117, + "rewards/rejected": -3.966428279876709, + "step": 1075 + }, + { + "epoch": 1.41, + "learning_rate": 4.796120531298206e-05, + "logits/chosen": -2.338197946548462, + "logits/rejected": -2.2645106315612793, + "logps/chosen": -201.51806640625, + "logps/rejected": -266.6871032714844, + "loss": 0.1477, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9344640374183655, + "rewards/margins": 3.64399790763855, + "rewards/rejected": -4.57846212387085, + "step": 1076 + }, + { + "epoch": 1.41, + "learning_rate": 4.795708319574109e-05, + "logits/chosen": -2.5567445755004883, + "logits/rejected": -2.5801315307617188, + "logps/chosen": -264.55853271484375, + "logps/rejected": -273.15447998046875, + "loss": 0.1021, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.982978343963623, + "rewards/margins": 4.4724040031433105, + "rewards/rejected": -5.455382823944092, + "step": 1077 + }, + { + "epoch": 1.41, + "learning_rate": 4.795295709308324e-05, + "logits/chosen": -2.221895694732666, + "logits/rejected": -2.1924304962158203, + "logps/chosen": -199.84429931640625, + "logps/rejected": -275.3081359863281, + "loss": 0.1031, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0770325660705566, + "rewards/margins": 4.073465347290039, + "rewards/rejected": -5.150498390197754, + "step": 1078 + }, + { + "epoch": 1.41, + "learning_rate": 4.7948827005724816e-05, + "logits/chosen": -2.3237662315368652, + "logits/rejected": -2.366284132003784, + "logps/chosen": -159.32749938964844, + "logps/rejected": -198.11053466796875, + "loss": 0.2565, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.221225380897522, + "rewards/margins": 2.5992751121520996, + "rewards/rejected": -3.820500373840332, + "step": 1079 + }, + { + "epoch": 1.41, + "learning_rate": 4.794469293438282e-05, + "logits/chosen": -2.443016767501831, + "logits/rejected": -2.443124771118164, + "logps/chosen": -185.0653076171875, + "logps/rejected": -246.856689453125, + "loss": 0.1283, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7620292901992798, + "rewards/margins": 3.877666711807251, + "rewards/rejected": -4.639695644378662, + "step": 1080 + }, + { + "epoch": 1.41, + "learning_rate": 4.7940554879774925e-05, + "logits/chosen": -2.181255578994751, + "logits/rejected": -2.305636167526245, + "logps/chosen": -266.7353210449219, + "logps/rejected": -283.93853759765625, + "loss": 0.2479, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.8880407810211182, + "rewards/margins": 3.7701759338378906, + "rewards/rejected": -4.65821647644043, + "step": 1081 + }, + { + "epoch": 1.42, + "learning_rate": 4.793641284261953e-05, + "logits/chosen": -2.369758367538452, + "logits/rejected": -2.4988489151000977, + "logps/chosen": -183.67445373535156, + "logps/rejected": -233.878173828125, + "loss": 0.1349, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2128649950027466, + "rewards/margins": 3.4675493240356445, + "rewards/rejected": -4.680413722991943, + "step": 1082 + }, + { + "epoch": 1.42, + "learning_rate": 4.793226682363568e-05, + "logits/chosen": -2.573646068572998, + "logits/rejected": -2.576871395111084, + "logps/chosen": -166.67950439453125, + "logps/rejected": -206.69732666015625, + "loss": 0.2356, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.9300564527511597, + "rewards/margins": 3.6901583671569824, + "rewards/rejected": -4.620214939117432, + "step": 1083 + }, + { + "epoch": 1.42, + "learning_rate": 4.7928116823543155e-05, + "logits/chosen": -2.3677217960357666, + "logits/rejected": -2.405235528945923, + "logps/chosen": -187.89686584472656, + "logps/rejected": -253.44918823242188, + "loss": 0.1928, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2416791915893555, + "rewards/margins": 3.303443670272827, + "rewards/rejected": -4.545123100280762, + "step": 1084 + }, + { + "epoch": 1.42, + "learning_rate": 4.79239628430624e-05, + "logits/chosen": -2.4083261489868164, + "logits/rejected": -2.4456753730773926, + "logps/chosen": -216.77603149414062, + "logps/rejected": -276.124755859375, + "loss": 0.0495, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0237876176834106, + "rewards/margins": 4.667983531951904, + "rewards/rejected": -5.691771030426025, + "step": 1085 + }, + { + "epoch": 1.42, + "learning_rate": 4.791980488291456e-05, + "logits/chosen": -2.082496404647827, + "logits/rejected": -2.0862114429473877, + "logps/chosen": -197.0124969482422, + "logps/rejected": -200.89968872070312, + "loss": 0.1189, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8787529468536377, + "rewards/margins": 3.3207461833953857, + "rewards/rejected": -4.199499130249023, + "step": 1086 + }, + { + "epoch": 1.42, + "learning_rate": 4.791564294382147e-05, + "logits/chosen": -2.251490831375122, + "logits/rejected": -2.3277134895324707, + "logps/chosen": -176.1337432861328, + "logps/rejected": -240.8201904296875, + "loss": 0.112, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4940316677093506, + "rewards/margins": 4.046961784362793, + "rewards/rejected": -4.5409932136535645, + "step": 1087 + }, + { + "epoch": 1.42, + "learning_rate": 4.7911477026505654e-05, + "logits/chosen": -2.273149251937866, + "logits/rejected": -2.302190065383911, + "logps/chosen": -198.4782257080078, + "logps/rejected": -251.56442260742188, + "loss": 0.1584, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0771201848983765, + "rewards/margins": 4.303647994995117, + "rewards/rejected": -5.380767822265625, + "step": 1088 + }, + { + "epoch": 1.43, + "learning_rate": 4.790730713169033e-05, + "logits/chosen": -2.35538649559021, + "logits/rejected": -2.484128475189209, + "logps/chosen": -200.436767578125, + "logps/rejected": -264.001708984375, + "loss": 0.088, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3317238092422485, + "rewards/margins": 3.762491226196289, + "rewards/rejected": -5.094214916229248, + "step": 1089 + }, + { + "epoch": 1.43, + "learning_rate": 4.7903133260099385e-05, + "logits/chosen": -2.3187928199768066, + "logits/rejected": -2.248053550720215, + "logps/chosen": -269.09881591796875, + "logps/rejected": -317.9813232421875, + "loss": 0.1617, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5169427394866943, + "rewards/margins": 4.865277290344238, + "rewards/rejected": -6.382220268249512, + "step": 1090 + }, + { + "epoch": 1.43, + "learning_rate": 4.789895541245745e-05, + "logits/chosen": -2.2053849697113037, + "logits/rejected": -2.249541759490967, + "logps/chosen": -211.03778076171875, + "logps/rejected": -254.56468200683594, + "loss": 0.1818, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5010100603103638, + "rewards/margins": 3.9072840213775635, + "rewards/rejected": -5.408294200897217, + "step": 1091 + }, + { + "epoch": 1.43, + "learning_rate": 4.7894773589489775e-05, + "logits/chosen": -2.25164794921875, + "logits/rejected": -2.364285945892334, + "logps/chosen": -188.2033233642578, + "logps/rejected": -213.60580444335938, + "loss": 0.1675, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2163310050964355, + "rewards/margins": 3.9235103130340576, + "rewards/rejected": -5.139841079711914, + "step": 1092 + }, + { + "epoch": 1.43, + "learning_rate": 4.7890587791922364e-05, + "logits/chosen": -2.3585565090179443, + "logits/rejected": -2.3559505939483643, + "logps/chosen": -183.8409423828125, + "logps/rejected": -233.60438537597656, + "loss": 0.2356, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3878989219665527, + "rewards/margins": 2.798985719680786, + "rewards/rejected": -4.186884880065918, + "step": 1093 + }, + { + "epoch": 1.43, + "learning_rate": 4.788639802048187e-05, + "logits/chosen": -2.4514660835266113, + "logits/rejected": -2.468566417694092, + "logps/chosen": -189.787841796875, + "logps/rejected": -232.9893798828125, + "loss": 0.1471, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1803812980651855, + "rewards/margins": 3.827017307281494, + "rewards/rejected": -5.00739860534668, + "step": 1094 + }, + { + "epoch": 1.43, + "learning_rate": 4.788220427589566e-05, + "logits/chosen": -2.278841257095337, + "logits/rejected": -2.3293325901031494, + "logps/chosen": -177.0409393310547, + "logps/rejected": -221.64451599121094, + "loss": 0.2064, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.7956516742706299, + "rewards/margins": 4.03370475769043, + "rewards/rejected": -4.8293561935424805, + "step": 1095 + }, + { + "epoch": 1.43, + "learning_rate": 4.787800655889176e-05, + "logits/chosen": -2.2903358936309814, + "logits/rejected": -2.3189077377319336, + "logps/chosen": -185.59381103515625, + "logps/rejected": -257.0823059082031, + "loss": 0.178, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8907773494720459, + "rewards/margins": 4.005603790283203, + "rewards/rejected": -4.896381378173828, + "step": 1096 + }, + { + "epoch": 1.44, + "learning_rate": 4.787380487019893e-05, + "logits/chosen": -2.3953933715820312, + "logits/rejected": -2.450338363647461, + "logps/chosen": -179.10519409179688, + "logps/rejected": -219.1466522216797, + "loss": 0.1598, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0823649168014526, + "rewards/margins": 3.8283493518829346, + "rewards/rejected": -4.910714149475098, + "step": 1097 + }, + { + "epoch": 1.44, + "learning_rate": 4.786959921054659e-05, + "logits/chosen": -2.3358683586120605, + "logits/rejected": -2.2801101207733154, + "logps/chosen": -209.03997802734375, + "logps/rejected": -293.61419677734375, + "loss": 0.1063, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2742486000061035, + "rewards/margins": 3.9310548305511475, + "rewards/rejected": -5.205303192138672, + "step": 1098 + }, + { + "epoch": 1.44, + "learning_rate": 4.7865389580664844e-05, + "logits/chosen": -2.279576301574707, + "logits/rejected": -2.350735664367676, + "logps/chosen": -146.66612243652344, + "logps/rejected": -216.35707092285156, + "loss": 0.1297, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8987219929695129, + "rewards/margins": 3.8822243213653564, + "rewards/rejected": -4.780946254730225, + "step": 1099 + }, + { + "epoch": 1.44, + "learning_rate": 4.78611759812845e-05, + "logits/chosen": -2.109327554702759, + "logits/rejected": -2.0962979793548584, + "logps/chosen": -180.84161376953125, + "logps/rejected": -209.8188018798828, + "loss": 0.3155, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.6152057647705078, + "rewards/margins": 3.317700147628784, + "rewards/rejected": -3.932906150817871, + "step": 1100 + }, + { + "epoch": 1.44, + "learning_rate": 4.785695841313706e-05, + "logits/chosen": -2.3722269535064697, + "logits/rejected": -2.3200840950012207, + "logps/chosen": -201.63294982910156, + "logps/rejected": -254.41258239746094, + "loss": 0.1841, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.35667884349823, + "rewards/margins": 4.864499092102051, + "rewards/rejected": -6.2211785316467285, + "step": 1101 + }, + { + "epoch": 1.44, + "learning_rate": 4.785273687695469e-05, + "logits/chosen": -2.3536105155944824, + "logits/rejected": -2.4441490173339844, + "logps/chosen": -174.35108947753906, + "logps/rejected": -237.75894165039062, + "loss": 0.116, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.450251579284668, + "rewards/margins": 4.218486785888672, + "rewards/rejected": -5.66873836517334, + "step": 1102 + }, + { + "epoch": 1.44, + "learning_rate": 4.784851137347028e-05, + "logits/chosen": -2.1573073863983154, + "logits/rejected": -2.2236359119415283, + "logps/chosen": -198.5815887451172, + "logps/rejected": -232.4602813720703, + "loss": 0.1035, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0396865606307983, + "rewards/margins": 3.5235390663146973, + "rewards/rejected": -4.563225269317627, + "step": 1103 + }, + { + "epoch": 1.44, + "learning_rate": 4.7844281903417376e-05, + "logits/chosen": -2.0304176807403564, + "logits/rejected": -2.100639820098877, + "logps/chosen": -198.72247314453125, + "logps/rejected": -221.755615234375, + "loss": 0.2035, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3415790796279907, + "rewards/margins": 2.7209715843200684, + "rewards/rejected": -4.0625505447387695, + "step": 1104 + }, + { + "epoch": 1.45, + "learning_rate": 4.784004846753023e-05, + "logits/chosen": -2.074061393737793, + "logits/rejected": -2.121853828430176, + "logps/chosen": -166.87387084960938, + "logps/rejected": -207.54183959960938, + "loss": 0.2033, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.6906567215919495, + "rewards/margins": 4.180294513702393, + "rewards/rejected": -4.870951175689697, + "step": 1105 + }, + { + "epoch": 1.45, + "learning_rate": 4.783581106654378e-05, + "logits/chosen": -2.173704147338867, + "logits/rejected": -2.194322109222412, + "logps/chosen": -160.7693328857422, + "logps/rejected": -202.16580200195312, + "loss": 0.2204, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.3304364681243896, + "rewards/margins": 2.7709617614746094, + "rewards/rejected": -4.101398468017578, + "step": 1106 + }, + { + "epoch": 1.45, + "learning_rate": 4.783156970119366e-05, + "logits/chosen": -1.8855363130569458, + "logits/rejected": -1.914825677871704, + "logps/chosen": -188.28408813476562, + "logps/rejected": -240.01547241210938, + "loss": 0.2478, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0375070571899414, + "rewards/margins": 3.5554230213165283, + "rewards/rejected": -4.592930793762207, + "step": 1107 + }, + { + "epoch": 1.45, + "learning_rate": 4.782732437221616e-05, + "logits/chosen": -2.5382192134857178, + "logits/rejected": -2.4349591732025146, + "logps/chosen": -257.2680969238281, + "logps/rejected": -267.1951904296875, + "loss": 0.1059, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0496553182601929, + "rewards/margins": 4.491069316864014, + "rewards/rejected": -5.540725231170654, + "step": 1108 + }, + { + "epoch": 1.45, + "learning_rate": 4.78230750803483e-05, + "logits/chosen": -2.325007200241089, + "logits/rejected": -2.4198029041290283, + "logps/chosen": -162.60813903808594, + "logps/rejected": -221.90960693359375, + "loss": 0.1349, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.49094444513320923, + "rewards/margins": 3.7173869609832764, + "rewards/rejected": -4.208331108093262, + "step": 1109 + }, + { + "epoch": 1.45, + "learning_rate": 4.781882182632776e-05, + "logits/chosen": -2.475273370742798, + "logits/rejected": -2.397214412689209, + "logps/chosen": -206.89404296875, + "logps/rejected": -258.7071838378906, + "loss": 0.1856, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.805166482925415, + "rewards/margins": 4.610111713409424, + "rewards/rejected": -5.415278434753418, + "step": 1110 + }, + { + "epoch": 1.45, + "learning_rate": 4.781456461089294e-05, + "logits/chosen": -2.5161237716674805, + "logits/rejected": -2.437736749649048, + "logps/chosen": -248.8204345703125, + "logps/rejected": -275.9759216308594, + "loss": 0.1836, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.067987322807312, + "rewards/margins": 4.009271144866943, + "rewards/rejected": -5.0772576332092285, + "step": 1111 + }, + { + "epoch": 1.46, + "learning_rate": 4.781030343478288e-05, + "logits/chosen": -2.2793309688568115, + "logits/rejected": -2.3616416454315186, + "logps/chosen": -213.34039306640625, + "logps/rejected": -262.52398681640625, + "loss": 0.1304, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2378281354904175, + "rewards/margins": 4.064054489135742, + "rewards/rejected": -5.301882743835449, + "step": 1112 + }, + { + "epoch": 1.46, + "learning_rate": 4.780603829873733e-05, + "logits/chosen": -2.2147018909454346, + "logits/rejected": -2.4757885932922363, + "logps/chosen": -275.67681884765625, + "logps/rejected": -306.8128662109375, + "loss": 0.1641, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.911482810974121, + "rewards/margins": 4.229495048522949, + "rewards/rejected": -6.14097785949707, + "step": 1113 + }, + { + "epoch": 1.46, + "learning_rate": 4.780176920349675e-05, + "logits/chosen": -2.2692694664001465, + "logits/rejected": -2.4222545623779297, + "logps/chosen": -181.1542510986328, + "logps/rejected": -243.48069763183594, + "loss": 0.1874, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.4185149669647217, + "rewards/margins": 2.8215103149414062, + "rewards/rejected": -4.240025520324707, + "step": 1114 + }, + { + "epoch": 1.46, + "learning_rate": 4.7797496149802256e-05, + "logits/chosen": -2.2589826583862305, + "logits/rejected": -2.2955431938171387, + "logps/chosen": -163.7628173828125, + "logps/rejected": -257.77764892578125, + "loss": 0.1802, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0072280168533325, + "rewards/margins": 4.691400527954102, + "rewards/rejected": -5.6986284255981445, + "step": 1115 + }, + { + "epoch": 1.46, + "learning_rate": 4.779321913839566e-05, + "logits/chosen": -2.281262159347534, + "logits/rejected": -2.2819738388061523, + "logps/chosen": -233.2173309326172, + "logps/rejected": -272.6235656738281, + "loss": 0.1662, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.1393500566482544, + "rewards/margins": 3.8536875247955322, + "rewards/rejected": -4.993037700653076, + "step": 1116 + }, + { + "epoch": 1.46, + "learning_rate": 4.778893817001948e-05, + "logits/chosen": -2.2208681106567383, + "logits/rejected": -2.1590828895568848, + "logps/chosen": -179.7089080810547, + "logps/rejected": -216.29774475097656, + "loss": 0.1497, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.060105323791504, + "rewards/margins": 3.382988214492798, + "rewards/rejected": -4.443093776702881, + "step": 1117 + }, + { + "epoch": 1.46, + "learning_rate": 4.7784653245416875e-05, + "logits/chosen": -2.1777501106262207, + "logits/rejected": -2.2831733226776123, + "logps/chosen": -191.6259307861328, + "logps/rejected": -239.21493530273438, + "loss": 0.1255, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.195249080657959, + "rewards/margins": 3.476778507232666, + "rewards/rejected": -4.672027587890625, + "step": 1118 + }, + { + "epoch": 1.46, + "learning_rate": 4.778036436533176e-05, + "logits/chosen": -2.345694065093994, + "logits/rejected": -2.400123357772827, + "logps/chosen": -150.64157104492188, + "logps/rejected": -217.12583923339844, + "loss": 0.1818, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1695704460144043, + "rewards/margins": 3.341207504272461, + "rewards/rejected": -4.510777950286865, + "step": 1119 + }, + { + "epoch": 1.47, + "learning_rate": 4.777607153050866e-05, + "logits/chosen": -2.27506422996521, + "logits/rejected": -2.3480772972106934, + "logps/chosen": -226.47450256347656, + "logps/rejected": -255.78909301757812, + "loss": 0.1828, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.286346673965454, + "rewards/margins": 3.472963333129883, + "rewards/rejected": -4.759309768676758, + "step": 1120 + }, + { + "epoch": 1.47, + "learning_rate": 4.7771774741692844e-05, + "logits/chosen": -2.4806315898895264, + "logits/rejected": -2.420682430267334, + "logps/chosen": -182.59005737304688, + "logps/rejected": -220.8089599609375, + "loss": 0.1512, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.9431337118148804, + "rewards/margins": 3.128722906112671, + "rewards/rejected": -4.071856498718262, + "step": 1121 + }, + { + "epoch": 1.47, + "learning_rate": 4.776747399963024e-05, + "logits/chosen": -2.4254984855651855, + "logits/rejected": -2.4268956184387207, + "logps/chosen": -181.3052215576172, + "logps/rejected": -244.41795349121094, + "loss": 0.1868, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.8446401953697205, + "rewards/margins": 3.5034520626068115, + "rewards/rejected": -4.348092555999756, + "step": 1122 + }, + { + "epoch": 1.47, + "learning_rate": 4.776316930506747e-05, + "logits/chosen": -2.3528339862823486, + "logits/rejected": -2.518993616104126, + "logps/chosen": -204.74118041992188, + "logps/rejected": -297.0326843261719, + "loss": 0.0927, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9256747364997864, + "rewards/margins": 4.606964111328125, + "rewards/rejected": -5.5326385498046875, + "step": 1123 + }, + { + "epoch": 1.47, + "learning_rate": 4.775886065875185e-05, + "logits/chosen": -2.428316593170166, + "logits/rejected": -2.4729697704315186, + "logps/chosen": -192.39427185058594, + "logps/rejected": -225.5529327392578, + "loss": 0.1996, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1067038774490356, + "rewards/margins": 2.738870620727539, + "rewards/rejected": -3.845574378967285, + "step": 1124 + }, + { + "epoch": 1.47, + "learning_rate": 4.775454806143137e-05, + "logits/chosen": -2.3744990825653076, + "logits/rejected": -2.2757740020751953, + "logps/chosen": -174.80218505859375, + "logps/rejected": -327.66351318359375, + "loss": 0.0678, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8998466730117798, + "rewards/margins": 4.191055774688721, + "rewards/rejected": -5.090902805328369, + "step": 1125 + }, + { + "epoch": 1.47, + "learning_rate": 4.775023151385469e-05, + "logits/chosen": -2.5411629676818848, + "logits/rejected": -2.5592217445373535, + "logps/chosen": -221.6334228515625, + "logps/rejected": -263.060546875, + "loss": 0.1147, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.669825553894043, + "rewards/margins": 3.485670804977417, + "rewards/rejected": -5.155496597290039, + "step": 1126 + }, + { + "epoch": 1.47, + "learning_rate": 4.774591101677121e-05, + "logits/chosen": -2.322519540786743, + "logits/rejected": -2.3820464611053467, + "logps/chosen": -185.3223876953125, + "logps/rejected": -212.63223266601562, + "loss": 0.2022, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0957611799240112, + "rewards/margins": 2.9892659187316895, + "rewards/rejected": -4.085027694702148, + "step": 1127 + }, + { + "epoch": 1.48, + "learning_rate": 4.774158657093095e-05, + "logits/chosen": -2.4653425216674805, + "logits/rejected": -2.5697944164276123, + "logps/chosen": -164.1728973388672, + "logps/rejected": -223.85023498535156, + "loss": 0.1749, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.4075347185134888, + "rewards/margins": 3.1060991287231445, + "rewards/rejected": -4.513633728027344, + "step": 1128 + }, + { + "epoch": 1.48, + "learning_rate": 4.7737258177084665e-05, + "logits/chosen": -2.2479147911071777, + "logits/rejected": -2.271726369857788, + "logps/chosen": -226.23324584960938, + "logps/rejected": -316.6651306152344, + "loss": 0.0813, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.880527138710022, + "rewards/margins": 5.674856662750244, + "rewards/rejected": -6.555383682250977, + "step": 1129 + }, + { + "epoch": 1.48, + "learning_rate": 4.7732925835983775e-05, + "logits/chosen": -2.1202504634857178, + "logits/rejected": -2.141789436340332, + "logps/chosen": -218.8267822265625, + "logps/rejected": -285.7803039550781, + "loss": 0.1359, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.9516122341156006, + "rewards/margins": 4.8950653076171875, + "rewards/rejected": -5.846677780151367, + "step": 1130 + }, + { + "epoch": 1.48, + "learning_rate": 4.772858954838038e-05, + "logits/chosen": -2.4971561431884766, + "logits/rejected": -2.5220043659210205, + "logps/chosen": -226.0542755126953, + "logps/rejected": -299.23779296875, + "loss": 0.1074, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.871485710144043, + "rewards/margins": 5.10019588470459, + "rewards/rejected": -5.971682071685791, + "step": 1131 + }, + { + "epoch": 1.48, + "learning_rate": 4.772424931502727e-05, + "logits/chosen": -2.4149010181427, + "logits/rejected": -2.431765556335449, + "logps/chosen": -280.7022399902344, + "logps/rejected": -361.54388427734375, + "loss": 0.1054, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6145488023757935, + "rewards/margins": 3.9257521629333496, + "rewards/rejected": -5.5403008460998535, + "step": 1132 + }, + { + "epoch": 1.48, + "learning_rate": 4.771990513667793e-05, + "logits/chosen": -2.3434276580810547, + "logits/rejected": -2.3973376750946045, + "logps/chosen": -183.86436462402344, + "logps/rejected": -243.69293212890625, + "loss": 0.2449, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6164230108261108, + "rewards/margins": 2.5519909858703613, + "rewards/rejected": -4.168414115905762, + "step": 1133 + }, + { + "epoch": 1.48, + "learning_rate": 4.771555701408652e-05, + "logits/chosen": -2.5856552124023438, + "logits/rejected": -2.4191064834594727, + "logps/chosen": -237.94381713867188, + "logps/rejected": -232.526611328125, + "loss": 0.1426, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.6252250671386719, + "rewards/margins": 3.2301154136657715, + "rewards/rejected": -3.8553402423858643, + "step": 1134 + }, + { + "epoch": 1.49, + "learning_rate": 4.771120494800789e-05, + "logits/chosen": -2.196472644805908, + "logits/rejected": -2.1777446269989014, + "logps/chosen": -198.78469848632812, + "logps/rejected": -220.56103515625, + "loss": 0.2213, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.9573407173156738, + "rewards/margins": 3.4439077377319336, + "rewards/rejected": -4.401248455047607, + "step": 1135 + }, + { + "epoch": 1.49, + "learning_rate": 4.7706848939197565e-05, + "logits/chosen": -2.4557721614837646, + "logits/rejected": -2.4789040088653564, + "logps/chosen": -183.08238220214844, + "logps/rejected": -270.4992370605469, + "loss": 0.1262, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.5791445970535278, + "rewards/margins": 4.073019027709961, + "rewards/rejected": -4.652163982391357, + "step": 1136 + }, + { + "epoch": 1.49, + "learning_rate": 4.7702488988411765e-05, + "logits/chosen": -2.4277684688568115, + "logits/rejected": -2.4676928520202637, + "logps/chosen": -206.0131378173828, + "logps/rejected": -222.62411499023438, + "loss": 0.2436, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.7216240763664246, + "rewards/margins": 2.8460915088653564, + "rewards/rejected": -3.5677154064178467, + "step": 1137 + }, + { + "epoch": 1.49, + "learning_rate": 4.7698125096407384e-05, + "logits/chosen": -2.500568151473999, + "logits/rejected": -2.5246546268463135, + "logps/chosen": -237.37734985351562, + "logps/rejected": -282.74267578125, + "loss": 0.0789, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6298863887786865, + "rewards/margins": 4.078357219696045, + "rewards/rejected": -4.7082438468933105, + "step": 1138 + }, + { + "epoch": 1.49, + "learning_rate": 4.7693757263942015e-05, + "logits/chosen": -2.5762946605682373, + "logits/rejected": -2.543735980987549, + "logps/chosen": -240.41571044921875, + "logps/rejected": -255.56991577148438, + "loss": 0.0945, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8246884346008301, + "rewards/margins": 4.176553726196289, + "rewards/rejected": -5.001242637634277, + "step": 1139 + }, + { + "epoch": 1.49, + "learning_rate": 4.768938549177393e-05, + "logits/chosen": -2.426650047302246, + "logits/rejected": -2.5201663970947266, + "logps/chosen": -195.04554748535156, + "logps/rejected": -247.7749786376953, + "loss": 0.0827, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9046878814697266, + "rewards/margins": 3.637204170227051, + "rewards/rejected": -4.541892051696777, + "step": 1140 + }, + { + "epoch": 1.49, + "learning_rate": 4.7685009780662074e-05, + "logits/chosen": -2.114713191986084, + "logits/rejected": -2.0542666912078857, + "logps/chosen": -184.0598602294922, + "logps/rejected": -212.22396850585938, + "loss": 0.1166, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.9158424735069275, + "rewards/margins": 3.6370954513549805, + "rewards/rejected": -4.5529375076293945, + "step": 1141 + }, + { + "epoch": 1.49, + "learning_rate": 4.768063013136607e-05, + "logits/chosen": -2.494703769683838, + "logits/rejected": -2.525118112564087, + "logps/chosen": -232.44784545898438, + "logps/rejected": -251.73089599609375, + "loss": 0.1935, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.5020594596862793, + "rewards/margins": 2.5554111003875732, + "rewards/rejected": -4.057470321655273, + "step": 1142 + }, + { + "epoch": 1.5, + "learning_rate": 4.7676246544646266e-05, + "logits/chosen": -2.445399522781372, + "logits/rejected": -2.4102258682250977, + "logps/chosen": -188.2194061279297, + "logps/rejected": -230.07611083984375, + "loss": 0.1288, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.5976011753082275, + "rewards/margins": 3.648184299468994, + "rewards/rejected": -4.245785713195801, + "step": 1143 + }, + { + "epoch": 1.5, + "learning_rate": 4.767185902126364e-05, + "logits/chosen": -2.579176664352417, + "logits/rejected": -2.610398292541504, + "logps/chosen": -194.1246795654297, + "logps/rejected": -236.80584716796875, + "loss": 0.1657, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.4905986785888672, + "rewards/margins": 3.536527395248413, + "rewards/rejected": -5.027125835418701, + "step": 1144 + }, + { + "epoch": 1.5, + "learning_rate": 4.766746756197989e-05, + "logits/chosen": -2.4602108001708984, + "logits/rejected": -2.4495668411254883, + "logps/chosen": -234.55178833007812, + "logps/rejected": -269.4857177734375, + "loss": 0.3061, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.2820649147033691, + "rewards/margins": 2.805851936340332, + "rewards/rejected": -4.087916851043701, + "step": 1145 + }, + { + "epoch": 1.5, + "learning_rate": 4.766307216755739e-05, + "logits/chosen": -2.402568817138672, + "logits/rejected": -2.4603004455566406, + "logps/chosen": -184.5412139892578, + "logps/rejected": -232.99713134765625, + "loss": 0.1395, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6240134239196777, + "rewards/margins": 3.225022077560425, + "rewards/rejected": -4.849035263061523, + "step": 1146 + }, + { + "epoch": 1.5, + "learning_rate": 4.765867283875919e-05, + "logits/chosen": -2.3566904067993164, + "logits/rejected": -2.315061092376709, + "logps/chosen": -194.67453002929688, + "logps/rejected": -208.98086547851562, + "loss": 0.1716, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.9168694019317627, + "rewards/margins": 3.383730173110962, + "rewards/rejected": -4.300599575042725, + "step": 1147 + }, + { + "epoch": 1.5, + "learning_rate": 4.765426957634903e-05, + "logits/chosen": -2.288248062133789, + "logits/rejected": -2.3149735927581787, + "logps/chosen": -248.4288330078125, + "logps/rejected": -294.727294921875, + "loss": 0.0246, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7581281661987305, + "rewards/margins": 4.977880954742432, + "rewards/rejected": -5.73600959777832, + "step": 1148 + }, + { + "epoch": 1.5, + "learning_rate": 4.7649862381091326e-05, + "logits/chosen": -2.441120147705078, + "logits/rejected": -2.507535934448242, + "logps/chosen": -145.71258544921875, + "logps/rejected": -208.50909423828125, + "loss": 0.2425, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.5849244594573975, + "rewards/margins": 2.7225358486175537, + "rewards/rejected": -4.307460308074951, + "step": 1149 + }, + { + "epoch": 1.5, + "learning_rate": 4.764545125375117e-05, + "logits/chosen": -2.0023226737976074, + "logits/rejected": -2.0600125789642334, + "logps/chosen": -152.4337615966797, + "logps/rejected": -222.3336639404297, + "loss": 0.1372, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2324447631835938, + "rewards/margins": 3.39628267288208, + "rewards/rejected": -4.628727436065674, + "step": 1150 + }, + { + "epoch": 1.51, + "learning_rate": 4.764103619509436e-05, + "logits/chosen": -2.4556169509887695, + "logits/rejected": -2.412785530090332, + "logps/chosen": -199.9222869873047, + "logps/rejected": -212.75326538085938, + "loss": 0.28, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.5000001192092896, + "rewards/margins": 3.5534515380859375, + "rewards/rejected": -5.053452014923096, + "step": 1151 + }, + { + "epoch": 1.51, + "learning_rate": 4.763661720588736e-05, + "logits/chosen": -2.436502456665039, + "logits/rejected": -2.4594924449920654, + "logps/chosen": -209.86231994628906, + "logps/rejected": -230.40859985351562, + "loss": 0.1923, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5364642143249512, + "rewards/margins": 3.644205093383789, + "rewards/rejected": -5.18066930770874, + "step": 1152 + }, + { + "epoch": 1.51, + "learning_rate": 4.7632194286897315e-05, + "logits/chosen": -2.570007801055908, + "logits/rejected": -2.5704219341278076, + "logps/chosen": -221.38475036621094, + "logps/rejected": -304.7647705078125, + "loss": 0.1645, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2075974941253662, + "rewards/margins": 5.129228591918945, + "rewards/rejected": -6.336826324462891, + "step": 1153 + }, + { + "epoch": 1.51, + "learning_rate": 4.762776743889207e-05, + "logits/chosen": -2.6236629486083984, + "logits/rejected": -2.678802967071533, + "logps/chosen": -311.6545104980469, + "logps/rejected": -333.4806823730469, + "loss": 0.1045, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.6849138736724854, + "rewards/margins": 4.670904636383057, + "rewards/rejected": -6.355818271636963, + "step": 1154 + }, + { + "epoch": 1.51, + "learning_rate": 4.7623336662640116e-05, + "logits/chosen": -1.7820888757705688, + "logits/rejected": -1.8834837675094604, + "logps/chosen": -162.43157958984375, + "logps/rejected": -243.75802612304688, + "loss": 0.0676, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.156273365020752, + "rewards/margins": 3.5598320960998535, + "rewards/rejected": -5.716104984283447, + "step": 1155 + }, + { + "epoch": 1.51, + "learning_rate": 4.761890195891067e-05, + "logits/chosen": -2.580514907836914, + "logits/rejected": -2.583678722381592, + "logps/chosen": -227.94639587402344, + "logps/rejected": -269.7225341796875, + "loss": 0.0754, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.5052528381347656, + "rewards/margins": 4.785727024078369, + "rewards/rejected": -6.290980815887451, + "step": 1156 + }, + { + "epoch": 1.51, + "learning_rate": 4.76144633284736e-05, + "logits/chosen": -2.2309510707855225, + "logits/rejected": -2.2391128540039062, + "logps/chosen": -204.0966033935547, + "logps/rejected": -237.06265258789062, + "loss": 0.2103, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.803736925125122, + "rewards/margins": 2.9250359535217285, + "rewards/rejected": -5.728772163391113, + "step": 1157 + }, + { + "epoch": 1.52, + "learning_rate": 4.761002077209946e-05, + "logits/chosen": -2.3017187118530273, + "logits/rejected": -2.1956186294555664, + "logps/chosen": -204.7859344482422, + "logps/rejected": -202.06703186035156, + "loss": 0.1633, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.668914794921875, + "rewards/margins": 3.471302032470703, + "rewards/rejected": -5.140216827392578, + "step": 1158 + }, + { + "epoch": 1.52, + "learning_rate": 4.760557429055951e-05, + "logits/chosen": -2.2529184818267822, + "logits/rejected": -2.1692707538604736, + "logps/chosen": -202.95797729492188, + "logps/rejected": -222.89297485351562, + "loss": 0.2165, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.9888824224472046, + "rewards/margins": 3.733574390411377, + "rewards/rejected": -5.722457408905029, + "step": 1159 + }, + { + "epoch": 1.52, + "learning_rate": 4.760112388462564e-05, + "logits/chosen": -2.5051965713500977, + "logits/rejected": -2.461354970932007, + "logps/chosen": -207.01870727539062, + "logps/rejected": -233.17376708984375, + "loss": 0.2913, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.4785971641540527, + "rewards/margins": 3.71671199798584, + "rewards/rejected": -6.195309638977051, + "step": 1160 + }, + { + "epoch": 1.52, + "learning_rate": 4.759666955507049e-05, + "logits/chosen": -2.3772454261779785, + "logits/rejected": -2.3350820541381836, + "logps/chosen": -194.571533203125, + "logps/rejected": -220.87799072265625, + "loss": 0.1533, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.7106103897094727, + "rewards/margins": 3.6144776344299316, + "rewards/rejected": -5.325088024139404, + "step": 1161 + }, + { + "epoch": 1.52, + "learning_rate": 4.759221130266732e-05, + "logits/chosen": -2.2555718421936035, + "logits/rejected": -2.2726223468780518, + "logps/chosen": -175.2388153076172, + "logps/rejected": -250.49134826660156, + "loss": 0.091, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.260037422180176, + "rewards/margins": 4.190368175506592, + "rewards/rejected": -6.450405120849609, + "step": 1162 + }, + { + "epoch": 1.52, + "learning_rate": 4.758774912819011e-05, + "logits/chosen": -2.5848538875579834, + "logits/rejected": -2.6254186630249023, + "logps/chosen": -295.5448913574219, + "logps/rejected": -309.3749084472656, + "loss": 0.0571, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.9205611944198608, + "rewards/margins": 4.6831841468811035, + "rewards/rejected": -6.603745460510254, + "step": 1163 + }, + { + "epoch": 1.52, + "learning_rate": 4.758328303241349e-05, + "logits/chosen": -2.445176124572754, + "logits/rejected": -2.4309027194976807, + "logps/chosen": -228.26736450195312, + "logps/rejected": -228.83767700195312, + "loss": 0.1921, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.568021297454834, + "rewards/margins": 3.549888849258423, + "rewards/rejected": -6.117909908294678, + "step": 1164 + }, + { + "epoch": 1.52, + "learning_rate": 4.75788130161128e-05, + "logits/chosen": -2.2904882431030273, + "logits/rejected": -2.3325557708740234, + "logps/chosen": -225.65245056152344, + "logps/rejected": -264.3974304199219, + "loss": 0.141, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.110083818435669, + "rewards/margins": 4.307819366455078, + "rewards/rejected": -6.417903423309326, + "step": 1165 + }, + { + "epoch": 1.53, + "learning_rate": 4.7574339080064044e-05, + "logits/chosen": -2.248126268386841, + "logits/rejected": -2.4054722785949707, + "logps/chosen": -179.7675323486328, + "logps/rejected": -227.87307739257812, + "loss": 0.1646, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.8558435440063477, + "rewards/margins": 3.661419153213501, + "rewards/rejected": -5.5172624588012695, + "step": 1166 + }, + { + "epoch": 1.53, + "learning_rate": 4.756986122504392e-05, + "logits/chosen": -2.5831997394561768, + "logits/rejected": -2.652710199356079, + "logps/chosen": -187.14601135253906, + "logps/rejected": -252.51132202148438, + "loss": 0.0897, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.8020074367523193, + "rewards/margins": 4.59144401550293, + "rewards/rejected": -6.393451690673828, + "step": 1167 + }, + { + "epoch": 1.53, + "learning_rate": 4.756537945182978e-05, + "logits/chosen": -2.3583719730377197, + "logits/rejected": -2.4774858951568604, + "logps/chosen": -182.31289672851562, + "logps/rejected": -262.7928161621094, + "loss": 0.0801, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.851448655128479, + "rewards/margins": 4.432652950286865, + "rewards/rejected": -6.284101486206055, + "step": 1168 + }, + { + "epoch": 1.53, + "learning_rate": 4.7560893761199685e-05, + "logits/chosen": -2.4317312240600586, + "logits/rejected": -2.548471689224243, + "logps/chosen": -224.8054656982422, + "logps/rejected": -264.10919189453125, + "loss": 0.2068, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.833303213119507, + "rewards/margins": 3.181811809539795, + "rewards/rejected": -6.015115261077881, + "step": 1169 + }, + { + "epoch": 1.53, + "learning_rate": 4.7556404153932356e-05, + "logits/chosen": -2.501443386077881, + "logits/rejected": -2.562654495239258, + "logps/chosen": -186.91287231445312, + "logps/rejected": -231.56109619140625, + "loss": 0.0915, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.9190934896469116, + "rewards/margins": 4.339428424835205, + "rewards/rejected": -6.258522033691406, + "step": 1170 + }, + { + "epoch": 1.53, + "learning_rate": 4.755191063080721e-05, + "logits/chosen": -2.152017116546631, + "logits/rejected": -2.183455467224121, + "logps/chosen": -195.06324768066406, + "logps/rejected": -249.9405517578125, + "loss": 0.1957, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.957331657409668, + "rewards/margins": 3.673862934112549, + "rewards/rejected": -5.631194591522217, + "step": 1171 + }, + { + "epoch": 1.53, + "learning_rate": 4.754741319260433e-05, + "logits/chosen": -2.1391313076019287, + "logits/rejected": -2.259127616882324, + "logps/chosen": -218.9195098876953, + "logps/rejected": -249.30020141601562, + "loss": 0.1433, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.8016879558563232, + "rewards/margins": 4.185688018798828, + "rewards/rejected": -6.9873762130737305, + "step": 1172 + }, + { + "epoch": 1.54, + "learning_rate": 4.754291184010449e-05, + "logits/chosen": -2.3584299087524414, + "logits/rejected": -2.4219892024993896, + "logps/chosen": -250.45458984375, + "logps/rejected": -282.7275390625, + "loss": 0.1347, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.450329303741455, + "rewards/margins": 3.8380470275878906, + "rewards/rejected": -6.288376808166504, + "step": 1173 + }, + { + "epoch": 1.54, + "learning_rate": 4.753840657408913e-05, + "logits/chosen": -2.546412706375122, + "logits/rejected": -2.5415077209472656, + "logps/chosen": -218.136474609375, + "logps/rejected": -255.75633239746094, + "loss": 0.0751, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.4324724674224854, + "rewards/margins": 4.358306884765625, + "rewards/rejected": -5.790779113769531, + "step": 1174 + }, + { + "epoch": 1.54, + "learning_rate": 4.7533897395340384e-05, + "logits/chosen": -2.3359920978546143, + "logits/rejected": -2.2689504623413086, + "logps/chosen": -175.5142364501953, + "logps/rejected": -207.70046997070312, + "loss": 0.2638, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.7872010469436646, + "rewards/margins": 3.380298376083374, + "rewards/rejected": -5.167499542236328, + "step": 1175 + }, + { + "epoch": 1.54, + "learning_rate": 4.752938430464105e-05, + "logits/chosen": -2.5895543098449707, + "logits/rejected": -2.645479679107666, + "logps/chosen": -250.36102294921875, + "logps/rejected": -294.0989074707031, + "loss": 0.0946, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.3163928985595703, + "rewards/margins": 5.06205940246582, + "rewards/rejected": -6.378452301025391, + "step": 1176 + }, + { + "epoch": 1.54, + "learning_rate": 4.752486730277463e-05, + "logits/chosen": -1.9727551937103271, + "logits/rejected": -2.0490756034851074, + "logps/chosen": -183.3731231689453, + "logps/rejected": -266.77679443359375, + "loss": 0.2344, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9805545806884766, + "rewards/margins": 3.784672975540161, + "rewards/rejected": -5.765227317810059, + "step": 1177 + }, + { + "epoch": 1.54, + "learning_rate": 4.752034639052527e-05, + "logits/chosen": -2.3057782649993896, + "logits/rejected": -2.363804817199707, + "logps/chosen": -210.0028533935547, + "logps/rejected": -274.5212707519531, + "loss": 0.1186, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.943210244178772, + "rewards/margins": 4.820004463195801, + "rewards/rejected": -6.763214588165283, + "step": 1178 + }, + { + "epoch": 1.54, + "learning_rate": 4.751582156867783e-05, + "logits/chosen": -2.40464186668396, + "logits/rejected": -2.3199591636657715, + "logps/chosen": -238.06265258789062, + "logps/rejected": -262.56134033203125, + "loss": 0.2005, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6629893779754639, + "rewards/margins": 3.4837026596069336, + "rewards/rejected": -5.146691799163818, + "step": 1179 + }, + { + "epoch": 1.54, + "learning_rate": 4.751129283801782e-05, + "logits/chosen": -2.4328413009643555, + "logits/rejected": -2.546642780303955, + "logps/chosen": -248.2761993408203, + "logps/rejected": -308.90130615234375, + "loss": 0.0414, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.027224540710449, + "rewards/margins": 5.0925612449646, + "rewards/rejected": -7.119786262512207, + "step": 1180 + }, + { + "epoch": 1.55, + "learning_rate": 4.7506760199331445e-05, + "logits/chosen": -2.329103469848633, + "logits/rejected": -2.442920207977295, + "logps/chosen": -196.090087890625, + "logps/rejected": -244.14735412597656, + "loss": 0.133, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.5378122329711914, + "rewards/margins": 3.5418591499328613, + "rewards/rejected": -6.079671859741211, + "step": 1181 + }, + { + "epoch": 1.55, + "learning_rate": 4.750222365340559e-05, + "logits/chosen": -2.4741384983062744, + "logits/rejected": -2.5175294876098633, + "logps/chosen": -194.5352325439453, + "logps/rejected": -271.7839660644531, + "loss": 0.1235, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.9755550622940063, + "rewards/margins": 4.500469207763672, + "rewards/rejected": -6.476024627685547, + "step": 1182 + }, + { + "epoch": 1.55, + "learning_rate": 4.749768320102781e-05, + "logits/chosen": -2.535355806350708, + "logits/rejected": -2.596473217010498, + "logps/chosen": -246.87472534179688, + "logps/rejected": -301.79296875, + "loss": 0.0759, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.202014446258545, + "rewards/margins": 4.9576096534729, + "rewards/rejected": -6.159624099731445, + "step": 1183 + }, + { + "epoch": 1.55, + "learning_rate": 4.749313884298633e-05, + "logits/chosen": -2.47165846824646, + "logits/rejected": -2.4970316886901855, + "logps/chosen": -244.66204833984375, + "logps/rejected": -291.098388671875, + "loss": 0.213, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.790406584739685, + "rewards/margins": 5.0259270668029785, + "rewards/rejected": -6.816333770751953, + "step": 1184 + }, + { + "epoch": 1.55, + "learning_rate": 4.7488590580070074e-05, + "logits/chosen": -2.3476386070251465, + "logits/rejected": -2.4502997398376465, + "logps/chosen": -223.2727508544922, + "logps/rejected": -293.4349365234375, + "loss": 0.1489, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.024826765060425, + "rewards/margins": 3.9169485569000244, + "rewards/rejected": -5.941775321960449, + "step": 1185 + }, + { + "epoch": 1.55, + "learning_rate": 4.748403841306863e-05, + "logits/chosen": -2.0279314517974854, + "logits/rejected": -2.0650389194488525, + "logps/chosen": -163.14549255371094, + "logps/rejected": -210.16258239746094, + "loss": 0.4483, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0485494136810303, + "rewards/margins": 2.879338264465332, + "rewards/rejected": -4.927887916564941, + "step": 1186 + }, + { + "epoch": 1.55, + "learning_rate": 4.747948234277228e-05, + "logits/chosen": -2.5351710319519043, + "logits/rejected": -2.5304994583129883, + "logps/chosen": -267.7850036621094, + "logps/rejected": -258.6395263671875, + "loss": 0.1651, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.5308005809783936, + "rewards/margins": 4.033406734466553, + "rewards/rejected": -5.564207077026367, + "step": 1187 + }, + { + "epoch": 1.55, + "learning_rate": 4.747492236997195e-05, + "logits/chosen": -2.3023691177368164, + "logits/rejected": -2.3658742904663086, + "logps/chosen": -192.9929656982422, + "logps/rejected": -237.79534912109375, + "loss": 0.2237, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.313674807548523, + "rewards/margins": 3.4801218509674072, + "rewards/rejected": -4.793796539306641, + "step": 1188 + }, + { + "epoch": 1.56, + "learning_rate": 4.747035849545928e-05, + "logits/chosen": -2.5478949546813965, + "logits/rejected": -2.51932954788208, + "logps/chosen": -239.89755249023438, + "logps/rejected": -269.83544921875, + "loss": 0.0833, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7208430767059326, + "rewards/margins": 4.114684104919434, + "rewards/rejected": -5.835526943206787, + "step": 1189 + }, + { + "epoch": 1.56, + "learning_rate": 4.746579072002657e-05, + "logits/chosen": -2.2504239082336426, + "logits/rejected": -2.3190526962280273, + "logps/chosen": -224.220703125, + "logps/rejected": -260.7024841308594, + "loss": 0.3915, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0131428241729736, + "rewards/margins": 2.687302589416504, + "rewards/rejected": -4.700445652008057, + "step": 1190 + }, + { + "epoch": 1.56, + "learning_rate": 4.7461219044466795e-05, + "logits/chosen": -2.1805527210235596, + "logits/rejected": -2.193903684616089, + "logps/chosen": -184.85508728027344, + "logps/rejected": -234.97364807128906, + "loss": 0.2377, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.2396941184997559, + "rewards/margins": 3.461569309234619, + "rewards/rejected": -4.701263427734375, + "step": 1191 + }, + { + "epoch": 1.56, + "learning_rate": 4.745664346957361e-05, + "logits/chosen": -2.694643974304199, + "logits/rejected": -2.705972194671631, + "logps/chosen": -231.14852905273438, + "logps/rejected": -266.03460693359375, + "loss": 0.189, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.598564624786377, + "rewards/margins": 3.681683301925659, + "rewards/rejected": -6.280247688293457, + "step": 1192 + }, + { + "epoch": 1.56, + "learning_rate": 4.745206399614135e-05, + "logits/chosen": -2.131896734237671, + "logits/rejected": -2.225205421447754, + "logps/chosen": -222.63446044921875, + "logps/rejected": -309.03814697265625, + "loss": 0.1005, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2715827226638794, + "rewards/margins": 4.563998222351074, + "rewards/rejected": -5.835580825805664, + "step": 1193 + }, + { + "epoch": 1.56, + "learning_rate": 4.744748062496503e-05, + "logits/chosen": -2.4835543632507324, + "logits/rejected": -2.552267551422119, + "logps/chosen": -166.58905029296875, + "logps/rejected": -213.53118896484375, + "loss": 0.2019, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.377805233001709, + "rewards/margins": 3.6189918518066406, + "rewards/rejected": -4.99679708480835, + "step": 1194 + }, + { + "epoch": 1.56, + "learning_rate": 4.744289335684034e-05, + "logits/chosen": -2.522873640060425, + "logits/rejected": -2.517055034637451, + "logps/chosen": -203.4969482421875, + "logps/rejected": -267.63873291015625, + "loss": 0.1573, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.246862530708313, + "rewards/margins": 4.703207492828369, + "rewards/rejected": -5.950070381164551, + "step": 1195 + }, + { + "epoch": 1.57, + "learning_rate": 4.7438302192563625e-05, + "logits/chosen": -2.3815245628356934, + "logits/rejected": -2.4553897380828857, + "logps/chosen": -231.88101196289062, + "logps/rejected": -279.2082824707031, + "loss": 0.1052, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.3264628648757935, + "rewards/margins": 4.349924564361572, + "rewards/rejected": -5.676387786865234, + "step": 1196 + }, + { + "epoch": 1.57, + "learning_rate": 4.743370713293194e-05, + "logits/chosen": -2.671739101409912, + "logits/rejected": -2.6977474689483643, + "logps/chosen": -245.71820068359375, + "logps/rejected": -263.50811767578125, + "loss": 0.2618, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.2178151607513428, + "rewards/margins": 3.918609142303467, + "rewards/rejected": -6.1364240646362305, + "step": 1197 + }, + { + "epoch": 1.57, + "learning_rate": 4.7429108178742985e-05, + "logits/chosen": -2.3009557723999023, + "logits/rejected": -2.2271933555603027, + "logps/chosen": -181.82748413085938, + "logps/rejected": -212.26087951660156, + "loss": 0.5133, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7228325605392456, + "rewards/margins": 3.406215190887451, + "rewards/rejected": -5.129047870635986, + "step": 1198 + }, + { + "epoch": 1.57, + "learning_rate": 4.742450533079518e-05, + "logits/chosen": -2.5072877407073975, + "logits/rejected": -2.4657020568847656, + "logps/chosen": -183.00059509277344, + "logps/rejected": -212.53353881835938, + "loss": 0.3148, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.878244161605835, + "rewards/margins": 2.4889166355133057, + "rewards/rejected": -4.367160797119141, + "step": 1199 + }, + { + "epoch": 1.57, + "learning_rate": 4.7419898589887566e-05, + "logits/chosen": -2.134070873260498, + "logits/rejected": -2.256047487258911, + "logps/chosen": -170.70077514648438, + "logps/rejected": -231.8829345703125, + "loss": 0.1302, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0568268299102783, + "rewards/margins": 4.093639373779297, + "rewards/rejected": -5.1504669189453125, + "step": 1200 + }, + { + "epoch": 1.57, + "learning_rate": 4.74152879568199e-05, + "logits/chosen": -2.6162269115448, + "logits/rejected": -2.652150869369507, + "logps/chosen": -233.90402221679688, + "logps/rejected": -244.74334716796875, + "loss": 0.0997, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6784412860870361, + "rewards/margins": 3.7588188648223877, + "rewards/rejected": -5.437260627746582, + "step": 1201 + }, + { + "epoch": 1.57, + "learning_rate": 4.7410673432392596e-05, + "logits/chosen": -2.484144687652588, + "logits/rejected": -2.5014195442199707, + "logps/chosen": -250.83456420898438, + "logps/rejected": -281.1622314453125, + "loss": 0.2326, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.0612598657608032, + "rewards/margins": 3.5049850940704346, + "rewards/rejected": -4.566245079040527, + "step": 1202 + }, + { + "epoch": 1.57, + "learning_rate": 4.7406055017406754e-05, + "logits/chosen": -2.6265833377838135, + "logits/rejected": -2.7116756439208984, + "logps/chosen": -236.57855224609375, + "logps/rejected": -276.8167724609375, + "loss": 0.1522, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.490135669708252, + "rewards/margins": 3.229520082473755, + "rewards/rejected": -4.719655513763428, + "step": 1203 + }, + { + "epoch": 1.58, + "learning_rate": 4.740143271266414e-05, + "logits/chosen": -2.5222041606903076, + "logits/rejected": -2.6354575157165527, + "logps/chosen": -229.79931640625, + "logps/rejected": -270.754638671875, + "loss": 0.0646, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0310989618301392, + "rewards/margins": 5.396121025085449, + "rewards/rejected": -6.427220344543457, + "step": 1204 + }, + { + "epoch": 1.58, + "learning_rate": 4.73968065189672e-05, + "logits/chosen": -2.396852731704712, + "logits/rejected": -2.4251794815063477, + "logps/chosen": -192.14022827148438, + "logps/rejected": -229.5162353515625, + "loss": 0.3256, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.227713704109192, + "rewards/margins": 2.811344861984253, + "rewards/rejected": -4.039058685302734, + "step": 1205 + }, + { + "epoch": 1.58, + "learning_rate": 4.739217643711906e-05, + "logits/chosen": -2.287642478942871, + "logits/rejected": -2.3326570987701416, + "logps/chosen": -164.69529724121094, + "logps/rejected": -231.8920135498047, + "loss": 0.3453, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.245210886001587, + "rewards/margins": 2.8619096279144287, + "rewards/rejected": -4.107120513916016, + "step": 1206 + }, + { + "epoch": 1.58, + "learning_rate": 4.7387542467923505e-05, + "logits/chosen": -2.479602575302124, + "logits/rejected": -2.5209507942199707, + "logps/chosen": -244.21514892578125, + "logps/rejected": -303.6318664550781, + "loss": 0.0309, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2694910764694214, + "rewards/margins": 5.570847034454346, + "rewards/rejected": -6.840338706970215, + "step": 1207 + }, + { + "epoch": 1.58, + "learning_rate": 4.738290461218502e-05, + "logits/chosen": -2.4771368503570557, + "logits/rejected": -2.456359386444092, + "logps/chosen": -230.273681640625, + "logps/rejected": -269.0933837890625, + "loss": 0.1615, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9786639213562012, + "rewards/margins": 3.434558629989624, + "rewards/rejected": -5.413222312927246, + "step": 1208 + }, + { + "epoch": 1.58, + "learning_rate": 4.737826287070874e-05, + "logits/chosen": -2.2615694999694824, + "logits/rejected": -2.326125383377075, + "logps/chosen": -157.7958526611328, + "logps/rejected": -257.5984191894531, + "loss": 0.1683, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3305480480194092, + "rewards/margins": 3.8238983154296875, + "rewards/rejected": -5.154446125030518, + "step": 1209 + }, + { + "epoch": 1.58, + "learning_rate": 4.737361724430048e-05, + "logits/chosen": -2.5612540245056152, + "logits/rejected": -2.6601011753082275, + "logps/chosen": -240.48211669921875, + "logps/rejected": -257.5409851074219, + "loss": 0.2828, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7493575811386108, + "rewards/margins": 2.682534694671631, + "rewards/rejected": -4.431892395019531, + "step": 1210 + }, + { + "epoch": 1.58, + "learning_rate": 4.7368967733766756e-05, + "logits/chosen": -2.0791330337524414, + "logits/rejected": -2.0077402591705322, + "logps/chosen": -228.59085083007812, + "logps/rejected": -240.2413787841797, + "loss": 0.3355, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.6128778457641602, + "rewards/margins": 3.4234275817871094, + "rewards/rejected": -5.036304950714111, + "step": 1211 + }, + { + "epoch": 1.59, + "learning_rate": 4.7364314339914716e-05, + "logits/chosen": -2.223177671432495, + "logits/rejected": -2.280726671218872, + "logps/chosen": -187.680419921875, + "logps/rejected": -279.1700134277344, + "loss": 0.0374, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6807892322540283, + "rewards/margins": 6.459383010864258, + "rewards/rejected": -7.140172004699707, + "step": 1212 + }, + { + "epoch": 1.59, + "learning_rate": 4.735965706355221e-05, + "logits/chosen": -2.3972856998443604, + "logits/rejected": -2.450835704803467, + "logps/chosen": -180.9072723388672, + "logps/rejected": -224.91830444335938, + "loss": 0.2092, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.4030086994171143, + "rewards/margins": 3.2653090953826904, + "rewards/rejected": -4.668317794799805, + "step": 1213 + }, + { + "epoch": 1.59, + "learning_rate": 4.735499590548775e-05, + "logits/chosen": -2.483901023864746, + "logits/rejected": -2.579958915710449, + "logps/chosen": -165.31178283691406, + "logps/rejected": -212.93484497070312, + "loss": 0.3618, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.7332736253738403, + "rewards/margins": 2.2384800910949707, + "rewards/rejected": -3.9717535972595215, + "step": 1214 + }, + { + "epoch": 1.59, + "learning_rate": 4.7350330866530536e-05, + "logits/chosen": -2.4746031761169434, + "logits/rejected": -2.5858569145202637, + "logps/chosen": -234.17544555664062, + "logps/rejected": -243.3843231201172, + "loss": 0.1585, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.7856451272964478, + "rewards/margins": 3.430792808532715, + "rewards/rejected": -5.216437816619873, + "step": 1215 + }, + { + "epoch": 1.59, + "learning_rate": 4.734566194749043e-05, + "logits/chosen": -2.1635468006134033, + "logits/rejected": -2.184338092803955, + "logps/chosen": -167.8646240234375, + "logps/rejected": -198.2296142578125, + "loss": 0.136, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4915691614151, + "rewards/margins": 3.2404394149780273, + "rewards/rejected": -4.732008934020996, + "step": 1216 + }, + { + "epoch": 1.59, + "learning_rate": 4.7340989149177955e-05, + "logits/chosen": -2.4515089988708496, + "logits/rejected": -2.4606142044067383, + "logps/chosen": -184.1483154296875, + "logps/rejected": -231.7574462890625, + "loss": 0.1365, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9022963047027588, + "rewards/margins": 4.141842842102051, + "rewards/rejected": -6.0441389083862305, + "step": 1217 + }, + { + "epoch": 1.59, + "learning_rate": 4.7336312472404345e-05, + "logits/chosen": -2.495224714279175, + "logits/rejected": -2.6189403533935547, + "logps/chosen": -187.76014709472656, + "logps/rejected": -287.38653564453125, + "loss": 0.1484, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5860087871551514, + "rewards/margins": 4.361400604248047, + "rewards/rejected": -5.947409629821777, + "step": 1218 + }, + { + "epoch": 1.6, + "learning_rate": 4.733163191798147e-05, + "logits/chosen": -2.3041515350341797, + "logits/rejected": -2.3410122394561768, + "logps/chosen": -182.3394775390625, + "logps/rejected": -240.0974884033203, + "loss": 0.2967, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.4822897911071777, + "rewards/margins": 3.956850290298462, + "rewards/rejected": -5.4391398429870605, + "step": 1219 + }, + { + "epoch": 1.6, + "learning_rate": 4.7326947486721894e-05, + "logits/chosen": -2.5003819465637207, + "logits/rejected": -2.486321449279785, + "logps/chosen": -175.6728057861328, + "logps/rejected": -249.07659912109375, + "loss": 0.1647, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0928356647491455, + "rewards/margins": 4.23591423034668, + "rewards/rejected": -5.328750133514404, + "step": 1220 + }, + { + "epoch": 1.6, + "learning_rate": 4.732225917943884e-05, + "logits/chosen": -2.34039306640625, + "logits/rejected": -2.2606441974639893, + "logps/chosen": -209.9175262451172, + "logps/rejected": -277.741455078125, + "loss": 0.167, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.1915336847305298, + "rewards/margins": 4.357934474945068, + "rewards/rejected": -5.54946756362915, + "step": 1221 + }, + { + "epoch": 1.6, + "learning_rate": 4.7317566996946235e-05, + "logits/chosen": -2.5158519744873047, + "logits/rejected": -2.391768455505371, + "logps/chosen": -282.4315490722656, + "logps/rejected": -324.89337158203125, + "loss": 0.0819, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.4069547653198242, + "rewards/margins": 4.744081497192383, + "rewards/rejected": -6.151035785675049, + "step": 1222 + }, + { + "epoch": 1.6, + "learning_rate": 4.731287094005863e-05, + "logits/chosen": -2.2263362407684326, + "logits/rejected": -2.276297092437744, + "logps/chosen": -216.80142211914062, + "logps/rejected": -220.71160888671875, + "loss": 0.2046, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.122260093688965, + "rewards/margins": 3.1513662338256836, + "rewards/rejected": -5.27362585067749, + "step": 1223 + }, + { + "epoch": 1.6, + "learning_rate": 4.730817100959128e-05, + "logits/chosen": -2.3766252994537354, + "logits/rejected": -2.406181812286377, + "logps/chosen": -199.9800567626953, + "logps/rejected": -254.28375244140625, + "loss": 0.2671, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7991282939910889, + "rewards/margins": 3.4454705715179443, + "rewards/rejected": -5.244598388671875, + "step": 1224 + }, + { + "epoch": 1.6, + "learning_rate": 4.730346720636011e-05, + "logits/chosen": -2.5629076957702637, + "logits/rejected": -2.5792136192321777, + "logps/chosen": -177.7575225830078, + "logps/rejected": -288.53857421875, + "loss": 0.0896, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.251127004623413, + "rewards/margins": 4.980374813079834, + "rewards/rejected": -6.231501579284668, + "step": 1225 + }, + { + "epoch": 1.6, + "learning_rate": 4.7298759531181717e-05, + "logits/chosen": -2.423495054244995, + "logits/rejected": -2.53666353225708, + "logps/chosen": -225.38986206054688, + "logps/rejected": -255.51467895507812, + "loss": 0.0392, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.030442476272583, + "rewards/margins": 4.617352485656738, + "rewards/rejected": -6.647795677185059, + "step": 1226 + }, + { + "epoch": 1.61, + "learning_rate": 4.729404798487337e-05, + "logits/chosen": -2.4218320846557617, + "logits/rejected": -2.541698455810547, + "logps/chosen": -189.56942749023438, + "logps/rejected": -243.72561645507812, + "loss": 0.0691, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.4716410636901855, + "rewards/margins": 4.515095233917236, + "rewards/rejected": -5.98673677444458, + "step": 1227 + }, + { + "epoch": 1.61, + "learning_rate": 4.7289332568252994e-05, + "logits/chosen": -2.4882519245147705, + "logits/rejected": -2.520144462585449, + "logps/chosen": -194.36102294921875, + "logps/rejected": -236.93948364257812, + "loss": 0.2329, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6610629558563232, + "rewards/margins": 3.4544930458068848, + "rewards/rejected": -5.115556716918945, + "step": 1228 + }, + { + "epoch": 1.61, + "learning_rate": 4.728461328213921e-05, + "logits/chosen": -2.3431355953216553, + "logits/rejected": -2.4315452575683594, + "logps/chosen": -184.54681396484375, + "logps/rejected": -237.89874267578125, + "loss": 0.1567, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.615098237991333, + "rewards/margins": 4.014577388763428, + "rewards/rejected": -5.629674911499023, + "step": 1229 + }, + { + "epoch": 1.61, + "learning_rate": 4.727989012735129e-05, + "logits/chosen": -2.3083677291870117, + "logits/rejected": -2.3745315074920654, + "logps/chosen": -230.08078002929688, + "logps/rejected": -273.61346435546875, + "loss": 0.1073, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.6126986742019653, + "rewards/margins": 4.383516311645508, + "rewards/rejected": -5.9962158203125, + "step": 1230 + }, + { + "epoch": 1.61, + "learning_rate": 4.72751631047092e-05, + "logits/chosen": -2.5195953845977783, + "logits/rejected": -2.4491376876831055, + "logps/chosen": -195.26785278320312, + "logps/rejected": -225.06417846679688, + "loss": 0.2724, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.1910969018936157, + "rewards/margins": 2.997663736343384, + "rewards/rejected": -4.188760757446289, + "step": 1231 + }, + { + "epoch": 1.61, + "learning_rate": 4.727043221503355e-05, + "logits/chosen": -2.375887632369995, + "logits/rejected": -2.3649067878723145, + "logps/chosen": -190.8800811767578, + "logps/rejected": -264.25982666015625, + "loss": 0.1837, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.8358839750289917, + "rewards/margins": 4.462348461151123, + "rewards/rejected": -5.298232078552246, + "step": 1232 + }, + { + "epoch": 1.61, + "learning_rate": 4.7265697459145656e-05, + "logits/chosen": -2.2352352142333984, + "logits/rejected": -2.3413479328155518, + "logps/chosen": -167.3771514892578, + "logps/rejected": -217.39353942871094, + "loss": 0.3485, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.4793394804000854, + "rewards/margins": 2.465003490447998, + "rewards/rejected": -3.944342851638794, + "step": 1233 + }, + { + "epoch": 1.61, + "learning_rate": 4.7260958837867474e-05, + "logits/chosen": -2.3950436115264893, + "logits/rejected": -2.3509271144866943, + "logps/chosen": -221.51156616210938, + "logps/rejected": -274.8773193359375, + "loss": 0.237, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2772961854934692, + "rewards/margins": 3.8298304080963135, + "rewards/rejected": -5.107126712799072, + "step": 1234 + }, + { + "epoch": 1.62, + "learning_rate": 4.725621635202164e-05, + "logits/chosen": -2.217377185821533, + "logits/rejected": -2.2314724922180176, + "logps/chosen": -218.805419921875, + "logps/rejected": -250.1678466796875, + "loss": 0.274, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.7175061702728271, + "rewards/margins": 3.0603437423706055, + "rewards/rejected": -4.777850151062012, + "step": 1235 + }, + { + "epoch": 1.62, + "learning_rate": 4.7251470002431463e-05, + "logits/chosen": -2.3143558502197266, + "logits/rejected": -2.3055665493011475, + "logps/chosen": -209.69020080566406, + "logps/rejected": -267.2593994140625, + "loss": 0.0749, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4336559772491455, + "rewards/margins": 4.277906894683838, + "rewards/rejected": -5.711562633514404, + "step": 1236 + }, + { + "epoch": 1.62, + "learning_rate": 4.724671978992093e-05, + "logits/chosen": -2.5497217178344727, + "logits/rejected": -2.357327699661255, + "logps/chosen": -242.28077697753906, + "logps/rejected": -254.4888916015625, + "loss": 0.287, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.279671311378479, + "rewards/margins": 4.20197057723999, + "rewards/rejected": -5.48164176940918, + "step": 1237 + }, + { + "epoch": 1.62, + "learning_rate": 4.724196571531469e-05, + "logits/chosen": -2.209345579147339, + "logits/rejected": -2.303694725036621, + "logps/chosen": -213.60232543945312, + "logps/rejected": -218.8167724609375, + "loss": 0.3236, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.613269329071045, + "rewards/margins": 2.6522085666656494, + "rewards/rejected": -4.265478134155273, + "step": 1238 + }, + { + "epoch": 1.62, + "learning_rate": 4.7237207779438066e-05, + "logits/chosen": -2.289630889892578, + "logits/rejected": -2.408109188079834, + "logps/chosen": -220.02053833007812, + "logps/rejected": -287.1589660644531, + "loss": 0.1712, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.9712376594543457, + "rewards/margins": 5.0107197761535645, + "rewards/rejected": -6.98195743560791, + "step": 1239 + }, + { + "epoch": 1.62, + "learning_rate": 4.7232445983117045e-05, + "logits/chosen": -2.221944808959961, + "logits/rejected": -2.331416606903076, + "logps/chosen": -204.5431365966797, + "logps/rejected": -249.51193237304688, + "loss": 0.2558, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.8107631206512451, + "rewards/margins": 2.60237455368042, + "rewards/rejected": -4.413137435913086, + "step": 1240 + }, + { + "epoch": 1.62, + "learning_rate": 4.7227680327178294e-05, + "logits/chosen": -2.1551241874694824, + "logits/rejected": -2.1501760482788086, + "logps/chosen": -209.62078857421875, + "logps/rejected": -237.73171997070312, + "loss": 0.1494, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.4279321432113647, + "rewards/margins": 3.582655429840088, + "rewards/rejected": -5.010587692260742, + "step": 1241 + }, + { + "epoch": 1.63, + "learning_rate": 4.7222910812449126e-05, + "logits/chosen": -2.5197300910949707, + "logits/rejected": -2.501941442489624, + "logps/chosen": -224.577880859375, + "logps/rejected": -275.0819091796875, + "loss": 0.1024, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8110631704330444, + "rewards/margins": 4.743729591369629, + "rewards/rejected": -5.554792404174805, + "step": 1242 + }, + { + "epoch": 1.63, + "learning_rate": 4.7218137439757574e-05, + "logits/chosen": -2.5621564388275146, + "logits/rejected": -2.604523181915283, + "logps/chosen": -160.7113037109375, + "logps/rejected": -195.0877227783203, + "loss": 0.2681, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.9945410490036011, + "rewards/margins": 2.9356656074523926, + "rewards/rejected": -3.930206298828125, + "step": 1243 + }, + { + "epoch": 1.63, + "learning_rate": 4.7213360209932286e-05, + "logits/chosen": -2.444904088973999, + "logits/rejected": -2.7083749771118164, + "logps/chosen": -293.1217041015625, + "logps/rejected": -356.0822448730469, + "loss": 0.1383, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.486177921295166, + "rewards/margins": 3.4033234119415283, + "rewards/rejected": -4.889501571655273, + "step": 1244 + }, + { + "epoch": 1.63, + "learning_rate": 4.720857912380261e-05, + "logits/chosen": -2.3962252140045166, + "logits/rejected": -2.4494524002075195, + "logps/chosen": -245.69485473632812, + "logps/rejected": -278.7045593261719, + "loss": 0.1499, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5282708406448364, + "rewards/margins": 3.444108486175537, + "rewards/rejected": -4.972379207611084, + "step": 1245 + }, + { + "epoch": 1.63, + "learning_rate": 4.7203794182198556e-05, + "logits/chosen": -2.1511898040771484, + "logits/rejected": -1.9930884838104248, + "logps/chosen": -233.6335906982422, + "logps/rejected": -218.43870544433594, + "loss": 0.2441, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1958537101745605, + "rewards/margins": 3.0618109703063965, + "rewards/rejected": -4.257665157318115, + "step": 1246 + }, + { + "epoch": 1.63, + "learning_rate": 4.7199005385950813e-05, + "logits/chosen": -2.366896629333496, + "logits/rejected": -2.437415361404419, + "logps/chosen": -172.52511596679688, + "logps/rejected": -218.31173706054688, + "loss": 0.2482, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.1920199394226074, + "rewards/margins": 3.408511161804199, + "rewards/rejected": -4.600531101226807, + "step": 1247 + }, + { + "epoch": 1.63, + "learning_rate": 4.719421273589071e-05, + "logits/chosen": -2.3530538082122803, + "logits/rejected": -2.39542555809021, + "logps/chosen": -214.77020263671875, + "logps/rejected": -258.3968811035156, + "loss": 0.2237, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.9770315289497375, + "rewards/margins": 3.545395851135254, + "rewards/rejected": -4.522427558898926, + "step": 1248 + }, + { + "epoch": 1.63, + "learning_rate": 4.718941623285028e-05, + "logits/chosen": -2.5741071701049805, + "logits/rejected": -2.619910478591919, + "logps/chosen": -190.63352966308594, + "logps/rejected": -251.1927947998047, + "loss": 0.135, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2246308326721191, + "rewards/margins": 3.43021559715271, + "rewards/rejected": -4.654846668243408, + "step": 1249 + }, + { + "epoch": 1.64, + "learning_rate": 4.718461587766221e-05, + "logits/chosen": -2.167393922805786, + "logits/rejected": -2.1985065937042236, + "logps/chosen": -184.55860900878906, + "logps/rejected": -208.03286743164062, + "loss": 0.2788, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.9492772221565247, + "rewards/margins": 3.1372551918029785, + "rewards/rejected": -4.086532115936279, + "step": 1250 + }, + { + "epoch": 1.64, + "learning_rate": 4.717981167115986e-05, + "logits/chosen": -2.3610846996307373, + "logits/rejected": -2.4981765747070312, + "logps/chosen": -181.03359985351562, + "logps/rejected": -254.43211364746094, + "loss": 0.1609, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5473884344100952, + "rewards/margins": 3.8110976219177246, + "rewards/rejected": -5.358486175537109, + "step": 1251 + }, + { + "epoch": 1.64, + "learning_rate": 4.717500361417724e-05, + "logits/chosen": -2.58233904838562, + "logits/rejected": -2.4361298084259033, + "logps/chosen": -227.0497589111328, + "logps/rejected": -225.55099487304688, + "loss": 0.2847, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3530316352844238, + "rewards/margins": 3.4566879272460938, + "rewards/rejected": -4.809719562530518, + "step": 1252 + }, + { + "epoch": 1.64, + "learning_rate": 4.717019170754905e-05, + "logits/chosen": -2.4212703704833984, + "logits/rejected": -2.353156805038452, + "logps/chosen": -244.638671875, + "logps/rejected": -234.03988647460938, + "loss": 0.1988, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9548088908195496, + "rewards/margins": 3.1238605976104736, + "rewards/rejected": -4.07866907119751, + "step": 1253 + }, + { + "epoch": 1.64, + "learning_rate": 4.716537595211065e-05, + "logits/chosen": -1.9767769575119019, + "logits/rejected": -2.0549771785736084, + "logps/chosen": -250.92616271972656, + "logps/rejected": -285.457763671875, + "loss": 0.1845, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.187235951423645, + "rewards/margins": 3.0077717304229736, + "rewards/rejected": -4.195007801055908, + "step": 1254 + }, + { + "epoch": 1.64, + "learning_rate": 4.716055634869807e-05, + "logits/chosen": -2.5653319358825684, + "logits/rejected": -2.4723970890045166, + "logps/chosen": -192.06153869628906, + "logps/rejected": -212.7266387939453, + "loss": 0.2448, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.9501335024833679, + "rewards/margins": 3.1784751415252686, + "rewards/rejected": -4.128608703613281, + "step": 1255 + }, + { + "epoch": 1.64, + "learning_rate": 4.715573289814801e-05, + "logits/chosen": -2.546515941619873, + "logits/rejected": -2.493617057800293, + "logps/chosen": -171.47616577148438, + "logps/rejected": -203.3109130859375, + "loss": 0.2335, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.9509350061416626, + "rewards/margins": 2.794473648071289, + "rewards/rejected": -3.745408535003662, + "step": 1256 + }, + { + "epoch": 1.65, + "learning_rate": 4.715090560129782e-05, + "logits/chosen": -1.9311320781707764, + "logits/rejected": -1.997302770614624, + "logps/chosen": -173.90911865234375, + "logps/rejected": -195.12417602539062, + "loss": 0.2128, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.22751878201961517, + "rewards/margins": 3.4993607997894287, + "rewards/rejected": -3.726879835128784, + "step": 1257 + }, + { + "epoch": 1.65, + "learning_rate": 4.714607445898556e-05, + "logits/chosen": -2.3790111541748047, + "logits/rejected": -2.4061365127563477, + "logps/chosen": -193.8656463623047, + "logps/rejected": -248.78428649902344, + "loss": 0.1781, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.7878875732421875, + "rewards/margins": 3.8841519355773926, + "rewards/rejected": -4.67203950881958, + "step": 1258 + }, + { + "epoch": 1.65, + "learning_rate": 4.714123947204991e-05, + "logits/chosen": -2.5352847576141357, + "logits/rejected": -2.5170440673828125, + "logps/chosen": -207.12197875976562, + "logps/rejected": -240.71539306640625, + "loss": 0.1627, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1929060220718384, + "rewards/margins": 3.0479984283447266, + "rewards/rejected": -4.240904331207275, + "step": 1259 + }, + { + "epoch": 1.65, + "learning_rate": 4.713640064133025e-05, + "logits/chosen": -2.321063756942749, + "logits/rejected": -2.323094367980957, + "logps/chosen": -178.62144470214844, + "logps/rejected": -220.08468627929688, + "loss": 0.1978, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.8668509721755981, + "rewards/margins": 3.2052500247955322, + "rewards/rejected": -4.07210111618042, + "step": 1260 + }, + { + "epoch": 1.65, + "learning_rate": 4.7131557967666604e-05, + "logits/chosen": -2.3726282119750977, + "logits/rejected": -2.4751479625701904, + "logps/chosen": -178.8557891845703, + "logps/rejected": -251.33251953125, + "loss": 0.1386, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6575889587402344, + "rewards/margins": 3.7119858264923096, + "rewards/rejected": -4.369575023651123, + "step": 1261 + }, + { + "epoch": 1.65, + "learning_rate": 4.712671145189968e-05, + "logits/chosen": -2.479706048965454, + "logits/rejected": -2.519768238067627, + "logps/chosen": -240.58731079101562, + "logps/rejected": -287.4944763183594, + "loss": 0.1786, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.6783007979393005, + "rewards/margins": 3.953369140625, + "rewards/rejected": -4.631669998168945, + "step": 1262 + }, + { + "epoch": 1.65, + "learning_rate": 4.712186109487085e-05, + "logits/chosen": -2.2034120559692383, + "logits/rejected": -2.2748920917510986, + "logps/chosen": -176.4800262451172, + "logps/rejected": -242.2554931640625, + "loss": 0.0759, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.833525538444519, + "rewards/margins": 4.219998836517334, + "rewards/rejected": -5.053524494171143, + "step": 1263 + }, + { + "epoch": 1.65, + "learning_rate": 4.7117006897422144e-05, + "logits/chosen": -2.3310108184814453, + "logits/rejected": -2.3973095417022705, + "logps/chosen": -159.26260375976562, + "logps/rejected": -178.69801330566406, + "loss": 0.2582, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5862424969673157, + "rewards/margins": 3.4215569496154785, + "rewards/rejected": -4.00779914855957, + "step": 1264 + }, + { + "epoch": 1.66, + "learning_rate": 4.7112148860396265e-05, + "logits/chosen": -2.289217710494995, + "logits/rejected": -2.3254284858703613, + "logps/chosen": -198.0190887451172, + "logps/rejected": -251.11077880859375, + "loss": 0.1246, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1426129341125488, + "rewards/margins": 4.011805534362793, + "rewards/rejected": -5.154417991638184, + "step": 1265 + }, + { + "epoch": 1.66, + "learning_rate": 4.7107286984636584e-05, + "logits/chosen": -2.2878618240356445, + "logits/rejected": -2.4422900676727295, + "logps/chosen": -184.45816040039062, + "logps/rejected": -227.22039794921875, + "loss": 0.2988, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.1406644582748413, + "rewards/margins": 2.195034980773926, + "rewards/rejected": -3.3356995582580566, + "step": 1266 + }, + { + "epoch": 1.66, + "learning_rate": 4.710242127098714e-05, + "logits/chosen": -2.411558151245117, + "logits/rejected": -2.433676242828369, + "logps/chosen": -276.55462646484375, + "logps/rejected": -287.45538330078125, + "loss": 0.1436, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3652961254119873, + "rewards/margins": 2.717362403869629, + "rewards/rejected": -4.082658290863037, + "step": 1267 + }, + { + "epoch": 1.66, + "learning_rate": 4.7097551720292634e-05, + "logits/chosen": -2.2663257122039795, + "logits/rejected": -2.3046867847442627, + "logps/chosen": -229.05191040039062, + "logps/rejected": -234.79849243164062, + "loss": 0.1054, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1178110837936401, + "rewards/margins": 3.994016408920288, + "rewards/rejected": -5.1118268966674805, + "step": 1268 + }, + { + "epoch": 1.66, + "learning_rate": 4.709267833339843e-05, + "logits/chosen": -2.4231812953948975, + "logits/rejected": -2.46999192237854, + "logps/chosen": -243.0467529296875, + "logps/rejected": -288.1591796875, + "loss": 0.0957, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.6723252534866333, + "rewards/margins": 4.546743869781494, + "rewards/rejected": -5.219069480895996, + "step": 1269 + }, + { + "epoch": 1.66, + "learning_rate": 4.708780111115057e-05, + "logits/chosen": -2.2736947536468506, + "logits/rejected": -2.318422794342041, + "logps/chosen": -240.78549194335938, + "logps/rejected": -262.1117858886719, + "loss": 0.0696, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0388414859771729, + "rewards/margins": 3.992526054382324, + "rewards/rejected": -5.031367301940918, + "step": 1270 + }, + { + "epoch": 1.66, + "learning_rate": 4.708292005439575e-05, + "logits/chosen": -2.3022494316101074, + "logits/rejected": -2.331955671310425, + "logps/chosen": -235.0003204345703, + "logps/rejected": -286.7228698730469, + "loss": 0.0884, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.9852278232574463, + "rewards/margins": 4.658226013183594, + "rewards/rejected": -5.643453598022461, + "step": 1271 + }, + { + "epoch": 1.66, + "learning_rate": 4.707803516398134e-05, + "logits/chosen": -2.302123546600342, + "logits/rejected": -2.2593071460723877, + "logps/chosen": -219.12557983398438, + "logps/rejected": -229.34735107421875, + "loss": 0.2727, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6084837913513184, + "rewards/margins": 2.6438004970550537, + "rewards/rejected": -4.252284049987793, + "step": 1272 + }, + { + "epoch": 1.67, + "learning_rate": 4.707314644075536e-05, + "logits/chosen": -2.2804455757141113, + "logits/rejected": -2.313739538192749, + "logps/chosen": -177.1968994140625, + "logps/rejected": -253.0790557861328, + "loss": 0.1938, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.1521004438400269, + "rewards/margins": 3.550403118133545, + "rewards/rejected": -4.702503204345703, + "step": 1273 + }, + { + "epoch": 1.67, + "learning_rate": 4.706825388556652e-05, + "logits/chosen": -2.3808209896087646, + "logits/rejected": -2.509162187576294, + "logps/chosen": -252.71810913085938, + "logps/rejected": -296.3233642578125, + "loss": 0.1607, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.4979945421218872, + "rewards/margins": 3.630544424057007, + "rewards/rejected": -5.128539085388184, + "step": 1274 + }, + { + "epoch": 1.67, + "learning_rate": 4.706335749926417e-05, + "logits/chosen": -2.3603124618530273, + "logits/rejected": -2.442885637283325, + "logps/chosen": -187.13800048828125, + "logps/rejected": -226.97061157226562, + "loss": 0.1575, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0760753154754639, + "rewards/margins": 3.3448987007141113, + "rewards/rejected": -4.420973777770996, + "step": 1275 + }, + { + "epoch": 1.67, + "learning_rate": 4.7058457282698336e-05, + "logits/chosen": -2.178051233291626, + "logits/rejected": -2.302097797393799, + "logps/chosen": -181.60252380371094, + "logps/rejected": -298.5541687011719, + "loss": 0.0962, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.3809150457382202, + "rewards/margins": 3.7876832485198975, + "rewards/rejected": -5.168598175048828, + "step": 1276 + }, + { + "epoch": 1.67, + "learning_rate": 4.7053553236719726e-05, + "logits/chosen": -2.2287707328796387, + "logits/rejected": -2.3011856079101562, + "logps/chosen": -193.0646514892578, + "logps/rejected": -230.15853881835938, + "loss": 0.1986, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.612187385559082, + "rewards/margins": 2.7522740364074707, + "rewards/rejected": -4.3644609451293945, + "step": 1277 + }, + { + "epoch": 1.67, + "learning_rate": 4.7048645362179686e-05, + "logits/chosen": -2.3724539279937744, + "logits/rejected": -2.3160030841827393, + "logps/chosen": -199.37181091308594, + "logps/rejected": -222.05191040039062, + "loss": 0.1415, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.6102732419967651, + "rewards/margins": 3.988173723220825, + "rewards/rejected": -4.598446369171143, + "step": 1278 + }, + { + "epoch": 1.67, + "learning_rate": 4.7043733659930235e-05, + "logits/chosen": -1.9961585998535156, + "logits/rejected": -2.158682346343994, + "logps/chosen": -148.25836181640625, + "logps/rejected": -226.0478515625, + "loss": 0.1392, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.005221962928772, + "rewards/margins": 3.8446872234344482, + "rewards/rejected": -4.84990930557251, + "step": 1279 + }, + { + "epoch": 1.68, + "learning_rate": 4.703881813082406e-05, + "logits/chosen": -2.171950101852417, + "logits/rejected": -2.174027442932129, + "logps/chosen": -188.34934997558594, + "logps/rejected": -226.32675170898438, + "loss": 0.2559, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.7046524286270142, + "rewards/margins": 3.046712875366211, + "rewards/rejected": -4.751364707946777, + "step": 1280 + }, + { + "epoch": 1.68, + "learning_rate": 4.703389877571451e-05, + "logits/chosen": -2.3705697059631348, + "logits/rejected": -2.3761351108551025, + "logps/chosen": -262.95513916015625, + "logps/rejected": -301.3341369628906, + "loss": 0.3501, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.5746833086013794, + "rewards/margins": 3.5883994102478027, + "rewards/rejected": -5.163082599639893, + "step": 1281 + }, + { + "epoch": 1.68, + "learning_rate": 4.7028975595455615e-05, + "logits/chosen": -2.2629284858703613, + "logits/rejected": -2.3212735652923584, + "logps/chosen": -148.93194580078125, + "logps/rejected": -209.88656616210938, + "loss": 0.2053, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2451274394989014, + "rewards/margins": 3.617295026779175, + "rewards/rejected": -4.862421989440918, + "step": 1282 + }, + { + "epoch": 1.68, + "learning_rate": 4.7024048590902036e-05, + "logits/chosen": -2.1855812072753906, + "logits/rejected": -2.266613721847534, + "logps/chosen": -208.891845703125, + "logps/rejected": -263.1065979003906, + "loss": 0.2646, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.979274868965149, + "rewards/margins": 2.8052868843078613, + "rewards/rejected": -4.784561634063721, + "step": 1283 + }, + { + "epoch": 1.68, + "learning_rate": 4.701911776290912e-05, + "logits/chosen": -2.373356342315674, + "logits/rejected": -2.4201443195343018, + "logps/chosen": -213.91403198242188, + "logps/rejected": -248.12171936035156, + "loss": 0.1366, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.634600281715393, + "rewards/margins": 3.5006790161132812, + "rewards/rejected": -5.135279655456543, + "step": 1284 + }, + { + "epoch": 1.68, + "learning_rate": 4.7014183112332874e-05, + "logits/chosen": -2.0015125274658203, + "logits/rejected": -2.0453169345855713, + "logps/chosen": -200.25445556640625, + "logps/rejected": -244.46153259277344, + "loss": 0.1716, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2220110893249512, + "rewards/margins": 4.0147528648376465, + "rewards/rejected": -5.236764430999756, + "step": 1285 + }, + { + "epoch": 1.68, + "learning_rate": 4.700924464002998e-05, + "logits/chosen": -2.3927388191223145, + "logits/rejected": -2.5242536067962646, + "logps/chosen": -209.62437438964844, + "logps/rejected": -284.5310974121094, + "loss": 0.1046, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.037900447845459, + "rewards/margins": 4.573543071746826, + "rewards/rejected": -5.611443996429443, + "step": 1286 + }, + { + "epoch": 1.68, + "learning_rate": 4.700430234685774e-05, + "logits/chosen": -1.9511386156082153, + "logits/rejected": -1.9067704677581787, + "logps/chosen": -237.9818115234375, + "logps/rejected": -313.5265197753906, + "loss": 0.2084, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6042085886001587, + "rewards/margins": 3.7971808910369873, + "rewards/rejected": -5.401390075683594, + "step": 1287 + }, + { + "epoch": 1.69, + "learning_rate": 4.699935623367419e-05, + "logits/chosen": -2.0798680782318115, + "logits/rejected": -2.097235679626465, + "logps/chosen": -164.66754150390625, + "logps/rejected": -223.6951904296875, + "loss": 0.1752, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.4378409385681152, + "rewards/margins": 3.772213935852051, + "rewards/rejected": -5.210054874420166, + "step": 1288 + }, + { + "epoch": 1.69, + "learning_rate": 4.699440630133794e-05, + "logits/chosen": -1.9853588342666626, + "logits/rejected": -2.1096606254577637, + "logps/chosen": -170.85031127929688, + "logps/rejected": -239.245361328125, + "loss": 0.153, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.4343587160110474, + "rewards/margins": 4.021684646606445, + "rewards/rejected": -5.456043720245361, + "step": 1289 + }, + { + "epoch": 1.69, + "learning_rate": 4.698945255070837e-05, + "logits/chosen": -2.266202449798584, + "logits/rejected": -2.3734381198883057, + "logps/chosen": -241.38861083984375, + "logps/rejected": -287.3223876953125, + "loss": 0.1812, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.899042010307312, + "rewards/margins": 4.466583728790283, + "rewards/rejected": -6.365625381469727, + "step": 1290 + }, + { + "epoch": 1.69, + "learning_rate": 4.698449498264543e-05, + "logits/chosen": -2.0678675174713135, + "logits/rejected": -2.1580851078033447, + "logps/chosen": -177.24830627441406, + "logps/rejected": -204.01907348632812, + "loss": 0.2167, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0937800407409668, + "rewards/margins": 3.3943605422973633, + "rewards/rejected": -4.48814058303833, + "step": 1291 + }, + { + "epoch": 1.69, + "learning_rate": 4.697953359800977e-05, + "logits/chosen": -2.3612008094787598, + "logits/rejected": -2.470273017883301, + "logps/chosen": -190.984375, + "logps/rejected": -306.0478515625, + "loss": 0.0471, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2556102275848389, + "rewards/margins": 5.612930774688721, + "rewards/rejected": -6.868541240692139, + "step": 1292 + }, + { + "epoch": 1.69, + "learning_rate": 4.697456839766271e-05, + "logits/chosen": -2.3923254013061523, + "logits/rejected": -2.461078643798828, + "logps/chosen": -164.07113647460938, + "logps/rejected": -214.48974609375, + "loss": 0.1813, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.171428918838501, + "rewards/margins": 2.888441562652588, + "rewards/rejected": -4.059870719909668, + "step": 1293 + }, + { + "epoch": 1.69, + "learning_rate": 4.6969599382466224e-05, + "logits/chosen": -1.8895044326782227, + "logits/rejected": -1.9640676975250244, + "logps/chosen": -289.2847595214844, + "logps/rejected": -292.143310546875, + "loss": 0.2552, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.2626152038574219, + "rewards/margins": 3.0650792121887207, + "rewards/rejected": -4.327694416046143, + "step": 1294 + }, + { + "epoch": 1.69, + "learning_rate": 4.696462655328294e-05, + "logits/chosen": -2.2127761840820312, + "logits/rejected": -2.1274161338806152, + "logps/chosen": -225.32937622070312, + "logps/rejected": -221.54954528808594, + "loss": 0.2618, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.374513864517212, + "rewards/margins": 2.8716228008270264, + "rewards/rejected": -4.246136665344238, + "step": 1295 + }, + { + "epoch": 1.7, + "learning_rate": 4.695964991097616e-05, + "logits/chosen": -2.295297861099243, + "logits/rejected": -2.401890754699707, + "logps/chosen": -183.36146545410156, + "logps/rejected": -258.7024230957031, + "loss": 0.0562, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.309377670288086, + "rewards/margins": 4.095472812652588, + "rewards/rejected": -5.404850959777832, + "step": 1296 + }, + { + "epoch": 1.7, + "learning_rate": 4.695466945640985e-05, + "logits/chosen": -2.3360753059387207, + "logits/rejected": -2.359584093093872, + "logps/chosen": -300.7992858886719, + "logps/rejected": -348.9504089355469, + "loss": 0.1097, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1483001708984375, + "rewards/margins": 5.027990818023682, + "rewards/rejected": -6.176291465759277, + "step": 1297 + }, + { + "epoch": 1.7, + "learning_rate": 4.694968519044862e-05, + "logits/chosen": -2.1906495094299316, + "logits/rejected": -2.2294445037841797, + "logps/chosen": -199.27716064453125, + "logps/rejected": -267.9615173339844, + "loss": 0.1167, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.9273327589035034, + "rewards/margins": 4.393675804138184, + "rewards/rejected": -5.321008682250977, + "step": 1298 + }, + { + "epoch": 1.7, + "learning_rate": 4.6944697113957756e-05, + "logits/chosen": -2.2078731060028076, + "logits/rejected": -2.256173849105835, + "logps/chosen": -191.87767028808594, + "logps/rejected": -235.9379425048828, + "loss": 0.229, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.396527886390686, + "rewards/margins": 2.8328018188476562, + "rewards/rejected": -4.229329586029053, + "step": 1299 + }, + { + "epoch": 1.7, + "learning_rate": 4.69397052278032e-05, + "logits/chosen": -2.1057164669036865, + "logits/rejected": -2.2075157165527344, + "logps/chosen": -159.22825622558594, + "logps/rejected": -249.6995849609375, + "loss": 0.2449, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.563463568687439, + "rewards/margins": 3.6448473930358887, + "rewards/rejected": -5.208311080932617, + "step": 1300 + }, + { + "epoch": 1.7, + "learning_rate": 4.6934709532851576e-05, + "logits/chosen": -1.976876974105835, + "logits/rejected": -2.1168007850646973, + "logps/chosen": -190.38943481445312, + "logps/rejected": -240.00660705566406, + "loss": 0.1593, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6499911546707153, + "rewards/margins": 4.014796257019043, + "rewards/rejected": -5.664787769317627, + "step": 1301 + }, + { + "epoch": 1.7, + "learning_rate": 4.692971002997013e-05, + "logits/chosen": -2.2851758003234863, + "logits/rejected": -2.3062691688537598, + "logps/chosen": -213.82559204101562, + "logps/rejected": -243.2498779296875, + "loss": 0.2007, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.8377220630645752, + "rewards/margins": 3.4279751777648926, + "rewards/rejected": -5.265697002410889, + "step": 1302 + }, + { + "epoch": 1.71, + "learning_rate": 4.69247067200268e-05, + "logits/chosen": -2.130112886428833, + "logits/rejected": -2.21236515045166, + "logps/chosen": -232.77862548828125, + "logps/rejected": -242.76791381835938, + "loss": 0.1511, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.000922679901123, + "rewards/margins": 3.678673028945923, + "rewards/rejected": -5.679595947265625, + "step": 1303 + }, + { + "epoch": 1.71, + "learning_rate": 4.691969960389017e-05, + "logits/chosen": -2.2007036209106445, + "logits/rejected": -2.316322088241577, + "logps/chosen": -195.4522247314453, + "logps/rejected": -319.2470397949219, + "loss": 0.0627, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5212204456329346, + "rewards/margins": 5.476848602294922, + "rewards/rejected": -6.998068809509277, + "step": 1304 + }, + { + "epoch": 1.71, + "learning_rate": 4.6914688682429496e-05, + "logits/chosen": -2.1540913581848145, + "logits/rejected": -2.2840263843536377, + "logps/chosen": -164.23712158203125, + "logps/rejected": -267.3701477050781, + "loss": 0.154, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.7769471406936646, + "rewards/margins": 4.149184226989746, + "rewards/rejected": -5.926130771636963, + "step": 1305 + }, + { + "epoch": 1.71, + "learning_rate": 4.690967395651469e-05, + "logits/chosen": -2.2483348846435547, + "logits/rejected": -2.29646372795105, + "logps/chosen": -198.18312072753906, + "logps/rejected": -250.00401306152344, + "loss": 0.059, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.107490301132202, + "rewards/margins": 4.931715965270996, + "rewards/rejected": -7.039206027984619, + "step": 1306 + }, + { + "epoch": 1.71, + "learning_rate": 4.6904655427016314e-05, + "logits/chosen": -2.344970703125, + "logits/rejected": -2.3008577823638916, + "logps/chosen": -242.4059600830078, + "logps/rejected": -261.99755859375, + "loss": 0.211, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.909515380859375, + "rewards/margins": 4.080948829650879, + "rewards/rejected": -5.990464687347412, + "step": 1307 + }, + { + "epoch": 1.71, + "learning_rate": 4.6899633094805604e-05, + "logits/chosen": -2.0280845165252686, + "logits/rejected": -2.0966038703918457, + "logps/chosen": -218.1485595703125, + "logps/rejected": -247.9756317138672, + "loss": 0.1563, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.803059697151184, + "rewards/margins": 3.7059831619262695, + "rewards/rejected": -5.509043216705322, + "step": 1308 + }, + { + "epoch": 1.71, + "learning_rate": 4.689460696075446e-05, + "logits/chosen": -2.481982707977295, + "logits/rejected": -2.4297902584075928, + "logps/chosen": -226.631103515625, + "logps/rejected": -258.3005676269531, + "loss": 0.1634, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.8385543823242188, + "rewards/margins": 3.6741104125976562, + "rewards/rejected": -5.512665271759033, + "step": 1309 + }, + { + "epoch": 1.71, + "learning_rate": 4.688957702573542e-05, + "logits/chosen": -2.086306095123291, + "logits/rejected": -2.0635809898376465, + "logps/chosen": -227.90188598632812, + "logps/rejected": -282.12567138671875, + "loss": 0.2416, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.798666000366211, + "rewards/margins": 3.5320932865142822, + "rewards/rejected": -5.330759048461914, + "step": 1310 + }, + { + "epoch": 1.72, + "learning_rate": 4.6884543290621706e-05, + "logits/chosen": -2.2293214797973633, + "logits/rejected": -2.256197452545166, + "logps/chosen": -173.7129669189453, + "logps/rejected": -233.62586975097656, + "loss": 0.4036, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.9006210565567017, + "rewards/margins": 2.86419677734375, + "rewards/rejected": -4.764817714691162, + "step": 1311 + }, + { + "epoch": 1.72, + "learning_rate": 4.687950575628718e-05, + "logits/chosen": -2.299515962600708, + "logits/rejected": -2.3766961097717285, + "logps/chosen": -179.56732177734375, + "logps/rejected": -205.10609436035156, + "loss": 0.2016, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3757731914520264, + "rewards/margins": 2.9517035484313965, + "rewards/rejected": -4.327476501464844, + "step": 1312 + }, + { + "epoch": 1.72, + "learning_rate": 4.6874464423606376e-05, + "logits/chosen": -2.057006597518921, + "logits/rejected": -2.160207748413086, + "logps/chosen": -161.31712341308594, + "logps/rejected": -270.31756591796875, + "loss": 0.2467, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.198062539100647, + "rewards/margins": 3.796602725982666, + "rewards/rejected": -4.994665145874023, + "step": 1313 + }, + { + "epoch": 1.72, + "learning_rate": 4.68694192934545e-05, + "logits/chosen": -2.2372817993164062, + "logits/rejected": -2.2993078231811523, + "logps/chosen": -181.70199584960938, + "logps/rejected": -230.60086059570312, + "loss": 0.1501, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5110223293304443, + "rewards/margins": 3.5902843475341797, + "rewards/rejected": -5.101306438446045, + "step": 1314 + }, + { + "epoch": 1.72, + "learning_rate": 4.6864370366707366e-05, + "logits/chosen": -2.2221570014953613, + "logits/rejected": -2.290891170501709, + "logps/chosen": -180.98440551757812, + "logps/rejected": -241.5121612548828, + "loss": 0.208, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.7390170097351074, + "rewards/margins": 3.494724988937378, + "rewards/rejected": -5.2337422370910645, + "step": 1315 + }, + { + "epoch": 1.72, + "learning_rate": 4.685931764424152e-05, + "logits/chosen": -2.3075151443481445, + "logits/rejected": -2.3091671466827393, + "logps/chosen": -141.8212890625, + "logps/rejected": -196.77894592285156, + "loss": 0.161, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0228283405303955, + "rewards/margins": 3.695592164993286, + "rewards/rejected": -4.718420505523682, + "step": 1316 + }, + { + "epoch": 1.72, + "learning_rate": 4.6854261126934104e-05, + "logits/chosen": -2.3127944469451904, + "logits/rejected": -2.3393845558166504, + "logps/chosen": -210.4659423828125, + "logps/rejected": -244.98114013671875, + "loss": 0.1695, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.8345630168914795, + "rewards/margins": 3.397213935852051, + "rewards/rejected": -5.231777191162109, + "step": 1317 + }, + { + "epoch": 1.72, + "learning_rate": 4.684920081566295e-05, + "logits/chosen": -2.4269180297851562, + "logits/rejected": -2.3492565155029297, + "logps/chosen": -233.5777130126953, + "logps/rejected": -288.9328918457031, + "loss": 0.2221, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.7820842266082764, + "rewards/margins": 4.016000270843506, + "rewards/rejected": -5.7980852127075195, + "step": 1318 + }, + { + "epoch": 1.73, + "learning_rate": 4.684413671130655e-05, + "logits/chosen": -2.320209503173828, + "logits/rejected": -2.377079486846924, + "logps/chosen": -207.27413940429688, + "logps/rejected": -261.8699645996094, + "loss": 0.1259, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6501234769821167, + "rewards/margins": 4.435462474822998, + "rewards/rejected": -6.085585594177246, + "step": 1319 + }, + { + "epoch": 1.73, + "learning_rate": 4.683906881474405e-05, + "logits/chosen": -2.298753499984741, + "logits/rejected": -2.3262522220611572, + "logps/chosen": -184.51959228515625, + "logps/rejected": -243.18218994140625, + "loss": 0.0632, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.8446322679519653, + "rewards/margins": 3.7827131748199463, + "rewards/rejected": -5.627345085144043, + "step": 1320 + }, + { + "epoch": 1.73, + "learning_rate": 4.6833997126855236e-05, + "logits/chosen": -2.3990066051483154, + "logits/rejected": -2.450573444366455, + "logps/chosen": -223.92630004882812, + "logps/rejected": -278.1924133300781, + "loss": 0.2113, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.5321887731552124, + "rewards/margins": 3.711320638656616, + "rewards/rejected": -5.243509292602539, + "step": 1321 + }, + { + "epoch": 1.73, + "learning_rate": 4.682892164852058e-05, + "logits/chosen": -2.2027530670166016, + "logits/rejected": -2.214686632156372, + "logps/chosen": -239.34274291992188, + "logps/rejected": -282.7929992675781, + "loss": 0.2324, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.4780960083007812, + "rewards/margins": 3.398163318634033, + "rewards/rejected": -4.8762593269348145, + "step": 1322 + }, + { + "epoch": 1.73, + "learning_rate": 4.68238423806212e-05, + "logits/chosen": -2.098435640335083, + "logits/rejected": -2.160342216491699, + "logps/chosen": -175.86923217773438, + "logps/rejected": -210.9672393798828, + "loss": 0.2131, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5760440826416016, + "rewards/margins": 2.946066379547119, + "rewards/rejected": -4.522110462188721, + "step": 1323 + }, + { + "epoch": 1.73, + "learning_rate": 4.681875932403885e-05, + "logits/chosen": -2.0780820846557617, + "logits/rejected": -2.2287166118621826, + "logps/chosen": -220.93051147460938, + "logps/rejected": -261.7267761230469, + "loss": 0.1903, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.6689494848251343, + "rewards/margins": 3.8627724647521973, + "rewards/rejected": -5.531722068786621, + "step": 1324 + }, + { + "epoch": 1.73, + "learning_rate": 4.6813672479656e-05, + "logits/chosen": -2.196786642074585, + "logits/rejected": -2.3375864028930664, + "logps/chosen": -191.28599548339844, + "logps/rejected": -281.8190612792969, + "loss": 0.1382, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.868288278579712, + "rewards/margins": 4.053330898284912, + "rewards/rejected": -5.921619415283203, + "step": 1325 + }, + { + "epoch": 1.74, + "learning_rate": 4.680858184835572e-05, + "logits/chosen": -2.0055956840515137, + "logits/rejected": -2.090541124343872, + "logps/chosen": -181.52813720703125, + "logps/rejected": -213.05496215820312, + "loss": 0.171, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.124434232711792, + "rewards/margins": 3.510894536972046, + "rewards/rejected": -4.635329246520996, + "step": 1326 + }, + { + "epoch": 1.74, + "learning_rate": 4.6803487431021766e-05, + "logits/chosen": -2.1428966522216797, + "logits/rejected": -2.2327733039855957, + "logps/chosen": -174.168701171875, + "logps/rejected": -196.87881469726562, + "loss": 0.222, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.8680024147033691, + "rewards/margins": 3.048375129699707, + "rewards/rejected": -4.916378021240234, + "step": 1327 + }, + { + "epoch": 1.74, + "learning_rate": 4.679838922853853e-05, + "logits/chosen": -2.3784139156341553, + "logits/rejected": -2.481027126312256, + "logps/chosen": -230.0206298828125, + "logps/rejected": -294.965087890625, + "loss": 0.168, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.9025079011917114, + "rewards/margins": 4.027909278869629, + "rewards/rejected": -5.930417537689209, + "step": 1328 + }, + { + "epoch": 1.74, + "learning_rate": 4.679328724179109e-05, + "logits/chosen": -2.269122362136841, + "logits/rejected": -2.279000997543335, + "logps/chosen": -191.70877075195312, + "logps/rejected": -215.52420043945312, + "loss": 0.1554, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.063866138458252, + "rewards/margins": 3.1374106407165527, + "rewards/rejected": -5.201276779174805, + "step": 1329 + }, + { + "epoch": 1.74, + "learning_rate": 4.6788181471665155e-05, + "logits/chosen": -1.8508931398391724, + "logits/rejected": -1.8643546104431152, + "logps/chosen": -205.4655303955078, + "logps/rejected": -203.88031005859375, + "loss": 0.1438, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.508162021636963, + "rewards/margins": 3.4267380237579346, + "rewards/rejected": -4.934900283813477, + "step": 1330 + }, + { + "epoch": 1.74, + "learning_rate": 4.67830719190471e-05, + "logits/chosen": -2.4449105262756348, + "logits/rejected": -2.512434720993042, + "logps/chosen": -239.13856506347656, + "logps/rejected": -255.4873046875, + "loss": 0.1123, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7588633298873901, + "rewards/margins": 4.075693607330322, + "rewards/rejected": -5.83455753326416, + "step": 1331 + }, + { + "epoch": 1.74, + "learning_rate": 4.677795858482398e-05, + "logits/chosen": -2.2655084133148193, + "logits/rejected": -2.2468154430389404, + "logps/chosen": -166.35696411132812, + "logps/rejected": -241.24954223632812, + "loss": 0.1517, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.7497742176055908, + "rewards/margins": 3.5795230865478516, + "rewards/rejected": -5.3292975425720215, + "step": 1332 + }, + { + "epoch": 1.74, + "learning_rate": 4.677284146988346e-05, + "logits/chosen": -2.377413749694824, + "logits/rejected": -2.4802167415618896, + "logps/chosen": -257.7947998046875, + "logps/rejected": -343.67376708984375, + "loss": 0.0402, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.074228048324585, + "rewards/margins": 5.048233985900879, + "rewards/rejected": -7.122462272644043, + "step": 1333 + }, + { + "epoch": 1.75, + "learning_rate": 4.67677205751139e-05, + "logits/chosen": -2.3548061847686768, + "logits/rejected": -2.4688026905059814, + "logps/chosen": -202.11593627929688, + "logps/rejected": -286.65057373046875, + "loss": 0.1916, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.193711519241333, + "rewards/margins": 5.112841606140137, + "rewards/rejected": -6.306553840637207, + "step": 1334 + }, + { + "epoch": 1.75, + "learning_rate": 4.67625959014043e-05, + "logits/chosen": -2.31046199798584, + "logits/rejected": -2.319591760635376, + "logps/chosen": -231.41473388671875, + "logps/rejected": -231.21859741210938, + "loss": 0.1506, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5231584310531616, + "rewards/margins": 3.534045934677124, + "rewards/rejected": -5.057204723358154, + "step": 1335 + }, + { + "epoch": 1.75, + "learning_rate": 4.675746744964432e-05, + "logits/chosen": -2.524745464324951, + "logits/rejected": -2.5354037284851074, + "logps/chosen": -280.01513671875, + "logps/rejected": -341.6401672363281, + "loss": 0.1042, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.0203945636749268, + "rewards/margins": 4.112698554992676, + "rewards/rejected": -6.133092880249023, + "step": 1336 + }, + { + "epoch": 1.75, + "learning_rate": 4.675233522072426e-05, + "logits/chosen": -2.4641785621643066, + "logits/rejected": -2.4546477794647217, + "logps/chosen": -204.9468536376953, + "logps/rejected": -250.145263671875, + "loss": 0.1739, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.4312738180160522, + "rewards/margins": 3.9612386226654053, + "rewards/rejected": -5.392512798309326, + "step": 1337 + }, + { + "epoch": 1.75, + "learning_rate": 4.674719921553511e-05, + "logits/chosen": -2.3155698776245117, + "logits/rejected": -2.3693156242370605, + "logps/chosen": -194.86477661132812, + "logps/rejected": -220.62965393066406, + "loss": 0.1797, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.0034408569335938, + "rewards/margins": 3.511406898498535, + "rewards/rejected": -5.514848709106445, + "step": 1338 + }, + { + "epoch": 1.75, + "learning_rate": 4.6742059434968476e-05, + "logits/chosen": -1.9812606573104858, + "logits/rejected": -2.0393075942993164, + "logps/chosen": -197.7154541015625, + "logps/rejected": -251.43502807617188, + "loss": 0.2036, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.391548991203308, + "rewards/margins": 3.891415596008301, + "rewards/rejected": -5.282964706420898, + "step": 1339 + }, + { + "epoch": 1.75, + "learning_rate": 4.673691587991667e-05, + "logits/chosen": -2.2613768577575684, + "logits/rejected": -2.2857589721679688, + "logps/chosen": -211.1699981689453, + "logps/rejected": -271.5691833496094, + "loss": 0.3857, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.122715950012207, + "rewards/margins": 3.556459903717041, + "rewards/rejected": -5.679176330566406, + "step": 1340 + }, + { + "epoch": 1.75, + "learning_rate": 4.673176855127258e-05, + "logits/chosen": -2.3039798736572266, + "logits/rejected": -2.286431312561035, + "logps/chosen": -209.6785430908203, + "logps/rejected": -263.2655334472656, + "loss": 0.182, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.8972321152687073, + "rewards/margins": 4.346026420593262, + "rewards/rejected": -5.243258953094482, + "step": 1341 + }, + { + "epoch": 1.76, + "learning_rate": 4.672661744992984e-05, + "logits/chosen": -2.2901010513305664, + "logits/rejected": -2.3650054931640625, + "logps/chosen": -195.44326782226562, + "logps/rejected": -268.2812805175781, + "loss": 0.1694, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.497087001800537, + "rewards/margins": 4.011203289031982, + "rewards/rejected": -5.5082902908325195, + "step": 1342 + }, + { + "epoch": 1.76, + "learning_rate": 4.672146257678268e-05, + "logits/chosen": -2.4022762775421143, + "logits/rejected": -2.3658864498138428, + "logps/chosen": -255.15890502929688, + "logps/rejected": -328.7245788574219, + "loss": 0.1034, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.900804877281189, + "rewards/margins": 4.233809471130371, + "rewards/rejected": -6.13461446762085, + "step": 1343 + }, + { + "epoch": 1.76, + "learning_rate": 4.6716303932726004e-05, + "logits/chosen": -2.4683516025543213, + "logits/rejected": -2.4238386154174805, + "logps/chosen": -181.5071563720703, + "logps/rejected": -225.39805603027344, + "loss": 0.2542, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.721781849861145, + "rewards/margins": 3.4932799339294434, + "rewards/rejected": -5.215061664581299, + "step": 1344 + }, + { + "epoch": 1.76, + "learning_rate": 4.671114151865536e-05, + "logits/chosen": -2.38026762008667, + "logits/rejected": -2.38236927986145, + "logps/chosen": -227.99789428710938, + "logps/rejected": -277.05975341796875, + "loss": 0.1147, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6720448732376099, + "rewards/margins": 4.081615447998047, + "rewards/rejected": -5.753660202026367, + "step": 1345 + }, + { + "epoch": 1.76, + "learning_rate": 4.6705975335466966e-05, + "logits/chosen": -2.504228115081787, + "logits/rejected": -2.5582098960876465, + "logps/chosen": -272.26788330078125, + "logps/rejected": -299.64056396484375, + "loss": 0.144, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.06329083442688, + "rewards/margins": 3.614590644836426, + "rewards/rejected": -5.677881240844727, + "step": 1346 + }, + { + "epoch": 1.76, + "learning_rate": 4.6700805384057687e-05, + "logits/chosen": -2.207796096801758, + "logits/rejected": -2.2440619468688965, + "logps/chosen": -251.6385040283203, + "logps/rejected": -267.8268127441406, + "loss": 0.0849, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2752389907836914, + "rewards/margins": 3.857135772705078, + "rewards/rejected": -5.1323747634887695, + "step": 1347 + }, + { + "epoch": 1.76, + "learning_rate": 4.669563166532503e-05, + "logits/chosen": -2.1402127742767334, + "logits/rejected": -2.2065505981445312, + "logps/chosen": -188.67706298828125, + "logps/rejected": -221.7375946044922, + "loss": 0.1975, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9871059656143188, + "rewards/margins": 2.8594772815704346, + "rewards/rejected": -4.846583366394043, + "step": 1348 + }, + { + "epoch": 1.77, + "learning_rate": 4.669045418016719e-05, + "logits/chosen": -2.2698256969451904, + "logits/rejected": -2.274237871170044, + "logps/chosen": -197.71026611328125, + "logps/rejected": -252.61614990234375, + "loss": 0.1191, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6768132448196411, + "rewards/margins": 3.3656845092773438, + "rewards/rejected": -5.042497634887695, + "step": 1349 + }, + { + "epoch": 1.77, + "learning_rate": 4.668527292948298e-05, + "logits/chosen": -2.256514072418213, + "logits/rejected": -2.34165358543396, + "logps/chosen": -176.2838592529297, + "logps/rejected": -236.6180877685547, + "loss": 0.1925, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.650253176689148, + "rewards/margins": 3.62697172164917, + "rewards/rejected": -5.277225017547607, + "step": 1350 + }, + { + "epoch": 1.77, + "learning_rate": 4.668008791417188e-05, + "logits/chosen": -2.31141996383667, + "logits/rejected": -2.3515350818634033, + "logps/chosen": -238.41036987304688, + "logps/rejected": -284.0400390625, + "loss": 0.0897, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.6227517127990723, + "rewards/margins": 4.4259443283081055, + "rewards/rejected": -6.048696041107178, + "step": 1351 + }, + { + "epoch": 1.77, + "learning_rate": 4.6674899135134024e-05, + "logits/chosen": -2.0742459297180176, + "logits/rejected": -2.1106269359588623, + "logps/chosen": -172.02809143066406, + "logps/rejected": -193.6383514404297, + "loss": 0.237, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.9648047089576721, + "rewards/margins": 2.872191905975342, + "rewards/rejected": -3.836996555328369, + "step": 1352 + }, + { + "epoch": 1.77, + "learning_rate": 4.666970659327019e-05, + "logits/chosen": -1.8485875129699707, + "logits/rejected": -1.9281960725784302, + "logps/chosen": -219.42877197265625, + "logps/rejected": -291.0449523925781, + "loss": 0.1646, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.602189540863037, + "rewards/margins": 3.566793441772461, + "rewards/rejected": -5.168982982635498, + "step": 1353 + }, + { + "epoch": 1.77, + "learning_rate": 4.666451028948183e-05, + "logits/chosen": -2.3288824558258057, + "logits/rejected": -2.427269458770752, + "logps/chosen": -238.8002471923828, + "logps/rejected": -297.3604431152344, + "loss": 0.0734, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7097375392913818, + "rewards/margins": 4.595492362976074, + "rewards/rejected": -6.305230140686035, + "step": 1354 + }, + { + "epoch": 1.77, + "learning_rate": 4.665931022467105e-05, + "logits/chosen": -2.096285104751587, + "logits/rejected": -2.1877405643463135, + "logps/chosen": -181.7442169189453, + "logps/rejected": -265.5242614746094, + "loss": 0.1665, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.873305320739746, + "rewards/margins": 3.2641849517822266, + "rewards/rejected": -5.137490272521973, + "step": 1355 + }, + { + "epoch": 1.77, + "learning_rate": 4.665410639974057e-05, + "logits/chosen": -2.1947834491729736, + "logits/rejected": -2.359546422958374, + "logps/chosen": -203.13490295410156, + "logps/rejected": -254.12127685546875, + "loss": 0.189, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1950178146362305, + "rewards/margins": 4.417459487915039, + "rewards/rejected": -5.612477779388428, + "step": 1356 + }, + { + "epoch": 1.78, + "learning_rate": 4.664889881559381e-05, + "logits/chosen": -2.353022336959839, + "logits/rejected": -2.4439656734466553, + "logps/chosen": -230.41729736328125, + "logps/rejected": -242.9822998046875, + "loss": 0.0859, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2911330461502075, + "rewards/margins": 4.007154941558838, + "rewards/rejected": -5.298287391662598, + "step": 1357 + }, + { + "epoch": 1.78, + "learning_rate": 4.66436874731348e-05, + "logits/chosen": -2.055748462677002, + "logits/rejected": -2.069406509399414, + "logps/chosen": -215.91287231445312, + "logps/rejected": -224.975341796875, + "loss": 0.1346, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.132076382637024, + "rewards/margins": 3.4015748500823975, + "rewards/rejected": -4.533651351928711, + "step": 1358 + }, + { + "epoch": 1.78, + "learning_rate": 4.663847237326827e-05, + "logits/chosen": -2.2017364501953125, + "logits/rejected": -2.1791625022888184, + "logps/chosen": -195.00912475585938, + "logps/rejected": -187.93539428710938, + "loss": 0.2966, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.539417028427124, + "rewards/margins": 2.387049674987793, + "rewards/rejected": -3.926466703414917, + "step": 1359 + }, + { + "epoch": 1.78, + "learning_rate": 4.663325351689956e-05, + "logits/chosen": -2.2987170219421387, + "logits/rejected": -2.302344560623169, + "logps/chosen": -186.64981079101562, + "logps/rejected": -223.81967163085938, + "loss": 0.182, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5449962615966797, + "rewards/margins": 3.964876413345337, + "rewards/rejected": -5.509873390197754, + "step": 1360 + }, + { + "epoch": 1.78, + "learning_rate": 4.662803090493469e-05, + "logits/chosen": -2.1506714820861816, + "logits/rejected": -2.18817400932312, + "logps/chosen": -257.8224792480469, + "logps/rejected": -328.1229248046875, + "loss": 0.1701, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.0192809104919434, + "rewards/margins": 3.9228391647338867, + "rewards/rejected": -5.94212007522583, + "step": 1361 + }, + { + "epoch": 1.78, + "learning_rate": 4.6622804538280305e-05, + "logits/chosen": -2.2823166847229004, + "logits/rejected": -2.34694242477417, + "logps/chosen": -195.22393798828125, + "logps/rejected": -255.1166534423828, + "loss": 0.1154, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.038851261138916, + "rewards/margins": 3.4105467796325684, + "rewards/rejected": -5.449398040771484, + "step": 1362 + }, + { + "epoch": 1.78, + "learning_rate": 4.661757441784373e-05, + "logits/chosen": -2.2843942642211914, + "logits/rejected": -2.220759630203247, + "logps/chosen": -219.33355712890625, + "logps/rejected": -290.8352966308594, + "loss": 0.0939, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6054083108901978, + "rewards/margins": 4.078475475311279, + "rewards/rejected": -5.6838836669921875, + "step": 1363 + }, + { + "epoch": 1.79, + "learning_rate": 4.6612340544532935e-05, + "logits/chosen": -2.2316360473632812, + "logits/rejected": -2.3092989921569824, + "logps/chosen": -200.7064971923828, + "logps/rejected": -254.74070739746094, + "loss": 0.1406, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5896083116531372, + "rewards/margins": 3.663377285003662, + "rewards/rejected": -5.252985954284668, + "step": 1364 + }, + { + "epoch": 1.79, + "learning_rate": 4.660710291925652e-05, + "logits/chosen": -2.365083694458008, + "logits/rejected": -2.388625144958496, + "logps/chosen": -230.10348510742188, + "logps/rejected": -289.7381286621094, + "loss": 0.1506, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6961638927459717, + "rewards/margins": 3.8952834606170654, + "rewards/rejected": -5.591447830200195, + "step": 1365 + }, + { + "epoch": 1.79, + "learning_rate": 4.660186154292375e-05, + "logits/chosen": -2.2262802124023438, + "logits/rejected": -2.279958486557007, + "logps/chosen": -163.28163146972656, + "logps/rejected": -202.38223266601562, + "loss": 0.2373, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6737639904022217, + "rewards/margins": 3.0033376216888428, + "rewards/rejected": -4.6771016120910645, + "step": 1366 + }, + { + "epoch": 1.79, + "learning_rate": 4.659661641644456e-05, + "logits/chosen": -2.447139263153076, + "logits/rejected": -2.4368622303009033, + "logps/chosen": -242.7307891845703, + "logps/rejected": -283.7666320800781, + "loss": 0.1133, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5830082893371582, + "rewards/margins": 5.0759172439575195, + "rewards/rejected": -6.6589250564575195, + "step": 1367 + }, + { + "epoch": 1.79, + "learning_rate": 4.6591367540729515e-05, + "logits/chosen": -2.524048328399658, + "logits/rejected": -2.555225133895874, + "logps/chosen": -243.4622039794922, + "logps/rejected": -252.66217041015625, + "loss": 0.1873, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.028730869293213, + "rewards/margins": 3.0651321411132812, + "rewards/rejected": -5.093863487243652, + "step": 1368 + }, + { + "epoch": 1.79, + "learning_rate": 4.658611491668983e-05, + "logits/chosen": -2.3555378913879395, + "logits/rejected": -2.4342939853668213, + "logps/chosen": -194.3604736328125, + "logps/rejected": -248.84271240234375, + "loss": 0.1389, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7463260889053345, + "rewards/margins": 3.3878800868988037, + "rewards/rejected": -5.1342058181762695, + "step": 1369 + }, + { + "epoch": 1.79, + "learning_rate": 4.658085854523737e-05, + "logits/chosen": -2.1884450912475586, + "logits/rejected": -2.365642786026001, + "logps/chosen": -185.33209228515625, + "logps/rejected": -265.621337890625, + "loss": 0.1218, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.218601942062378, + "rewards/margins": 3.699212074279785, + "rewards/rejected": -5.917815208435059, + "step": 1370 + }, + { + "epoch": 1.79, + "learning_rate": 4.657559842728467e-05, + "logits/chosen": -2.103135347366333, + "logits/rejected": -2.149179697036743, + "logps/chosen": -214.1287841796875, + "logps/rejected": -276.39312744140625, + "loss": 0.1939, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.805924654006958, + "rewards/margins": 4.1208319664001465, + "rewards/rejected": -5.926756381988525, + "step": 1371 + }, + { + "epoch": 1.8, + "learning_rate": 4.657033456374489e-05, + "logits/chosen": -1.8944284915924072, + "logits/rejected": -1.9346532821655273, + "logps/chosen": -216.1498565673828, + "logps/rejected": -308.89453125, + "loss": 0.109, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.7429003715515137, + "rewards/margins": 4.299877643585205, + "rewards/rejected": -6.0427775382995605, + "step": 1372 + }, + { + "epoch": 1.8, + "learning_rate": 4.6565066955531863e-05, + "logits/chosen": -1.657898187637329, + "logits/rejected": -1.6330841779708862, + "logps/chosen": -213.9786834716797, + "logps/rejected": -256.35113525390625, + "loss": 0.3878, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.641648054122925, + "rewards/margins": 2.252370834350586, + "rewards/rejected": -4.89401912689209, + "step": 1373 + }, + { + "epoch": 1.8, + "learning_rate": 4.655979560356005e-05, + "logits/chosen": -2.202993154525757, + "logits/rejected": -2.2177088260650635, + "logps/chosen": -236.91598510742188, + "logps/rejected": -295.3562927246094, + "loss": 0.033, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.0502257347106934, + "rewards/margins": 5.525638103485107, + "rewards/rejected": -7.575864315032959, + "step": 1374 + }, + { + "epoch": 1.8, + "learning_rate": 4.655452050874459e-05, + "logits/chosen": -2.14239501953125, + "logits/rejected": -2.1285078525543213, + "logps/chosen": -187.77191162109375, + "logps/rejected": -228.86451721191406, + "loss": 0.214, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.5740537643432617, + "rewards/margins": 3.127923011779785, + "rewards/rejected": -5.701977252960205, + "step": 1375 + }, + { + "epoch": 1.8, + "learning_rate": 4.654924167200123e-05, + "logits/chosen": -2.3964757919311523, + "logits/rejected": -2.476527214050293, + "logps/chosen": -214.957763671875, + "logps/rejected": -286.00726318359375, + "loss": 0.1093, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0560764074325562, + "rewards/margins": 5.730381965637207, + "rewards/rejected": -6.786458969116211, + "step": 1376 + }, + { + "epoch": 1.8, + "learning_rate": 4.6543959094246417e-05, + "logits/chosen": -2.488255739212036, + "logits/rejected": -2.6213529109954834, + "logps/chosen": -249.02215576171875, + "logps/rejected": -291.32379150390625, + "loss": 0.1211, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5869184732437134, + "rewards/margins": 4.215513706207275, + "rewards/rejected": -5.802432060241699, + "step": 1377 + }, + { + "epoch": 1.8, + "learning_rate": 4.653867277639721e-05, + "logits/chosen": -2.311802387237549, + "logits/rejected": -2.3098201751708984, + "logps/chosen": -172.78297424316406, + "logps/rejected": -222.96755981445312, + "loss": 0.1179, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.1166205406188965, + "rewards/margins": 3.5452730655670166, + "rewards/rejected": -5.661893367767334, + "step": 1378 + }, + { + "epoch": 1.8, + "learning_rate": 4.653338271937132e-05, + "logits/chosen": -2.316065788269043, + "logits/rejected": -2.3509740829467773, + "logps/chosen": -202.41773986816406, + "logps/rejected": -270.28985595703125, + "loss": 0.1453, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.8998184204101562, + "rewards/margins": 3.892240524291992, + "rewards/rejected": -5.79205846786499, + "step": 1379 + }, + { + "epoch": 1.81, + "learning_rate": 4.6528088924087134e-05, + "logits/chosen": -2.3619024753570557, + "logits/rejected": -2.308654308319092, + "logps/chosen": -263.7723083496094, + "logps/rejected": -302.5215759277344, + "loss": 0.2607, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.3372600078582764, + "rewards/margins": 3.7634713649749756, + "rewards/rejected": -6.100731372833252, + "step": 1380 + }, + { + "epoch": 1.81, + "learning_rate": 4.652279139146366e-05, + "logits/chosen": -2.067979097366333, + "logits/rejected": -2.1202504634857178, + "logps/chosen": -210.7864532470703, + "logps/rejected": -255.50363159179688, + "loss": 0.1086, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.8815836906433105, + "rewards/margins": 4.555849552154541, + "rewards/rejected": -6.437433242797852, + "step": 1381 + }, + { + "epoch": 1.81, + "learning_rate": 4.651749012242057e-05, + "logits/chosen": -2.3216161727905273, + "logits/rejected": -2.276050329208374, + "logps/chosen": -210.85401916503906, + "logps/rejected": -233.73196411132812, + "loss": 0.311, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.473651885986328, + "rewards/margins": 2.5943374633789062, + "rewards/rejected": -5.067989349365234, + "step": 1382 + }, + { + "epoch": 1.81, + "learning_rate": 4.6512185117878184e-05, + "logits/chosen": -2.3193976879119873, + "logits/rejected": -2.3813366889953613, + "logps/chosen": -205.80455017089844, + "logps/rejected": -243.294921875, + "loss": 0.2155, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.865469217300415, + "rewards/margins": 4.035120487213135, + "rewards/rejected": -5.900589466094971, + "step": 1383 + }, + { + "epoch": 1.81, + "learning_rate": 4.6506876378757445e-05, + "logits/chosen": -2.170536994934082, + "logits/rejected": -2.177908182144165, + "logps/chosen": -169.406494140625, + "logps/rejected": -262.20025634765625, + "loss": 0.1339, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.5915058851242065, + "rewards/margins": 4.764673233032227, + "rewards/rejected": -6.356178283691406, + "step": 1384 + }, + { + "epoch": 1.81, + "learning_rate": 4.650156390598e-05, + "logits/chosen": -1.956441879272461, + "logits/rejected": -1.9708319902420044, + "logps/chosen": -194.17105102539062, + "logps/rejected": -242.59666442871094, + "loss": 0.2926, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.0661697387695312, + "rewards/margins": 3.1960580348968506, + "rewards/rejected": -5.262228012084961, + "step": 1385 + }, + { + "epoch": 1.81, + "learning_rate": 4.6496247700468064e-05, + "logits/chosen": -2.279972553253174, + "logits/rejected": -2.320099115371704, + "logps/chosen": -180.3889617919922, + "logps/rejected": -240.9708709716797, + "loss": 0.1561, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.7037296295166016, + "rewards/margins": 3.612116575241089, + "rewards/rejected": -5.3158464431762695, + "step": 1386 + }, + { + "epoch": 1.82, + "learning_rate": 4.6490927763144586e-05, + "logits/chosen": -2.3013417720794678, + "logits/rejected": -2.2452809810638428, + "logps/chosen": -198.49270629882812, + "logps/rejected": -250.34814453125, + "loss": 0.2868, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.7538549900054932, + "rewards/margins": 3.3283233642578125, + "rewards/rejected": -5.082179069519043, + "step": 1387 + }, + { + "epoch": 1.82, + "learning_rate": 4.6485604094933113e-05, + "logits/chosen": -2.6034908294677734, + "logits/rejected": -2.501837730407715, + "logps/chosen": -241.82763671875, + "logps/rejected": -259.59930419921875, + "loss": 0.226, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.8224003314971924, + "rewards/margins": 4.394843101501465, + "rewards/rejected": -6.217243671417236, + "step": 1388 + }, + { + "epoch": 1.82, + "learning_rate": 4.648027669675784e-05, + "logits/chosen": -1.8462001085281372, + "logits/rejected": -1.890073537826538, + "logps/chosen": -166.96856689453125, + "logps/rejected": -255.78610229492188, + "loss": 0.2828, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7682936191558838, + "rewards/margins": 3.327087640762329, + "rewards/rejected": -5.095381259918213, + "step": 1389 + }, + { + "epoch": 1.82, + "learning_rate": 4.647494556954363e-05, + "logits/chosen": -2.199425458908081, + "logits/rejected": -2.260338306427002, + "logps/chosen": -194.01919555664062, + "logps/rejected": -243.82003784179688, + "loss": 0.2729, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.3266735076904297, + "rewards/margins": 3.0939698219299316, + "rewards/rejected": -5.420642852783203, + "step": 1390 + }, + { + "epoch": 1.82, + "learning_rate": 4.646961071421597e-05, + "logits/chosen": -2.479255199432373, + "logits/rejected": -2.396368980407715, + "logps/chosen": -178.45083618164062, + "logps/rejected": -193.24459838867188, + "loss": 0.1679, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2833914756774902, + "rewards/margins": 2.904581308364868, + "rewards/rejected": -4.187972545623779, + "step": 1391 + }, + { + "epoch": 1.82, + "learning_rate": 4.646427213170102e-05, + "logits/chosen": -2.217855453491211, + "logits/rejected": -2.2055749893188477, + "logps/chosen": -197.3143310546875, + "logps/rejected": -273.6307067871094, + "loss": 0.2277, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.942773699760437, + "rewards/margins": 3.781911849975586, + "rewards/rejected": -4.7246856689453125, + "step": 1392 + }, + { + "epoch": 1.82, + "learning_rate": 4.6458929822925566e-05, + "logits/chosen": -2.133974552154541, + "logits/rejected": -2.2947840690612793, + "logps/chosen": -196.52053833007812, + "logps/rejected": -273.97747802734375, + "loss": 0.2357, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.2982733249664307, + "rewards/margins": 3.5180678367614746, + "rewards/rejected": -4.816340923309326, + "step": 1393 + }, + { + "epoch": 1.82, + "learning_rate": 4.645358378881704e-05, + "logits/chosen": -2.507310152053833, + "logits/rejected": -2.5766844749450684, + "logps/chosen": -179.09832763671875, + "logps/rejected": -204.78404235839844, + "loss": 0.3219, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1929552555084229, + "rewards/margins": 2.7457828521728516, + "rewards/rejected": -3.9387383460998535, + "step": 1394 + }, + { + "epoch": 1.83, + "learning_rate": 4.644823403030355e-05, + "logits/chosen": -2.13411545753479, + "logits/rejected": -2.2497849464416504, + "logps/chosen": -193.55628967285156, + "logps/rejected": -234.6313934326172, + "loss": 0.1959, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2598943710327148, + "rewards/margins": 3.595004081726074, + "rewards/rejected": -4.854898452758789, + "step": 1395 + }, + { + "epoch": 1.83, + "learning_rate": 4.644288054831381e-05, + "logits/chosen": -2.157371997833252, + "logits/rejected": -2.2184062004089355, + "logps/chosen": -171.0150146484375, + "logps/rejected": -226.55062866210938, + "loss": 0.1302, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0181159973144531, + "rewards/margins": 3.4827423095703125, + "rewards/rejected": -4.500858306884766, + "step": 1396 + }, + { + "epoch": 1.83, + "learning_rate": 4.643752334377721e-05, + "logits/chosen": -2.3214855194091797, + "logits/rejected": -2.367702007293701, + "logps/chosen": -180.56222534179688, + "logps/rejected": -226.59286499023438, + "loss": 0.1739, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2419052124023438, + "rewards/margins": 3.2632222175598145, + "rewards/rejected": -4.505127429962158, + "step": 1397 + }, + { + "epoch": 1.83, + "learning_rate": 4.643216241762377e-05, + "logits/chosen": -2.4462032318115234, + "logits/rejected": -2.4511361122131348, + "logps/chosen": -193.83444213867188, + "logps/rejected": -230.96469116210938, + "loss": 0.2973, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.9721289873123169, + "rewards/margins": 2.934811592102051, + "rewards/rejected": -3.9069406986236572, + "step": 1398 + }, + { + "epoch": 1.83, + "learning_rate": 4.642679777078417e-05, + "logits/chosen": -2.4158241748809814, + "logits/rejected": -2.453239679336548, + "logps/chosen": -198.2194061279297, + "logps/rejected": -246.531005859375, + "loss": 0.0574, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5831554532051086, + "rewards/margins": 4.099462985992432, + "rewards/rejected": -4.682618141174316, + "step": 1399 + }, + { + "epoch": 1.83, + "learning_rate": 4.642142940418973e-05, + "logits/chosen": -2.476083517074585, + "logits/rejected": -2.496399402618408, + "logps/chosen": -254.40286254882812, + "logps/rejected": -267.8065490722656, + "loss": 0.1294, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.5694355964660645, + "rewards/margins": 4.015815734863281, + "rewards/rejected": -4.585251808166504, + "step": 1400 + }, + { + "epoch": 1.83, + "learning_rate": 4.641605731877241e-05, + "logits/chosen": -2.420027256011963, + "logits/rejected": -2.445709466934204, + "logps/chosen": -193.45823669433594, + "logps/rejected": -229.07589721679688, + "loss": 0.1181, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.91693115234375, + "rewards/margins": 3.285738706588745, + "rewards/rejected": -4.202670097351074, + "step": 1401 + }, + { + "epoch": 1.83, + "learning_rate": 4.6410681515464815e-05, + "logits/chosen": -2.453171491622925, + "logits/rejected": -2.4817469120025635, + "logps/chosen": -167.9488525390625, + "logps/rejected": -211.94888305664062, + "loss": 0.2742, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.7758989334106445, + "rewards/margins": 2.3340702056884766, + "rewards/rejected": -3.109968900680542, + "step": 1402 + }, + { + "epoch": 1.84, + "learning_rate": 4.640530199520021e-05, + "logits/chosen": -2.478593587875366, + "logits/rejected": -2.4444589614868164, + "logps/chosen": -274.9038391113281, + "logps/rejected": -317.6105041503906, + "loss": 0.1598, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0752410888671875, + "rewards/margins": 3.3734335899353027, + "rewards/rejected": -4.44867467880249, + "step": 1403 + }, + { + "epoch": 1.84, + "learning_rate": 4.639991875891248e-05, + "logits/chosen": -1.9536941051483154, + "logits/rejected": -2.0602893829345703, + "logps/chosen": -164.46133422851562, + "logps/rejected": -239.60308837890625, + "loss": 0.1324, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1708040237426758, + "rewards/margins": 3.239039659500122, + "rewards/rejected": -4.409843444824219, + "step": 1404 + }, + { + "epoch": 1.84, + "learning_rate": 4.639453180753619e-05, + "logits/chosen": -2.305426597595215, + "logits/rejected": -2.310289144515991, + "logps/chosen": -179.86114501953125, + "logps/rejected": -254.47314453125, + "loss": 0.1235, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.4751981496810913, + "rewards/margins": 4.430361270904541, + "rewards/rejected": -4.905559539794922, + "step": 1405 + }, + { + "epoch": 1.84, + "learning_rate": 4.638914114200652e-05, + "logits/chosen": -1.8774938583374023, + "logits/rejected": -1.9185031652450562, + "logps/chosen": -173.3539276123047, + "logps/rejected": -220.61851501464844, + "loss": 0.2646, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2160310745239258, + "rewards/margins": 2.826627016067505, + "rewards/rejected": -4.042657852172852, + "step": 1406 + }, + { + "epoch": 1.84, + "learning_rate": 4.638374676325931e-05, + "logits/chosen": -2.312690496444702, + "logits/rejected": -2.472609519958496, + "logps/chosen": -178.3687744140625, + "logps/rejected": -245.68804931640625, + "loss": 0.1414, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.2376123666763306, + "rewards/margins": 3.6323118209838867, + "rewards/rejected": -4.8699235916137695, + "step": 1407 + }, + { + "epoch": 1.84, + "learning_rate": 4.637834867223102e-05, + "logits/chosen": -2.4253156185150146, + "logits/rejected": -2.5353803634643555, + "logps/chosen": -192.8672332763672, + "logps/rejected": -227.572265625, + "loss": 0.1833, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8559720516204834, + "rewards/margins": 3.2029502391815186, + "rewards/rejected": -4.05892276763916, + "step": 1408 + }, + { + "epoch": 1.84, + "learning_rate": 4.63729468698588e-05, + "logits/chosen": -2.4336678981781006, + "logits/rejected": -2.5701303482055664, + "logps/chosen": -169.47442626953125, + "logps/rejected": -245.16151428222656, + "loss": 0.1092, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9117591381072998, + "rewards/margins": 3.5345542430877686, + "rewards/rejected": -4.446313381195068, + "step": 1409 + }, + { + "epoch": 1.85, + "learning_rate": 4.636754135708041e-05, + "logits/chosen": -2.405302047729492, + "logits/rejected": -2.4277071952819824, + "logps/chosen": -194.17935180664062, + "logps/rejected": -215.50399780273438, + "loss": 0.2359, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0027927160263062, + "rewards/margins": 3.0121240615844727, + "rewards/rejected": -4.014916896820068, + "step": 1410 + }, + { + "epoch": 1.85, + "learning_rate": 4.636213213483427e-05, + "logits/chosen": -2.4292078018188477, + "logits/rejected": -2.4207870960235596, + "logps/chosen": -178.75831604003906, + "logps/rejected": -193.9530029296875, + "loss": 0.2236, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.8789994716644287, + "rewards/margins": 2.7491774559020996, + "rewards/rejected": -3.62817645072937, + "step": 1411 + }, + { + "epoch": 1.85, + "learning_rate": 4.635671920405942e-05, + "logits/chosen": -2.354250192642212, + "logits/rejected": -2.4390838146209717, + "logps/chosen": -170.90542602539062, + "logps/rejected": -211.04017639160156, + "loss": 0.1892, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.9057228565216064, + "rewards/margins": 2.9422061443328857, + "rewards/rejected": -3.847928762435913, + "step": 1412 + }, + { + "epoch": 1.85, + "learning_rate": 4.635130256569558e-05, + "logits/chosen": -2.297285795211792, + "logits/rejected": -2.392590045928955, + "logps/chosen": -168.90338134765625, + "logps/rejected": -240.53675842285156, + "loss": 0.1883, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.4901635646820068, + "rewards/margins": 3.915353775024414, + "rewards/rejected": -5.405517578125, + "step": 1413 + }, + { + "epoch": 1.85, + "learning_rate": 4.634588222068307e-05, + "logits/chosen": -2.1840977668762207, + "logits/rejected": -2.1191656589508057, + "logps/chosen": -253.16091918945312, + "logps/rejected": -288.4150695800781, + "loss": 0.2244, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.961954653263092, + "rewards/margins": 3.244297504425049, + "rewards/rejected": -4.206252098083496, + "step": 1414 + }, + { + "epoch": 1.85, + "learning_rate": 4.63404581699629e-05, + "logits/chosen": -2.4387269020080566, + "logits/rejected": -2.4661765098571777, + "logps/chosen": -205.93373107910156, + "logps/rejected": -220.48907470703125, + "loss": 0.3181, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.2564948797225952, + "rewards/margins": 2.7682459354400635, + "rewards/rejected": -4.024740695953369, + "step": 1415 + }, + { + "epoch": 1.85, + "learning_rate": 4.633503041447669e-05, + "logits/chosen": -2.3245558738708496, + "logits/rejected": -2.4340667724609375, + "logps/chosen": -189.07936096191406, + "logps/rejected": -247.6138916015625, + "loss": 0.1197, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.9286269545555115, + "rewards/margins": 4.146144390106201, + "rewards/rejected": -5.074770927429199, + "step": 1416 + }, + { + "epoch": 1.85, + "learning_rate": 4.632959895516672e-05, + "logits/chosen": -2.2829172611236572, + "logits/rejected": -2.447025775909424, + "logps/chosen": -224.01718139648438, + "logps/rejected": -285.6311950683594, + "loss": 0.1772, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.3857183456420898, + "rewards/margins": 3.0930581092834473, + "rewards/rejected": -4.478775978088379, + "step": 1417 + }, + { + "epoch": 1.86, + "learning_rate": 4.6324163792975906e-05, + "logits/chosen": -2.3150711059570312, + "logits/rejected": -2.3638498783111572, + "logps/chosen": -239.34747314453125, + "logps/rejected": -272.7017822265625, + "loss": 0.124, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.116297721862793, + "rewards/margins": 4.121063232421875, + "rewards/rejected": -5.237360954284668, + "step": 1418 + }, + { + "epoch": 1.86, + "learning_rate": 4.63187249288478e-05, + "logits/chosen": -2.2839043140411377, + "logits/rejected": -2.2470085620880127, + "logps/chosen": -180.84278869628906, + "logps/rejected": -192.83535766601562, + "loss": 0.2197, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1662708520889282, + "rewards/margins": 2.785388946533203, + "rewards/rejected": -3.951659917831421, + "step": 1419 + }, + { + "epoch": 1.86, + "learning_rate": 4.631328236372662e-05, + "logits/chosen": -2.356921672821045, + "logits/rejected": -2.308830738067627, + "logps/chosen": -184.10110473632812, + "logps/rejected": -209.7080078125, + "loss": 0.2305, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.795234203338623, + "rewards/margins": 2.5254967212677, + "rewards/rejected": -4.320731163024902, + "step": 1420 + }, + { + "epoch": 1.86, + "learning_rate": 4.630783609855719e-05, + "logits/chosen": -1.8619661331176758, + "logits/rejected": -1.8003392219543457, + "logps/chosen": -173.55296325683594, + "logps/rejected": -202.5664520263672, + "loss": 0.2976, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.6667727828025818, + "rewards/margins": 3.061650037765503, + "rewards/rejected": -3.7284226417541504, + "step": 1421 + }, + { + "epoch": 1.86, + "learning_rate": 4.6302386134285026e-05, + "logits/chosen": -2.434025287628174, + "logits/rejected": -2.4339070320129395, + "logps/chosen": -221.09487915039062, + "logps/rejected": -241.01637268066406, + "loss": 0.2679, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.9217256307601929, + "rewards/margins": 2.6306025981903076, + "rewards/rejected": -4.552328109741211, + "step": 1422 + }, + { + "epoch": 1.86, + "learning_rate": 4.629693247185624e-05, + "logits/chosen": -2.406184673309326, + "logits/rejected": -2.3965976238250732, + "logps/chosen": -233.80032348632812, + "logps/rejected": -279.02288818359375, + "loss": 0.2578, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.3796286582946777, + "rewards/margins": 3.7771825790405273, + "rewards/rejected": -5.156811237335205, + "step": 1423 + }, + { + "epoch": 1.86, + "learning_rate": 4.629147511221759e-05, + "logits/chosen": -2.2653684616088867, + "logits/rejected": -2.3729100227355957, + "logps/chosen": -148.75238037109375, + "logps/rejected": -179.93528747558594, + "loss": 0.1934, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1495842933654785, + "rewards/margins": 2.708228349685669, + "rewards/rejected": -3.8578124046325684, + "step": 1424 + }, + { + "epoch": 1.86, + "learning_rate": 4.628601405631652e-05, + "logits/chosen": -2.2800092697143555, + "logits/rejected": -2.31841778755188, + "logps/chosen": -181.82470703125, + "logps/rejected": -225.3800506591797, + "loss": 0.1134, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0458605289459229, + "rewards/margins": 3.472771406173706, + "rewards/rejected": -4.518631458282471, + "step": 1425 + }, + { + "epoch": 1.87, + "learning_rate": 4.6280549305101063e-05, + "logits/chosen": -2.3783209323883057, + "logits/rejected": -2.4350523948669434, + "logps/chosen": -259.91650390625, + "logps/rejected": -312.69049072265625, + "loss": 0.1717, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.517119288444519, + "rewards/margins": 3.184443950653076, + "rewards/rejected": -4.701562881469727, + "step": 1426 + }, + { + "epoch": 1.87, + "learning_rate": 4.6275080859519926e-05, + "logits/chosen": -2.379199981689453, + "logits/rejected": -2.4917795658111572, + "logps/chosen": -213.16268920898438, + "logps/rejected": -263.35296630859375, + "loss": 0.1122, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0016521215438843, + "rewards/margins": 4.081844806671143, + "rewards/rejected": -5.08349609375, + "step": 1427 + }, + { + "epoch": 1.87, + "learning_rate": 4.626960872052245e-05, + "logits/chosen": -2.497894525527954, + "logits/rejected": -2.427412509918213, + "logps/chosen": -236.83901977539062, + "logps/rejected": -263.41766357421875, + "loss": 0.1137, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0873581171035767, + "rewards/margins": 3.7583518028259277, + "rewards/rejected": -4.845709323883057, + "step": 1428 + }, + { + "epoch": 1.87, + "learning_rate": 4.62641328890586e-05, + "logits/chosen": -1.8372935056686401, + "logits/rejected": -1.892336130142212, + "logps/chosen": -197.54855346679688, + "logps/rejected": -207.12667846679688, + "loss": 0.2948, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.3495253324508667, + "rewards/margins": 2.6124796867370605, + "rewards/rejected": -3.9620046615600586, + "step": 1429 + }, + { + "epoch": 1.87, + "learning_rate": 4.625865336607901e-05, + "logits/chosen": -2.2670607566833496, + "logits/rejected": -2.344111680984497, + "logps/chosen": -214.10784912109375, + "logps/rejected": -266.7064208984375, + "loss": 0.1006, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1706020832061768, + "rewards/margins": 4.015186786651611, + "rewards/rejected": -5.185789108276367, + "step": 1430 + }, + { + "epoch": 1.87, + "learning_rate": 4.625317015253493e-05, + "logits/chosen": -2.4320297241210938, + "logits/rejected": -2.542137622833252, + "logps/chosen": -179.33941650390625, + "logps/rejected": -225.77798461914062, + "loss": 0.1293, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0320031642913818, + "rewards/margins": 3.152787685394287, + "rewards/rejected": -4.184791088104248, + "step": 1431 + }, + { + "epoch": 1.87, + "learning_rate": 4.624768324937827e-05, + "logits/chosen": -1.8749370574951172, + "logits/rejected": -1.8897876739501953, + "logps/chosen": -208.49624633789062, + "logps/rejected": -241.69107055664062, + "loss": 0.258, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.2621411085128784, + "rewards/margins": 2.4016261100769043, + "rewards/rejected": -3.663767099380493, + "step": 1432 + }, + { + "epoch": 1.88, + "learning_rate": 4.624219265756158e-05, + "logits/chosen": -2.201465606689453, + "logits/rejected": -2.145068883895874, + "logps/chosen": -205.04013061523438, + "logps/rejected": -254.39373779296875, + "loss": 0.113, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8891902565956116, + "rewards/margins": 4.0929059982299805, + "rewards/rejected": -4.9820966720581055, + "step": 1433 + }, + { + "epoch": 1.88, + "learning_rate": 4.6236698378038026e-05, + "logits/chosen": -2.4089853763580322, + "logits/rejected": -2.4648375511169434, + "logps/chosen": -199.67010498046875, + "logps/rejected": -249.1591796875, + "loss": 0.1373, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.287391185760498, + "rewards/margins": 4.044575214385986, + "rewards/rejected": -5.331966400146484, + "step": 1434 + }, + { + "epoch": 1.88, + "learning_rate": 4.6231200411761444e-05, + "logits/chosen": -2.464130401611328, + "logits/rejected": -2.5502843856811523, + "logps/chosen": -243.13787841796875, + "logps/rejected": -283.830322265625, + "loss": 0.04, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.265555500984192, + "rewards/margins": 5.3822526931762695, + "rewards/rejected": -6.647809028625488, + "step": 1435 + }, + { + "epoch": 1.88, + "learning_rate": 4.622569875968629e-05, + "logits/chosen": -2.321964979171753, + "logits/rejected": -2.423978805541992, + "logps/chosen": -205.4765625, + "logps/rejected": -242.30911254882812, + "loss": 0.1069, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.7557665705680847, + "rewards/margins": 4.173945903778076, + "rewards/rejected": -4.929712772369385, + "step": 1436 + }, + { + "epoch": 1.88, + "learning_rate": 4.6220193422767665e-05, + "logits/chosen": -2.3185129165649414, + "logits/rejected": -2.246967077255249, + "logps/chosen": -190.0450439453125, + "logps/rejected": -195.37098693847656, + "loss": 0.1683, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.310142159461975, + "rewards/margins": 3.645517587661743, + "rewards/rejected": -4.955659866333008, + "step": 1437 + }, + { + "epoch": 1.88, + "learning_rate": 4.6214684401961314e-05, + "logits/chosen": -2.327990770339966, + "logits/rejected": -2.4045562744140625, + "logps/chosen": -195.2279052734375, + "logps/rejected": -307.27642822265625, + "loss": 0.1309, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.452347993850708, + "rewards/margins": 4.286505222320557, + "rewards/rejected": -5.738852500915527, + "step": 1438 + }, + { + "epoch": 1.88, + "learning_rate": 4.620917169822363e-05, + "logits/chosen": -2.33237361907959, + "logits/rejected": -2.329893112182617, + "logps/chosen": -197.7900390625, + "logps/rejected": -215.33221435546875, + "loss": 0.3258, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.448103666305542, + "rewards/margins": 3.1846346855163574, + "rewards/rejected": -4.63273811340332, + "step": 1439 + }, + { + "epoch": 1.88, + "learning_rate": 4.6203655312511616e-05, + "logits/chosen": -2.4939942359924316, + "logits/rejected": -2.5082345008850098, + "logps/chosen": -257.9974060058594, + "logps/rejected": -305.877197265625, + "loss": 0.2161, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.5948214530944824, + "rewards/margins": 3.7099242210388184, + "rewards/rejected": -5.304745197296143, + "step": 1440 + }, + { + "epoch": 1.89, + "learning_rate": 4.619813524578295e-05, + "logits/chosen": -2.354602813720703, + "logits/rejected": -2.3939921855926514, + "logps/chosen": -215.87257385253906, + "logps/rejected": -281.49603271484375, + "loss": 0.0665, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.500991940498352, + "rewards/margins": 4.147805690765381, + "rewards/rejected": -5.648797512054443, + "step": 1441 + }, + { + "epoch": 1.89, + "learning_rate": 4.619261149899592e-05, + "logits/chosen": -2.2306058406829834, + "logits/rejected": -2.329472780227661, + "logps/chosen": -195.59654235839844, + "logps/rejected": -269.08892822265625, + "loss": 0.1455, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8193372488021851, + "rewards/margins": 4.419328212738037, + "rewards/rejected": -5.23866605758667, + "step": 1442 + }, + { + "epoch": 1.89, + "learning_rate": 4.618708407310947e-05, + "logits/chosen": -2.4757118225097656, + "logits/rejected": -2.430291175842285, + "logps/chosen": -287.3864440917969, + "logps/rejected": -302.011962890625, + "loss": 0.1739, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6464817523956299, + "rewards/margins": 3.5614006519317627, + "rewards/rejected": -5.207882881164551, + "step": 1443 + }, + { + "epoch": 1.89, + "learning_rate": 4.6181552969083165e-05, + "logits/chosen": -2.157444715499878, + "logits/rejected": -2.1985301971435547, + "logps/chosen": -194.09764099121094, + "logps/rejected": -267.64715576171875, + "loss": 0.1754, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1710000038146973, + "rewards/margins": 3.7776384353637695, + "rewards/rejected": -4.948638439178467, + "step": 1444 + }, + { + "epoch": 1.89, + "learning_rate": 4.617601818787724e-05, + "logits/chosen": -1.9728646278381348, + "logits/rejected": -2.009904623031616, + "logps/chosen": -160.30093383789062, + "logps/rejected": -225.08453369140625, + "loss": 0.1576, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.134428858757019, + "rewards/margins": 4.066022872924805, + "rewards/rejected": -5.200451374053955, + "step": 1445 + }, + { + "epoch": 1.89, + "learning_rate": 4.617047973045254e-05, + "logits/chosen": -2.374601125717163, + "logits/rejected": -2.431764841079712, + "logps/chosen": -255.8111114501953, + "logps/rejected": -308.56298828125, + "loss": 0.0817, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.9789934158325195, + "rewards/margins": 5.199550628662109, + "rewards/rejected": -6.178544044494629, + "step": 1446 + }, + { + "epoch": 1.89, + "learning_rate": 4.6164937597770555e-05, + "logits/chosen": -2.4467074871063232, + "logits/rejected": -2.5327441692352295, + "logps/chosen": -227.62490844726562, + "logps/rejected": -285.66241455078125, + "loss": 0.0685, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1844345331192017, + "rewards/margins": 4.270139217376709, + "rewards/rejected": -5.454573631286621, + "step": 1447 + }, + { + "epoch": 1.89, + "learning_rate": 4.615939179079342e-05, + "logits/chosen": -1.8311939239501953, + "logits/rejected": -1.9591543674468994, + "logps/chosen": -181.0715789794922, + "logps/rejected": -255.28375244140625, + "loss": 0.2963, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9211238622665405, + "rewards/margins": 4.1576642990112305, + "rewards/rejected": -6.078787803649902, + "step": 1448 + }, + { + "epoch": 1.9, + "learning_rate": 4.615384231048391e-05, + "logits/chosen": -2.24050235748291, + "logits/rejected": -2.2615864276885986, + "logps/chosen": -227.12237548828125, + "logps/rejected": -235.6379852294922, + "loss": 0.2308, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.8242486715316772, + "rewards/margins": 3.2859408855438232, + "rewards/rejected": -5.110189437866211, + "step": 1449 + }, + { + "epoch": 1.9, + "learning_rate": 4.6148289157805406e-05, + "logits/chosen": -2.395170211791992, + "logits/rejected": -2.547234535217285, + "logps/chosen": -170.02056884765625, + "logps/rejected": -283.0645751953125, + "loss": 0.0816, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.996381402015686, + "rewards/margins": 3.851898670196533, + "rewards/rejected": -5.848280429840088, + "step": 1450 + }, + { + "epoch": 1.9, + "learning_rate": 4.614273233372198e-05, + "logits/chosen": -2.4565505981445312, + "logits/rejected": -2.4085004329681396, + "logps/chosen": -275.6915283203125, + "logps/rejected": -321.6641845703125, + "loss": 0.219, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.4337880611419678, + "rewards/margins": 4.953248023986816, + "rewards/rejected": -6.387036323547363, + "step": 1451 + }, + { + "epoch": 1.9, + "learning_rate": 4.61371718391983e-05, + "logits/chosen": -2.3917980194091797, + "logits/rejected": -2.414000988006592, + "logps/chosen": -201.67922973632812, + "logps/rejected": -291.4822998046875, + "loss": 0.0235, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.536219596862793, + "rewards/margins": 5.365948677062988, + "rewards/rejected": -6.902168273925781, + "step": 1452 + }, + { + "epoch": 1.9, + "learning_rate": 4.6131607675199686e-05, + "logits/chosen": -2.4047164916992188, + "logits/rejected": -2.3522493839263916, + "logps/chosen": -228.041748046875, + "logps/rejected": -243.53179931640625, + "loss": 0.1546, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.3639187812805176, + "rewards/margins": 3.640273332595825, + "rewards/rejected": -6.004191875457764, + "step": 1453 + }, + { + "epoch": 1.9, + "learning_rate": 4.612603984269208e-05, + "logits/chosen": -2.2534050941467285, + "logits/rejected": -2.179338216781616, + "logps/chosen": -238.51016235351562, + "logps/rejected": -278.68157958984375, + "loss": 0.2575, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.542654037475586, + "rewards/margins": 4.30103874206543, + "rewards/rejected": -6.843692779541016, + "step": 1454 + }, + { + "epoch": 1.9, + "learning_rate": 4.61204683426421e-05, + "logits/chosen": -2.132852077484131, + "logits/rejected": -2.1885266304016113, + "logps/chosen": -198.23362731933594, + "logps/rejected": -253.08970642089844, + "loss": 0.2702, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.3060522079467773, + "rewards/margins": 3.5543110370635986, + "rewards/rejected": -5.860363006591797, + "step": 1455 + }, + { + "epoch": 1.91, + "learning_rate": 4.611489317601696e-05, + "logits/chosen": -2.1477696895599365, + "logits/rejected": -2.235487937927246, + "logps/chosen": -261.10101318359375, + "logps/rejected": -321.97259521484375, + "loss": 0.0589, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.0646214485168457, + "rewards/margins": 4.98212194442749, + "rewards/rejected": -7.046743392944336, + "step": 1456 + }, + { + "epoch": 1.91, + "learning_rate": 4.6109314343784524e-05, + "logits/chosen": -2.1931657791137695, + "logits/rejected": -2.2348809242248535, + "logps/chosen": -168.6276397705078, + "logps/rejected": -263.44561767578125, + "loss": 0.1359, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.65928053855896, + "rewards/margins": 3.9209232330322266, + "rewards/rejected": -5.580203533172607, + "step": 1457 + }, + { + "epoch": 1.91, + "learning_rate": 4.61037318469133e-05, + "logits/chosen": -2.286311149597168, + "logits/rejected": -2.3383705615997314, + "logps/chosen": -178.47718811035156, + "logps/rejected": -235.9649200439453, + "loss": 0.2152, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.8744279146194458, + "rewards/margins": 3.764946937561035, + "rewards/rejected": -5.63937520980835, + "step": 1458 + }, + { + "epoch": 1.91, + "learning_rate": 4.6098145686372415e-05, + "logits/chosen": -1.8387361764907837, + "logits/rejected": -1.8523186445236206, + "logps/chosen": -215.5361328125, + "logps/rejected": -265.8430480957031, + "loss": 0.1596, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.201512575149536, + "rewards/margins": 3.9642109870910645, + "rewards/rejected": -6.16572380065918, + "step": 1459 + }, + { + "epoch": 1.91, + "learning_rate": 4.609255586313166e-05, + "logits/chosen": -2.1031899452209473, + "logits/rejected": -2.1518192291259766, + "logps/chosen": -231.7281494140625, + "logps/rejected": -219.5949249267578, + "loss": 0.1425, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.150951385498047, + "rewards/margins": 3.822639226913452, + "rewards/rejected": -5.973590850830078, + "step": 1460 + }, + { + "epoch": 1.91, + "learning_rate": 4.608696237816143e-05, + "logits/chosen": -2.3749542236328125, + "logits/rejected": -2.404801845550537, + "logps/chosen": -190.76507568359375, + "logps/rejected": -212.51536560058594, + "loss": 0.2917, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.243725061416626, + "rewards/margins": 3.3376617431640625, + "rewards/rejected": -5.581386566162109, + "step": 1461 + }, + { + "epoch": 1.91, + "learning_rate": 4.6081365232432766e-05, + "logits/chosen": -2.2417476177215576, + "logits/rejected": -2.29201602935791, + "logps/chosen": -309.34808349609375, + "logps/rejected": -365.32110595703125, + "loss": 0.1465, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.222254753112793, + "rewards/margins": 4.027994155883789, + "rewards/rejected": -7.250248908996582, + "step": 1462 + }, + { + "epoch": 1.91, + "learning_rate": 4.607576442691737e-05, + "logits/chosen": -2.15925931930542, + "logits/rejected": -2.148500919342041, + "logps/chosen": -222.72410583496094, + "logps/rejected": -263.0013427734375, + "loss": 0.2245, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.0160765647888184, + "rewards/margins": 4.10772705078125, + "rewards/rejected": -6.12380313873291, + "step": 1463 + }, + { + "epoch": 1.92, + "learning_rate": 4.607015996258753e-05, + "logits/chosen": -2.2752912044525146, + "logits/rejected": -2.4492502212524414, + "logps/chosen": -156.12005615234375, + "logps/rejected": -226.24057006835938, + "loss": 0.15, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.519330382347107, + "rewards/margins": 4.664186477661133, + "rewards/rejected": -6.183516979217529, + "step": 1464 + }, + { + "epoch": 1.92, + "learning_rate": 4.606455184041622e-05, + "logits/chosen": -2.2248048782348633, + "logits/rejected": -2.3291237354278564, + "logps/chosen": -167.07228088378906, + "logps/rejected": -236.86578369140625, + "loss": 0.2488, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.011113166809082, + "rewards/margins": 3.412134885787964, + "rewards/rejected": -5.423248767852783, + "step": 1465 + }, + { + "epoch": 1.92, + "learning_rate": 4.6058940061377034e-05, + "logits/chosen": -2.4281818866729736, + "logits/rejected": -2.378067970275879, + "logps/chosen": -217.27587890625, + "logps/rejected": -243.82916259765625, + "loss": 0.3441, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.679200291633606, + "rewards/margins": 3.3170528411865234, + "rewards/rejected": -4.996253490447998, + "step": 1466 + }, + { + "epoch": 1.92, + "learning_rate": 4.605332462644417e-05, + "logits/chosen": -2.0087335109710693, + "logits/rejected": -2.0646257400512695, + "logps/chosen": -196.29052734375, + "logps/rejected": -237.742431640625, + "loss": 0.3309, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.7196638584136963, + "rewards/margins": 3.8741507530212402, + "rewards/rejected": -5.593814373016357, + "step": 1467 + }, + { + "epoch": 1.92, + "learning_rate": 4.604770553659249e-05, + "logits/chosen": -2.3972527980804443, + "logits/rejected": -2.4377150535583496, + "logps/chosen": -185.716796875, + "logps/rejected": -259.7029113769531, + "loss": 0.1088, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.643090844154358, + "rewards/margins": 3.906405448913574, + "rewards/rejected": -5.549496173858643, + "step": 1468 + }, + { + "epoch": 1.92, + "learning_rate": 4.604208279279749e-05, + "logits/chosen": -2.144949436187744, + "logits/rejected": -2.169041156768799, + "logps/chosen": -160.50648498535156, + "logps/rejected": -240.07061767578125, + "loss": 0.2066, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.6182748079299927, + "rewards/margins": 3.4854378700256348, + "rewards/rejected": -5.103713035583496, + "step": 1469 + }, + { + "epoch": 1.92, + "learning_rate": 4.6036456396035294e-05, + "logits/chosen": -2.12722110748291, + "logits/rejected": -2.192887783050537, + "logps/chosen": -230.61842346191406, + "logps/rejected": -281.3009338378906, + "loss": 0.1416, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.241001605987549, + "rewards/margins": 4.063772678375244, + "rewards/rejected": -6.304775238037109, + "step": 1470 + }, + { + "epoch": 1.93, + "learning_rate": 4.603082634728266e-05, + "logits/chosen": -2.2297396659851074, + "logits/rejected": -2.20890736579895, + "logps/chosen": -216.92306518554688, + "logps/rejected": -302.16015625, + "loss": 0.0356, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4535167217254639, + "rewards/margins": 5.6094746589660645, + "rewards/rejected": -7.062991619110107, + "step": 1471 + }, + { + "epoch": 1.93, + "learning_rate": 4.602519264751697e-05, + "logits/chosen": -2.5697968006134033, + "logits/rejected": -2.4660468101501465, + "logps/chosen": -272.8087158203125, + "logps/rejected": -276.1620788574219, + "loss": 0.2615, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.8285648822784424, + "rewards/margins": 3.2634503841400146, + "rewards/rejected": -5.092015266418457, + "step": 1472 + }, + { + "epoch": 1.93, + "learning_rate": 4.601955529771628e-05, + "logits/chosen": -2.385674476623535, + "logits/rejected": -2.5492734909057617, + "logps/chosen": -209.6891632080078, + "logps/rejected": -247.408935546875, + "loss": 0.175, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.8343769311904907, + "rewards/margins": 3.4775278568267822, + "rewards/rejected": -5.311904430389404, + "step": 1473 + }, + { + "epoch": 1.93, + "learning_rate": 4.601391429885922e-05, + "logits/chosen": -2.5167958736419678, + "logits/rejected": -2.5103375911712646, + "logps/chosen": -239.463623046875, + "logps/rejected": -271.7610778808594, + "loss": 0.1614, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.7672418355941772, + "rewards/margins": 4.362864971160889, + "rewards/rejected": -6.130106449127197, + "step": 1474 + }, + { + "epoch": 1.93, + "learning_rate": 4.600826965192509e-05, + "logits/chosen": -2.1353819370269775, + "logits/rejected": -2.2745490074157715, + "logps/chosen": -144.68832397460938, + "logps/rejected": -287.36920166015625, + "loss": 0.0856, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.7524394989013672, + "rewards/margins": 4.990352630615234, + "rewards/rejected": -6.742793083190918, + "step": 1475 + }, + { + "epoch": 1.93, + "learning_rate": 4.6002621357893826e-05, + "logits/chosen": -2.486030340194702, + "logits/rejected": -2.4928431510925293, + "logps/chosen": -223.55197143554688, + "logps/rejected": -286.7210693359375, + "loss": 0.0372, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6188063621520996, + "rewards/margins": 5.330704212188721, + "rewards/rejected": -6.9495110511779785, + "step": 1476 + }, + { + "epoch": 1.93, + "learning_rate": 4.5996969417745986e-05, + "logits/chosen": -2.2136309146881104, + "logits/rejected": -2.255204200744629, + "logps/chosen": -209.3388214111328, + "logps/rejected": -260.0314025878906, + "loss": 0.1217, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.289798378944397, + "rewards/margins": 4.679617404937744, + "rewards/rejected": -5.96941614151001, + "step": 1477 + }, + { + "epoch": 1.93, + "learning_rate": 4.599131383246277e-05, + "logits/chosen": -2.519814968109131, + "logits/rejected": -2.547168254852295, + "logps/chosen": -209.82684326171875, + "logps/rejected": -275.3058166503906, + "loss": 0.1756, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.6560988426208496, + "rewards/margins": 4.22465705871582, + "rewards/rejected": -5.880756378173828, + "step": 1478 + }, + { + "epoch": 1.94, + "learning_rate": 4.598565460302599e-05, + "logits/chosen": -2.0395500659942627, + "logits/rejected": -2.038343906402588, + "logps/chosen": -179.4903564453125, + "logps/rejected": -207.68734741210938, + "loss": 0.2468, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0823073387145996, + "rewards/margins": 4.204888820648193, + "rewards/rejected": -5.287196159362793, + "step": 1479 + }, + { + "epoch": 1.94, + "learning_rate": 4.5979991730418105e-05, + "logits/chosen": -2.3252034187316895, + "logits/rejected": -2.3550727367401123, + "logps/chosen": -188.61257934570312, + "logps/rejected": -251.78277587890625, + "loss": 0.1684, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.603855848312378, + "rewards/margins": 4.1132965087890625, + "rewards/rejected": -5.7171525955200195, + "step": 1480 + }, + { + "epoch": 1.94, + "learning_rate": 4.5974325215622225e-05, + "logits/chosen": -2.103459119796753, + "logits/rejected": -2.1562860012054443, + "logps/chosen": -180.7230987548828, + "logps/rejected": -237.24388122558594, + "loss": 0.2603, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9204591512680054, + "rewards/margins": 2.8535118103027344, + "rewards/rejected": -4.773970603942871, + "step": 1481 + }, + { + "epoch": 1.94, + "learning_rate": 4.596865505962205e-05, + "logits/chosen": -2.130312919616699, + "logits/rejected": -2.2995500564575195, + "logps/chosen": -201.30421447753906, + "logps/rejected": -257.9309387207031, + "loss": 0.212, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.0479345321655273, + "rewards/margins": 3.392012596130371, + "rewards/rejected": -5.439947128295898, + "step": 1482 + }, + { + "epoch": 1.94, + "learning_rate": 4.596298126340195e-05, + "logits/chosen": -2.275320291519165, + "logits/rejected": -2.270385980606079, + "logps/chosen": -256.5345458984375, + "logps/rejected": -315.8306884765625, + "loss": 0.0989, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.094556450843811, + "rewards/margins": 4.61052131652832, + "rewards/rejected": -5.705077648162842, + "step": 1483 + }, + { + "epoch": 1.94, + "learning_rate": 4.595730382794691e-05, + "logits/chosen": -1.9736368656158447, + "logits/rejected": -2.069291353225708, + "logps/chosen": -214.57188415527344, + "logps/rejected": -224.57400512695312, + "loss": 0.2066, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.7135539054870605, + "rewards/margins": 3.323864459991455, + "rewards/rejected": -5.037418842315674, + "step": 1484 + }, + { + "epoch": 1.94, + "learning_rate": 4.595162275424255e-05, + "logits/chosen": -2.2768514156341553, + "logits/rejected": -2.439985513687134, + "logps/chosen": -168.1714630126953, + "logps/rejected": -264.5210876464844, + "loss": 0.046, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3404206037521362, + "rewards/margins": 4.816141128540039, + "rewards/rejected": -6.156561374664307, + "step": 1485 + }, + { + "epoch": 1.94, + "learning_rate": 4.594593804327513e-05, + "logits/chosen": -2.3617050647735596, + "logits/rejected": -2.4006364345550537, + "logps/chosen": -223.51766967773438, + "logps/rejected": -264.58575439453125, + "loss": 0.1137, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4779276847839355, + "rewards/margins": 3.821845054626465, + "rewards/rejected": -5.299773216247559, + "step": 1486 + }, + { + "epoch": 1.95, + "learning_rate": 4.594024969603151e-05, + "logits/chosen": -2.4014294147491455, + "logits/rejected": -2.345073938369751, + "logps/chosen": -219.4610137939453, + "logps/rejected": -262.7455749511719, + "loss": 0.0748, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.476982831954956, + "rewards/margins": 3.7781622409820557, + "rewards/rejected": -5.2551445960998535, + "step": 1487 + }, + { + "epoch": 1.95, + "learning_rate": 4.5934557713499214e-05, + "logits/chosen": -2.1576740741729736, + "logits/rejected": -2.2681870460510254, + "logps/chosen": -248.980712890625, + "logps/rejected": -353.2499694824219, + "loss": 0.1602, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.409653902053833, + "rewards/margins": 5.4672088623046875, + "rewards/rejected": -6.8768630027771, + "step": 1488 + }, + { + "epoch": 1.95, + "learning_rate": 4.592886209666639e-05, + "logits/chosen": -2.3278369903564453, + "logits/rejected": -2.4039418697357178, + "logps/chosen": -175.85903930664062, + "logps/rejected": -243.25254821777344, + "loss": 0.0638, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.090004563331604, + "rewards/margins": 4.7786030769348145, + "rewards/rejected": -5.868607997894287, + "step": 1489 + }, + { + "epoch": 1.95, + "learning_rate": 4.5923162846521824e-05, + "logits/chosen": -2.2458479404449463, + "logits/rejected": -2.3095836639404297, + "logps/chosen": -252.07955932617188, + "logps/rejected": -280.0485534667969, + "loss": 0.2264, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.0053060054779053, + "rewards/margins": 3.449532985687256, + "rewards/rejected": -5.454838752746582, + "step": 1490 + }, + { + "epoch": 1.95, + "learning_rate": 4.5917459964054896e-05, + "logits/chosen": -2.519906997680664, + "logits/rejected": -2.511167049407959, + "logps/chosen": -198.07257080078125, + "logps/rejected": -263.76898193359375, + "loss": 0.1654, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.930348813533783, + "rewards/margins": 4.16477632522583, + "rewards/rejected": -5.095125675201416, + "step": 1491 + }, + { + "epoch": 1.95, + "learning_rate": 4.5911753450255665e-05, + "logits/chosen": -2.2251136302948, + "logits/rejected": -2.2730202674865723, + "logps/chosen": -236.90675354003906, + "logps/rejected": -275.7073059082031, + "loss": 0.1027, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.889454960823059, + "rewards/margins": 3.976149082183838, + "rewards/rejected": -5.865603446960449, + "step": 1492 + }, + { + "epoch": 1.95, + "learning_rate": 4.59060433061148e-05, + "logits/chosen": -2.395418643951416, + "logits/rejected": -2.4963815212249756, + "logps/chosen": -207.62921142578125, + "logps/rejected": -221.30831909179688, + "loss": 0.2778, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.6155648231506348, + "rewards/margins": 3.7796285152435303, + "rewards/rejected": -5.395193099975586, + "step": 1493 + }, + { + "epoch": 1.96, + "learning_rate": 4.5900329532623585e-05, + "logits/chosen": -2.3171846866607666, + "logits/rejected": -2.2144782543182373, + "logps/chosen": -192.01412963867188, + "logps/rejected": -202.37173461914062, + "loss": 0.4487, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.814784288406372, + "rewards/margins": 2.9413533210754395, + "rewards/rejected": -4.756137847900391, + "step": 1494 + }, + { + "epoch": 1.96, + "learning_rate": 4.589461213077395e-05, + "logits/chosen": -1.8302745819091797, + "logits/rejected": -1.8340033292770386, + "logps/chosen": -185.82254028320312, + "logps/rejected": -226.58468627929688, + "loss": 0.2119, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.3006807565689087, + "rewards/margins": 3.979428291320801, + "rewards/rejected": -5.280109882354736, + "step": 1495 + }, + { + "epoch": 1.96, + "learning_rate": 4.588889110155845e-05, + "logits/chosen": -2.277804374694824, + "logits/rejected": -2.3033041954040527, + "logps/chosen": -245.10751342773438, + "logps/rejected": -303.5085754394531, + "loss": 0.1516, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.6710920333862305, + "rewards/margins": 4.150707244873047, + "rewards/rejected": -5.821799278259277, + "step": 1496 + }, + { + "epoch": 1.96, + "learning_rate": 4.5883166445970296e-05, + "logits/chosen": -2.103105068206787, + "logits/rejected": -2.114781618118286, + "logps/chosen": -219.78988647460938, + "logps/rejected": -306.8294982910156, + "loss": 0.2936, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6369211673736572, + "rewards/margins": 2.928783893585205, + "rewards/rejected": -4.565704822540283, + "step": 1497 + }, + { + "epoch": 1.96, + "learning_rate": 4.587743816500328e-05, + "logits/chosen": -2.03912091255188, + "logits/rejected": -2.0565059185028076, + "logps/chosen": -232.88314819335938, + "logps/rejected": -227.30206298828125, + "loss": 0.1022, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.0165839195251465, + "rewards/margins": 3.8692049980163574, + "rewards/rejected": -4.885788917541504, + "step": 1498 + }, + { + "epoch": 1.96, + "learning_rate": 4.587170625965185e-05, + "logits/chosen": -1.9884145259857178, + "logits/rejected": -1.9488219022750854, + "logps/chosen": -204.64010620117188, + "logps/rejected": -283.6075134277344, + "loss": 0.15, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.870671033859253, + "rewards/margins": 4.004744529724121, + "rewards/rejected": -5.875415802001953, + "step": 1499 + }, + { + "epoch": 1.96, + "learning_rate": 4.586597073091109e-05, + "logits/chosen": -2.0228824615478516, + "logits/rejected": -2.1101438999176025, + "logps/chosen": -197.7698211669922, + "logps/rejected": -271.5038757324219, + "loss": 0.1237, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.7399284243583679, + "rewards/margins": 3.924260139465332, + "rewards/rejected": -4.664188861846924, + "step": 1500 + }, + { + "epoch": 1.96, + "learning_rate": 4.586023157977671e-05, + "logits/chosen": -2.292987585067749, + "logits/rejected": -2.3368139266967773, + "logps/chosen": -216.46487426757812, + "logps/rejected": -270.67730712890625, + "loss": 0.1502, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.036461591720581, + "rewards/margins": 3.9408953189849854, + "rewards/rejected": -5.977356910705566, + "step": 1501 + }, + { + "epoch": 1.97, + "learning_rate": 4.5854488807245025e-05, + "logits/chosen": -2.458857774734497, + "logits/rejected": -2.3832597732543945, + "logps/chosen": -200.65383911132812, + "logps/rejected": -245.40292358398438, + "loss": 0.1366, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4562873840332031, + "rewards/margins": 3.750645637512207, + "rewards/rejected": -5.20693302154541, + "step": 1502 + }, + { + "epoch": 1.97, + "learning_rate": 4.584874241431301e-05, + "logits/chosen": -2.4760890007019043, + "logits/rejected": -2.3838562965393066, + "logps/chosen": -230.46986389160156, + "logps/rejected": -249.74501037597656, + "loss": 0.2681, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6505513191223145, + "rewards/margins": 3.109177589416504, + "rewards/rejected": -4.75972843170166, + "step": 1503 + }, + { + "epoch": 1.97, + "learning_rate": 4.5842992401978255e-05, + "logits/chosen": -2.338324785232544, + "logits/rejected": -2.331594705581665, + "logps/chosen": -207.29168701171875, + "logps/rejected": -252.10321044921875, + "loss": 0.1927, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0485973358154297, + "rewards/margins": 3.800584316253662, + "rewards/rejected": -4.84918212890625, + "step": 1504 + }, + { + "epoch": 1.97, + "learning_rate": 4.5837238771238975e-05, + "logits/chosen": -2.3720309734344482, + "logits/rejected": -2.443889856338501, + "logps/chosen": -230.22865295410156, + "logps/rejected": -280.8548889160156, + "loss": 0.0657, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3540129661560059, + "rewards/margins": 4.325744152069092, + "rewards/rejected": -5.6797566413879395, + "step": 1505 + }, + { + "epoch": 1.97, + "learning_rate": 4.583148152309403e-05, + "logits/chosen": -2.3643789291381836, + "logits/rejected": -2.5145559310913086, + "logps/chosen": -185.69049072265625, + "logps/rejected": -248.0831756591797, + "loss": 0.207, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.0387358665466309, + "rewards/margins": 3.8081166744232178, + "rewards/rejected": -4.846852779388428, + "step": 1506 + }, + { + "epoch": 1.97, + "learning_rate": 4.582572065854287e-05, + "logits/chosen": -2.0276918411254883, + "logits/rejected": -1.997923493385315, + "logps/chosen": -213.56065368652344, + "logps/rejected": -217.77130126953125, + "loss": 0.3128, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1286025047302246, + "rewards/margins": 1.9039826393127441, + "rewards/rejected": -4.032585144042969, + "step": 1507 + }, + { + "epoch": 1.97, + "learning_rate": 4.581995617858562e-05, + "logits/chosen": -2.252142906188965, + "logits/rejected": -2.3322079181671143, + "logps/chosen": -168.25584411621094, + "logps/rejected": -226.796630859375, + "loss": 0.1596, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.414211630821228, + "rewards/margins": 3.2070906162261963, + "rewards/rejected": -4.621302604675293, + "step": 1508 + }, + { + "epoch": 1.97, + "learning_rate": 4.5814188084223006e-05, + "logits/chosen": -2.3944835662841797, + "logits/rejected": -2.393568515777588, + "logps/chosen": -191.02215576171875, + "logps/rejected": -210.15518188476562, + "loss": 0.3093, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.204001545906067, + "rewards/margins": 3.173072338104248, + "rewards/rejected": -4.377073764801025, + "step": 1509 + }, + { + "epoch": 1.98, + "learning_rate": 4.580841637645638e-05, + "logits/chosen": -2.025181531906128, + "logits/rejected": -2.0204172134399414, + "logps/chosen": -163.89816284179688, + "logps/rejected": -185.0117645263672, + "loss": 0.2107, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1706594228744507, + "rewards/margins": 2.758812427520752, + "rewards/rejected": -3.929471731185913, + "step": 1510 + }, + { + "epoch": 1.98, + "learning_rate": 4.5802641056287724e-05, + "logits/chosen": -2.3880228996276855, + "logits/rejected": -2.4138453006744385, + "logps/chosen": -217.76901245117188, + "logps/rejected": -264.3223876953125, + "loss": 0.2101, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1917675733566284, + "rewards/margins": 3.443121910095215, + "rewards/rejected": -4.634889602661133, + "step": 1511 + }, + { + "epoch": 1.98, + "learning_rate": 4.5796862124719663e-05, + "logits/chosen": -2.2867062091827393, + "logits/rejected": -2.23726487159729, + "logps/chosen": -228.87283325195312, + "logps/rejected": -269.2378845214844, + "loss": 0.0988, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0112980604171753, + "rewards/margins": 4.46527099609375, + "rewards/rejected": -5.476568698883057, + "step": 1512 + }, + { + "epoch": 1.98, + "learning_rate": 4.5791079582755434e-05, + "logits/chosen": -2.4388363361358643, + "logits/rejected": -2.466726779937744, + "logps/chosen": -193.21365356445312, + "logps/rejected": -249.07179260253906, + "loss": 0.1101, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1325361728668213, + "rewards/margins": 3.8823916912078857, + "rewards/rejected": -5.014928340911865, + "step": 1513 + }, + { + "epoch": 1.98, + "learning_rate": 4.578529343139889e-05, + "logits/chosen": -2.3350844383239746, + "logits/rejected": -2.263835906982422, + "logps/chosen": -151.61380004882812, + "logps/rejected": -199.18035888671875, + "loss": 0.2984, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3873097896575928, + "rewards/margins": 2.5545272827148438, + "rewards/rejected": -3.9418370723724365, + "step": 1514 + }, + { + "epoch": 1.98, + "learning_rate": 4.5779503671654544e-05, + "logits/chosen": -2.3566839694976807, + "logits/rejected": -2.525066375732422, + "logps/chosen": -205.33551025390625, + "logps/rejected": -275.2447204589844, + "loss": 0.0893, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1993653774261475, + "rewards/margins": 4.662095069885254, + "rewards/rejected": -5.861460208892822, + "step": 1515 + }, + { + "epoch": 1.98, + "learning_rate": 4.5773710304527493e-05, + "logits/chosen": -2.0783872604370117, + "logits/rejected": -2.117863893508911, + "logps/chosen": -190.528076171875, + "logps/rejected": -224.16598510742188, + "loss": 0.2068, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.4108234643936157, + "rewards/margins": 2.3509793281555176, + "rewards/rejected": -3.761802911758423, + "step": 1516 + }, + { + "epoch": 1.99, + "learning_rate": 4.5767913331023506e-05, + "logits/chosen": -2.3093221187591553, + "logits/rejected": -2.437835693359375, + "logps/chosen": -177.2186737060547, + "logps/rejected": -241.48597717285156, + "loss": 0.1371, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2239621877670288, + "rewards/margins": 3.5838568210601807, + "rewards/rejected": -4.80781888961792, + "step": 1517 + }, + { + "epoch": 1.99, + "learning_rate": 4.576211275214894e-05, + "logits/chosen": -2.2390902042388916, + "logits/rejected": -2.2308666706085205, + "logps/chosen": -193.17189025878906, + "logps/rejected": -195.62255859375, + "loss": 0.1861, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.9776548147201538, + "rewards/margins": 3.1817262172698975, + "rewards/rejected": -4.15938138961792, + "step": 1518 + }, + { + "epoch": 1.99, + "learning_rate": 4.5756308568910786e-05, + "logits/chosen": -2.1870338916778564, + "logits/rejected": -2.4006149768829346, + "logps/chosen": -186.26927185058594, + "logps/rejected": -255.21742248535156, + "loss": 0.2058, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.5204411745071411, + "rewards/margins": 3.8511171340942383, + "rewards/rejected": -5.371558666229248, + "step": 1519 + }, + { + "epoch": 1.99, + "learning_rate": 4.5750500782316684e-05, + "logits/chosen": -2.27748966217041, + "logits/rejected": -2.291846752166748, + "logps/chosen": -179.44192504882812, + "logps/rejected": -219.5472412109375, + "loss": 0.2301, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.0973180532455444, + "rewards/margins": 2.7823243141174316, + "rewards/rejected": -3.8796424865722656, + "step": 1520 + }, + { + "epoch": 1.99, + "learning_rate": 4.574468939337488e-05, + "logits/chosen": -2.2131187915802, + "logits/rejected": -2.104618549346924, + "logps/chosen": -228.83322143554688, + "logps/rejected": -260.3489074707031, + "loss": 0.167, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.7398812770843506, + "rewards/margins": 3.2543160915374756, + "rewards/rejected": -3.994197130203247, + "step": 1521 + }, + { + "epoch": 1.99, + "learning_rate": 4.573887440309424e-05, + "logits/chosen": -2.2128376960754395, + "logits/rejected": -2.3602702617645264, + "logps/chosen": -169.43557739257812, + "logps/rejected": -223.89573669433594, + "loss": 0.2419, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.455782413482666, + "rewards/margins": 3.0371322631835938, + "rewards/rejected": -4.49291467666626, + "step": 1522 + }, + { + "epoch": 1.99, + "learning_rate": 4.5733055812484263e-05, + "logits/chosen": -2.367260456085205, + "logits/rejected": -2.4905948638916016, + "logps/chosen": -256.87579345703125, + "logps/rejected": -341.8155212402344, + "loss": 0.0541, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.6214128732681274, + "rewards/margins": 4.6316304206848145, + "rewards/rejected": -6.253043174743652, + "step": 1523 + }, + { + "epoch": 1.99, + "learning_rate": 4.5727233622555076e-05, + "logits/chosen": -2.360616445541382, + "logits/rejected": -2.4576802253723145, + "logps/chosen": -181.0323944091797, + "logps/rejected": -226.46189880371094, + "loss": 0.2441, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.4320255517959595, + "rewards/margins": 2.9019103050231934, + "rewards/rejected": -4.333935737609863, + "step": 1524 + }, + { + "epoch": 2.0, + "learning_rate": 4.5721407834317436e-05, + "logits/chosen": -2.28127121925354, + "logits/rejected": -2.286454677581787, + "logps/chosen": -194.4881591796875, + "logps/rejected": -241.7583465576172, + "loss": 0.2224, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.432761549949646, + "rewards/margins": 3.1377148628234863, + "rewards/rejected": -4.570476531982422, + "step": 1525 + }, + { + "epoch": 2.0, + "learning_rate": 4.571557844878271e-05, + "logits/chosen": -1.9688351154327393, + "logits/rejected": -2.005974292755127, + "logps/chosen": -192.5892791748047, + "logps/rejected": -242.64523315429688, + "loss": 0.1622, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3402042388916016, + "rewards/margins": 3.8366048336029053, + "rewards/rejected": -5.1768083572387695, + "step": 1526 + }, + { + "epoch": 2.0, + "learning_rate": 4.57097454669629e-05, + "logits/chosen": -2.3066444396972656, + "logits/rejected": -2.3153388500213623, + "logps/chosen": -145.78077697753906, + "logps/rejected": -192.21163940429688, + "loss": 0.1604, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.429026484489441, + "rewards/margins": 3.6721160411834717, + "rewards/rejected": -5.101142406463623, + "step": 1527 + }, + { + "epoch": 2.0, + "learning_rate": 4.5703908889870605e-05, + "logits/chosen": -2.4281747341156006, + "logits/rejected": -2.4992830753326416, + "logps/chosen": -247.89735412597656, + "logps/rejected": -306.4549560546875, + "loss": 0.1279, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3649252653121948, + "rewards/margins": 4.181218147277832, + "rewards/rejected": -5.546143054962158, + "step": 1528 + }, + { + "epoch": 2.0, + "learning_rate": 4.569806871851912e-05, + "logits/chosen": -2.430555582046509, + "logits/rejected": -2.3919808864593506, + "logps/chosen": -304.0908508300781, + "logps/rejected": -360.90771484375, + "loss": 0.0538, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.44168150424957275, + "rewards/margins": 5.871484279632568, + "rewards/rejected": -6.313165664672852, + "step": 1529 + }, + { + "epoch": 2.0, + "learning_rate": 4.5692224953922266e-05, + "logits/chosen": -1.9216489791870117, + "logits/rejected": -2.0158095359802246, + "logps/chosen": -159.58489990234375, + "logps/rejected": -228.5690460205078, + "loss": 0.083, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.9349645376205444, + "rewards/margins": 5.259486675262451, + "rewards/rejected": -6.194450855255127, + "step": 1530 + }, + { + "epoch": 2.0, + "learning_rate": 4.5686377597094565e-05, + "logits/chosen": -2.469989776611328, + "logits/rejected": -2.4002585411071777, + "logps/chosen": -270.3118896484375, + "logps/rejected": -255.894287109375, + "loss": 0.1311, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.131705641746521, + "rewards/margins": 4.982357978820801, + "rewards/rejected": -6.114063739776611, + "step": 1531 + }, + { + "epoch": 2.0, + "learning_rate": 4.5680526649051134e-05, + "logits/chosen": -2.191519260406494, + "logits/rejected": -2.2068800926208496, + "logps/chosen": -182.21990966796875, + "logps/rejected": -229.68392944335938, + "loss": 0.1128, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.6737257838249207, + "rewards/margins": 5.293555736541748, + "rewards/rejected": -5.967280864715576, + "step": 1532 + }, + { + "epoch": 2.01, + "learning_rate": 4.5674672110807713e-05, + "logits/chosen": -2.2452383041381836, + "logits/rejected": -2.2837133407592773, + "logps/chosen": -190.92742919921875, + "logps/rejected": -264.4825439453125, + "loss": 0.0464, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.028515443205833435, + "rewards/margins": 6.9200439453125, + "rewards/rejected": -6.948559761047363, + "step": 1533 + }, + { + "epoch": 2.01, + "learning_rate": 4.566881398338066e-05, + "logits/chosen": -2.3002426624298096, + "logits/rejected": -2.3245689868927, + "logps/chosen": -150.93450927734375, + "logps/rejected": -229.3854217529297, + "loss": 0.0205, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.013595461845398, + "rewards/margins": 5.568351745605469, + "rewards/rejected": -6.581946849822998, + "step": 1534 + }, + { + "epoch": 2.01, + "learning_rate": 4.5662952267786974e-05, + "logits/chosen": -2.2059741020202637, + "logits/rejected": -2.230137825012207, + "logps/chosen": -226.76358032226562, + "logps/rejected": -293.0763854980469, + "loss": 0.0507, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1337634325027466, + "rewards/margins": 6.117223739624023, + "rewards/rejected": -7.250986576080322, + "step": 1535 + }, + { + "epoch": 2.01, + "learning_rate": 4.565708696504425e-05, + "logits/chosen": -2.300882577896118, + "logits/rejected": -2.2846155166625977, + "logps/chosen": -219.44903564453125, + "logps/rejected": -288.78460693359375, + "loss": 0.0896, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.2709065675735474, + "rewards/margins": 5.382192611694336, + "rewards/rejected": -6.653099536895752, + "step": 1536 + }, + { + "epoch": 2.01, + "learning_rate": 4.565121807617074e-05, + "logits/chosen": -2.345656156539917, + "logits/rejected": -2.3673782348632812, + "logps/chosen": -218.91525268554688, + "logps/rejected": -299.89044189453125, + "loss": 0.0923, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.8808358907699585, + "rewards/margins": 5.190125465393066, + "rewards/rejected": -6.070960998535156, + "step": 1537 + }, + { + "epoch": 2.01, + "learning_rate": 4.5645345602185285e-05, + "logits/chosen": -2.1208252906799316, + "logits/rejected": -2.2008652687072754, + "logps/chosen": -197.03170776367188, + "logps/rejected": -291.86151123046875, + "loss": 0.0368, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3258799314498901, + "rewards/margins": 6.69577693939209, + "rewards/rejected": -8.02165699005127, + "step": 1538 + }, + { + "epoch": 2.01, + "learning_rate": 4.563946954410738e-05, + "logits/chosen": -2.2394111156463623, + "logits/rejected": -2.2324044704437256, + "logps/chosen": -207.92111206054688, + "logps/rejected": -307.79339599609375, + "loss": 0.0473, + "rewards/accuracies": 0.9375, + "rewards/chosen": -1.1493899822235107, + "rewards/margins": 6.962444305419922, + "rewards/rejected": -8.111834526062012, + "step": 1539 + }, + { + "epoch": 2.02, + "learning_rate": 4.563358990295711e-05, + "logits/chosen": -2.33305287361145, + "logits/rejected": -2.1754109859466553, + "logps/chosen": -282.1322937011719, + "logps/rejected": -337.01824951171875, + "loss": 0.0921, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3482136726379395, + "rewards/margins": 6.027097225189209, + "rewards/rejected": -7.37531042098999, + "step": 1540 + }, + { + "epoch": 2.02, + "learning_rate": 4.56277066797552e-05, + "logits/chosen": -1.8929749727249146, + "logits/rejected": -1.9339991807937622, + "logps/chosen": -230.7352294921875, + "logps/rejected": -318.597412109375, + "loss": 0.0909, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.755754828453064, + "rewards/margins": 6.48954439163208, + "rewards/rejected": -8.245298385620117, + "step": 1541 + }, + { + "epoch": 2.02, + "learning_rate": 4.562181987552301e-05, + "logits/chosen": -2.1031947135925293, + "logits/rejected": -2.1524558067321777, + "logps/chosen": -184.82020568847656, + "logps/rejected": -264.2920227050781, + "loss": 0.0814, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6590917110443115, + "rewards/margins": 7.221358299255371, + "rewards/rejected": -7.8804497718811035, + "step": 1542 + }, + { + "epoch": 2.02, + "learning_rate": 4.561592949128248e-05, + "logits/chosen": -1.9674711227416992, + "logits/rejected": -1.9534603357315063, + "logps/chosen": -139.9609375, + "logps/rejected": -188.78677368164062, + "loss": 0.111, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1200040578842163, + "rewards/margins": 5.612722396850586, + "rewards/rejected": -6.73272705078125, + "step": 1543 + }, + { + "epoch": 2.02, + "learning_rate": 4.561003552805623e-05, + "logits/chosen": -2.174128770828247, + "logits/rejected": -2.183529853820801, + "logps/chosen": -185.51148986816406, + "logps/rejected": -273.14007568359375, + "loss": 0.016, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.1483352184295654, + "rewards/margins": 7.032803058624268, + "rewards/rejected": -9.181138038635254, + "step": 1544 + }, + { + "epoch": 2.02, + "learning_rate": 4.560413798686745e-05, + "logits/chosen": -2.0740859508514404, + "logits/rejected": -2.122446060180664, + "logps/chosen": -200.520263671875, + "logps/rejected": -316.67706298828125, + "loss": 0.0874, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.2280356884002686, + "rewards/margins": 7.310361385345459, + "rewards/rejected": -9.538396835327148, + "step": 1545 + }, + { + "epoch": 2.02, + "learning_rate": 4.559823686873996e-05, + "logits/chosen": -2.041712999343872, + "logits/rejected": -2.1124374866485596, + "logps/chosen": -183.34530639648438, + "logps/rejected": -269.2987365722656, + "loss": 0.0216, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7693595886230469, + "rewards/margins": 7.529676914215088, + "rewards/rejected": -9.29903793334961, + "step": 1546 + }, + { + "epoch": 2.02, + "learning_rate": 4.5592332174698236e-05, + "logits/chosen": -1.954218864440918, + "logits/rejected": -2.0636560916900635, + "logps/chosen": -155.83811950683594, + "logps/rejected": -233.9814910888672, + "loss": 0.1034, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.49021577835083, + "rewards/margins": 6.273719787597656, + "rewards/rejected": -8.763935089111328, + "step": 1547 + }, + { + "epoch": 2.03, + "learning_rate": 4.558642390576734e-05, + "logits/chosen": -2.2698183059692383, + "logits/rejected": -2.0585403442382812, + "logps/chosen": -223.47071838378906, + "logps/rejected": -262.3471374511719, + "loss": 0.0279, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.512855052947998, + "rewards/margins": 6.98197078704834, + "rewards/rejected": -9.494826316833496, + "step": 1548 + }, + { + "epoch": 2.03, + "learning_rate": 4.558051206297296e-05, + "logits/chosen": -2.0435798168182373, + "logits/rejected": -2.0743608474731445, + "logps/chosen": -216.14540100097656, + "logps/rejected": -329.14093017578125, + "loss": 0.0452, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.84786319732666, + "rewards/margins": 7.801297187805176, + "rewards/rejected": -10.649162292480469, + "step": 1549 + }, + { + "epoch": 2.03, + "learning_rate": 4.557459664734141e-05, + "logits/chosen": -2.118642807006836, + "logits/rejected": -2.2871828079223633, + "logps/chosen": -207.33213806152344, + "logps/rejected": -308.4573669433594, + "loss": 0.0035, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.6876513957977295, + "rewards/margins": 6.419556140899658, + "rewards/rejected": -9.107207298278809, + "step": 1550 + }, + { + "epoch": 2.03, + "learning_rate": 4.5568677659899625e-05, + "logits/chosen": -1.952514886856079, + "logits/rejected": -2.027898073196411, + "logps/chosen": -198.62474060058594, + "logps/rejected": -294.1047058105469, + "loss": 0.0231, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.833251953125, + "rewards/margins": 7.694541931152344, + "rewards/rejected": -10.52779483795166, + "step": 1551 + }, + { + "epoch": 2.03, + "learning_rate": 4.556275510167517e-05, + "logits/chosen": -2.3004631996154785, + "logits/rejected": -2.2994089126586914, + "logps/chosen": -228.01748657226562, + "logps/rejected": -291.3690185546875, + "loss": 0.0492, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.6044530868530273, + "rewards/margins": 7.906800270080566, + "rewards/rejected": -10.511253356933594, + "step": 1552 + }, + { + "epoch": 2.03, + "learning_rate": 4.55568289736962e-05, + "logits/chosen": -2.0798513889312744, + "logits/rejected": -2.0810325145721436, + "logps/chosen": -222.9461669921875, + "logps/rejected": -292.3095397949219, + "loss": 0.0485, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.89154052734375, + "rewards/margins": 7.462136745452881, + "rewards/rejected": -10.353677749633789, + "step": 1553 + }, + { + "epoch": 2.03, + "learning_rate": 4.555089927699152e-05, + "logits/chosen": -1.8029263019561768, + "logits/rejected": -1.9061977863311768, + "logps/chosen": -146.78433227539062, + "logps/rejected": -235.67723083496094, + "loss": 0.1222, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9760775566101074, + "rewards/margins": 6.793957710266113, + "rewards/rejected": -8.770035743713379, + "step": 1554 + }, + { + "epoch": 2.04, + "learning_rate": 4.554496601259054e-05, + "logits/chosen": -2.177738666534424, + "logits/rejected": -2.196629524230957, + "logps/chosen": -190.3204345703125, + "logps/rejected": -254.96054077148438, + "loss": 0.133, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.102139472961426, + "rewards/margins": 5.640347480773926, + "rewards/rejected": -8.742486953735352, + "step": 1555 + }, + { + "epoch": 2.04, + "learning_rate": 4.553902918152329e-05, + "logits/chosen": -1.7947801351547241, + "logits/rejected": -1.947811484336853, + "logps/chosen": -137.8947296142578, + "logps/rejected": -254.4237823486328, + "loss": 0.0797, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.180742025375366, + "rewards/margins": 7.438215255737305, + "rewards/rejected": -9.61895751953125, + "step": 1556 + }, + { + "epoch": 2.04, + "learning_rate": 4.5533088784820424e-05, + "logits/chosen": -2.155184030532837, + "logits/rejected": -2.0984225273132324, + "logps/chosen": -244.7468719482422, + "logps/rejected": -332.7095947265625, + "loss": 0.0025, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.4840188026428223, + "rewards/margins": 7.49399995803833, + "rewards/rejected": -10.978019714355469, + "step": 1557 + }, + { + "epoch": 2.04, + "learning_rate": 4.55271448235132e-05, + "logits/chosen": -2.0170555114746094, + "logits/rejected": -2.008880615234375, + "logps/chosen": -227.8103485107422, + "logps/rejected": -325.2026062011719, + "loss": 0.0275, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.504916191101074, + "rewards/margins": 9.110494613647461, + "rewards/rejected": -12.615411758422852, + "step": 1558 + }, + { + "epoch": 2.04, + "learning_rate": 4.552119729863353e-05, + "logits/chosen": -1.8247603178024292, + "logits/rejected": -1.9594895839691162, + "logps/chosen": -191.88177490234375, + "logps/rejected": -302.3210144042969, + "loss": 0.0024, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.6051971912384033, + "rewards/margins": 8.71071720123291, + "rewards/rejected": -12.315914154052734, + "step": 1559 + }, + { + "epoch": 2.04, + "learning_rate": 4.5515246211213906e-05, + "logits/chosen": -2.076730251312256, + "logits/rejected": -2.1215121746063232, + "logps/chosen": -228.9929962158203, + "logps/rejected": -319.8890075683594, + "loss": 0.0136, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.890929698944092, + "rewards/margins": 8.486113548278809, + "rewards/rejected": -12.377043724060059, + "step": 1560 + }, + { + "epoch": 2.04, + "learning_rate": 4.550929156228746e-05, + "logits/chosen": -1.861020803451538, + "logits/rejected": -1.8230705261230469, + "logps/chosen": -229.69725036621094, + "logps/rejected": -298.89349365234375, + "loss": 0.0487, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.429899215698242, + "rewards/margins": 6.64678955078125, + "rewards/rejected": -10.076687812805176, + "step": 1561 + }, + { + "epoch": 2.04, + "learning_rate": 4.550333335288793e-05, + "logits/chosen": -2.0620155334472656, + "logits/rejected": -2.1032028198242188, + "logps/chosen": -206.2206268310547, + "logps/rejected": -270.24456787109375, + "loss": 0.0924, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.5137698650360107, + "rewards/margins": 6.249174118041992, + "rewards/rejected": -9.762944221496582, + "step": 1562 + }, + { + "epoch": 2.05, + "learning_rate": 4.5497371584049685e-05, + "logits/chosen": -1.9463286399841309, + "logits/rejected": -2.0142786502838135, + "logps/chosen": -198.6859130859375, + "logps/rejected": -268.861572265625, + "loss": 0.0877, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.667863368988037, + "rewards/margins": 6.892669200897217, + "rewards/rejected": -10.560531616210938, + "step": 1563 + }, + { + "epoch": 2.05, + "learning_rate": 4.5491406256807704e-05, + "logits/chosen": -1.9232243299484253, + "logits/rejected": -2.064542055130005, + "logps/chosen": -228.21578979492188, + "logps/rejected": -361.20440673828125, + "loss": 0.0362, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.642152309417725, + "rewards/margins": 8.147041320800781, + "rewards/rejected": -12.789194107055664, + "step": 1564 + }, + { + "epoch": 2.05, + "learning_rate": 4.5485437372197584e-05, + "logits/chosen": -1.951233148574829, + "logits/rejected": -2.0665435791015625, + "logps/chosen": -221.44845581054688, + "logps/rejected": -282.0459289550781, + "loss": 0.0875, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.239589214324951, + "rewards/margins": 7.598293304443359, + "rewards/rejected": -10.837882041931152, + "step": 1565 + }, + { + "epoch": 2.05, + "learning_rate": 4.5479464931255546e-05, + "logits/chosen": -1.8419092893600464, + "logits/rejected": -1.904127597808838, + "logps/chosen": -164.92803955078125, + "logps/rejected": -252.30120849609375, + "loss": 0.0261, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.9283933639526367, + "rewards/margins": 7.417722702026367, + "rewards/rejected": -11.34611701965332, + "step": 1566 + }, + { + "epoch": 2.05, + "learning_rate": 4.547348893501842e-05, + "logits/chosen": -2.302267551422119, + "logits/rejected": -2.3350188732147217, + "logps/chosen": -292.8721618652344, + "logps/rejected": -408.52679443359375, + "loss": 0.0009, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.343358516693115, + "rewards/margins": 8.37453556060791, + "rewards/rejected": -12.717894554138184, + "step": 1567 + }, + { + "epoch": 2.05, + "learning_rate": 4.5467509384523647e-05, + "logits/chosen": -1.922215461730957, + "logits/rejected": -1.9204285144805908, + "logps/chosen": -255.0386505126953, + "logps/rejected": -299.27880859375, + "loss": 0.1312, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.7221174240112305, + "rewards/margins": 6.29898738861084, + "rewards/rejected": -11.02110481262207, + "step": 1568 + }, + { + "epoch": 2.05, + "learning_rate": 4.5461526280809294e-05, + "logits/chosen": -1.9684267044067383, + "logits/rejected": -1.9687612056732178, + "logps/chosen": -248.02020263671875, + "logps/rejected": -312.8629455566406, + "loss": 0.0442, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.499403715133667, + "rewards/margins": 8.902366638183594, + "rewards/rejected": -12.401771545410156, + "step": 1569 + }, + { + "epoch": 2.05, + "learning_rate": 4.5455539624914066e-05, + "logits/chosen": -1.8702149391174316, + "logits/rejected": -1.8160890340805054, + "logps/chosen": -214.181640625, + "logps/rejected": -280.1296081542969, + "loss": 0.0828, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.006368160247803, + "rewards/margins": 6.077723503112793, + "rewards/rejected": -10.084092140197754, + "step": 1570 + }, + { + "epoch": 2.06, + "learning_rate": 4.544954941787726e-05, + "logits/chosen": -1.7309317588806152, + "logits/rejected": -1.8509042263031006, + "logps/chosen": -254.923828125, + "logps/rejected": -311.92462158203125, + "loss": 0.0631, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.9646990299224854, + "rewards/margins": 7.722902297973633, + "rewards/rejected": -11.687602043151855, + "step": 1571 + }, + { + "epoch": 2.06, + "learning_rate": 4.5443555660738774e-05, + "logits/chosen": -2.0469980239868164, + "logits/rejected": -2.2485647201538086, + "logps/chosen": -238.11585998535156, + "logps/rejected": -320.4139099121094, + "loss": 0.0535, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.201352119445801, + "rewards/margins": 8.41773796081543, + "rewards/rejected": -13.619089126586914, + "step": 1572 + }, + { + "epoch": 2.06, + "learning_rate": 4.543755835453916e-05, + "logits/chosen": -1.935351848602295, + "logits/rejected": -1.998105525970459, + "logps/chosen": -319.79559326171875, + "logps/rejected": -425.8599853515625, + "loss": 0.0012, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.961547374725342, + "rewards/margins": 10.631295204162598, + "rewards/rejected": -15.592842102050781, + "step": 1573 + }, + { + "epoch": 2.06, + "learning_rate": 4.5431557500319556e-05, + "logits/chosen": -2.1832823753356934, + "logits/rejected": -2.1295218467712402, + "logps/chosen": -223.51441955566406, + "logps/rejected": -256.72711181640625, + "loss": 0.1762, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.3360276222229, + "rewards/margins": 6.326767444610596, + "rewards/rejected": -10.662795066833496, + "step": 1574 + }, + { + "epoch": 2.06, + "learning_rate": 4.542555309912173e-05, + "logits/chosen": -1.9234460592269897, + "logits/rejected": -2.1079142093658447, + "logps/chosen": -220.8172607421875, + "logps/rejected": -400.11480712890625, + "loss": 0.0031, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.353205680847168, + "rewards/margins": 10.59358024597168, + "rewards/rejected": -14.946784973144531, + "step": 1575 + }, + { + "epoch": 2.06, + "learning_rate": 4.541954515198807e-05, + "logits/chosen": -2.037827491760254, + "logits/rejected": -2.095088481903076, + "logps/chosen": -209.502685546875, + "logps/rejected": -316.57293701171875, + "loss": 0.1245, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.286071300506592, + "rewards/margins": 7.779160499572754, + "rewards/rejected": -12.065232276916504, + "step": 1576 + }, + { + "epoch": 2.06, + "learning_rate": 4.541353365996157e-05, + "logits/chosen": -1.951758623123169, + "logits/rejected": -2.0389015674591064, + "logps/chosen": -265.8280029296875, + "logps/rejected": -333.02001953125, + "loss": 0.0911, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.675872802734375, + "rewards/margins": 7.314758777618408, + "rewards/rejected": -11.990632057189941, + "step": 1577 + }, + { + "epoch": 2.07, + "learning_rate": 4.540751862408585e-05, + "logits/chosen": -1.473084568977356, + "logits/rejected": -1.511156678199768, + "logps/chosen": -172.9165496826172, + "logps/rejected": -276.5952453613281, + "loss": 0.1628, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.1226019859313965, + "rewards/margins": 7.142666816711426, + "rewards/rejected": -11.265267372131348, + "step": 1578 + }, + { + "epoch": 2.07, + "learning_rate": 4.540150004540512e-05, + "logits/chosen": -1.8454240560531616, + "logits/rejected": -1.6991164684295654, + "logps/chosen": -222.4569091796875, + "logps/rejected": -299.02056884765625, + "loss": 0.0387, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.377161026000977, + "rewards/margins": 8.041433334350586, + "rewards/rejected": -12.418593406677246, + "step": 1579 + }, + { + "epoch": 2.07, + "learning_rate": 4.539547792496424e-05, + "logits/chosen": -2.0039446353912354, + "logits/rejected": -2.05035662651062, + "logps/chosen": -202.79391479492188, + "logps/rejected": -289.2221984863281, + "loss": 0.0882, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.039854526519775, + "rewards/margins": 7.609333515167236, + "rewards/rejected": -11.649188995361328, + "step": 1580 + }, + { + "epoch": 2.07, + "learning_rate": 4.538945226380866e-05, + "logits/chosen": -1.8065241575241089, + "logits/rejected": -2.129643678665161, + "logps/chosen": -215.80911254882812, + "logps/rejected": -362.2320861816406, + "loss": 0.0037, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.694056272506714, + "rewards/margins": 11.569538116455078, + "rewards/rejected": -15.263594627380371, + "step": 1581 + }, + { + "epoch": 2.07, + "learning_rate": 4.5383423062984455e-05, + "logits/chosen": -1.9546061754226685, + "logits/rejected": -2.007032632827759, + "logps/chosen": -214.36041259765625, + "logps/rejected": -333.48345947265625, + "loss": 0.0447, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.047149181365967, + "rewards/margins": 8.15298843383789, + "rewards/rejected": -12.200136184692383, + "step": 1582 + }, + { + "epoch": 2.07, + "learning_rate": 4.5377390323538305e-05, + "logits/chosen": -2.035081148147583, + "logits/rejected": -2.1894919872283936, + "logps/chosen": -214.34088134765625, + "logps/rejected": -353.6617736816406, + "loss": 0.0014, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.582559108734131, + "rewards/margins": 9.10820198059082, + "rewards/rejected": -13.690760612487793, + "step": 1583 + }, + { + "epoch": 2.07, + "learning_rate": 4.5371354046517516e-05, + "logits/chosen": -1.7794345617294312, + "logits/rejected": -1.7997311353683472, + "logps/chosen": -249.14747619628906, + "logps/rejected": -335.20977783203125, + "loss": 0.0029, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.026828289031982, + "rewards/margins": 8.605960845947266, + "rewards/rejected": -13.632789611816406, + "step": 1584 + }, + { + "epoch": 2.07, + "learning_rate": 4.536531423297001e-05, + "logits/chosen": -2.006917953491211, + "logits/rejected": -2.064652681350708, + "logps/chosen": -224.6466064453125, + "logps/rejected": -331.7161865234375, + "loss": 0.042, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.988997459411621, + "rewards/margins": 8.227411270141602, + "rewards/rejected": -13.216409683227539, + "step": 1585 + }, + { + "epoch": 2.08, + "learning_rate": 4.535927088394431e-05, + "logits/chosen": -1.6211562156677246, + "logits/rejected": -1.7851699590682983, + "logps/chosen": -235.837890625, + "logps/rejected": -384.5337829589844, + "loss": 0.0442, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.735535621643066, + "rewards/margins": 7.668628215789795, + "rewards/rejected": -13.404163360595703, + "step": 1586 + }, + { + "epoch": 2.08, + "learning_rate": 4.5353224000489555e-05, + "logits/chosen": -2.165318250656128, + "logits/rejected": -2.2333309650421143, + "logps/chosen": -211.68421936035156, + "logps/rejected": -291.989501953125, + "loss": 0.088, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.371261119842529, + "rewards/margins": 7.36502742767334, + "rewards/rejected": -11.736289024353027, + "step": 1587 + }, + { + "epoch": 2.08, + "learning_rate": 4.534717358365551e-05, + "logits/chosen": -1.9365509748458862, + "logits/rejected": -1.9755022525787354, + "logps/chosen": -214.47874450683594, + "logps/rejected": -273.87493896484375, + "loss": 0.0655, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.31796932220459, + "rewards/margins": 5.916989803314209, + "rewards/rejected": -10.234959602355957, + "step": 1588 + }, + { + "epoch": 2.08, + "learning_rate": 4.534111963449254e-05, + "logits/chosen": -1.9963104724884033, + "logits/rejected": -2.051135778427124, + "logps/chosen": -246.76168823242188, + "logps/rejected": -350.1937561035156, + "loss": 0.0887, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.5803420543670654, + "rewards/margins": 8.423789978027344, + "rewards/rejected": -12.004133224487305, + "step": 1589 + }, + { + "epoch": 2.08, + "learning_rate": 4.5335062154051625e-05, + "logits/chosen": -2.1684515476226807, + "logits/rejected": -2.0885422229766846, + "logps/chosen": -198.07298278808594, + "logps/rejected": -275.5907287597656, + "loss": 0.0503, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.1592936515808105, + "rewards/margins": 7.279091835021973, + "rewards/rejected": -10.438385009765625, + "step": 1590 + }, + { + "epoch": 2.08, + "learning_rate": 4.532900114338437e-05, + "logits/chosen": -1.7232838869094849, + "logits/rejected": -1.7098734378814697, + "logps/chosen": -180.58285522460938, + "logps/rejected": -214.90383911132812, + "loss": 0.1814, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.376774787902832, + "rewards/margins": 5.187775611877441, + "rewards/rejected": -8.56455135345459, + "step": 1591 + }, + { + "epoch": 2.08, + "learning_rate": 4.5322936603542984e-05, + "logits/chosen": -2.0076513290405273, + "logits/rejected": -2.0869903564453125, + "logps/chosen": -233.26451110839844, + "logps/rejected": -340.70947265625, + "loss": 0.0886, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.3201494216918945, + "rewards/margins": 6.867827415466309, + "rewards/rejected": -11.187976837158203, + "step": 1592 + }, + { + "epoch": 2.08, + "learning_rate": 4.531686853558028e-05, + "logits/chosen": -2.168454170227051, + "logits/rejected": -2.149108648300171, + "logps/chosen": -235.9113311767578, + "logps/rejected": -306.544677734375, + "loss": 0.1488, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.6051135063171387, + "rewards/margins": 6.618370056152344, + "rewards/rejected": -10.223483085632324, + "step": 1593 + }, + { + "epoch": 2.09, + "learning_rate": 4.53107969405497e-05, + "logits/chosen": -2.172264575958252, + "logits/rejected": -2.203526735305786, + "logps/chosen": -381.8232116699219, + "logps/rejected": -438.64007568359375, + "loss": 0.1118, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.092607021331787, + "rewards/margins": 7.635544300079346, + "rewards/rejected": -11.728151321411133, + "step": 1594 + }, + { + "epoch": 2.09, + "learning_rate": 4.530472181950528e-05, + "logits/chosen": -2.014963388442993, + "logits/rejected": -1.9751347303390503, + "logps/chosen": -268.2021179199219, + "logps/rejected": -323.82379150390625, + "loss": 0.0456, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.099268436431885, + "rewards/margins": 8.01276683807373, + "rewards/rejected": -12.112035751342773, + "step": 1595 + }, + { + "epoch": 2.09, + "learning_rate": 4.5298643173501693e-05, + "logits/chosen": -2.157127618789673, + "logits/rejected": -2.2155063152313232, + "logps/chosen": -326.23492431640625, + "logps/rejected": -361.61077880859375, + "loss": 0.063, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.8939595222473145, + "rewards/margins": 7.392580032348633, + "rewards/rejected": -11.286539077758789, + "step": 1596 + }, + { + "epoch": 2.09, + "learning_rate": 4.5292561003594205e-05, + "logits/chosen": -1.866766333580017, + "logits/rejected": -1.8167577981948853, + "logps/chosen": -219.24777221679688, + "logps/rejected": -265.5517578125, + "loss": 0.1723, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.417174816131592, + "rewards/margins": 5.938780307769775, + "rewards/rejected": -10.355955123901367, + "step": 1597 + }, + { + "epoch": 2.09, + "learning_rate": 4.5286475310838695e-05, + "logits/chosen": -2.137641191482544, + "logits/rejected": -2.1744580268859863, + "logps/chosen": -211.7652587890625, + "logps/rejected": -323.32415771484375, + "loss": 0.0537, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.528270959854126, + "rewards/margins": 9.253046989440918, + "rewards/rejected": -12.781318664550781, + "step": 1598 + }, + { + "epoch": 2.09, + "learning_rate": 4.5280386096291664e-05, + "logits/chosen": -2.074810266494751, + "logits/rejected": -2.0623838901519775, + "logps/chosen": -218.1590576171875, + "logps/rejected": -339.4794006347656, + "loss": 0.044, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.1840147972106934, + "rewards/margins": 8.886053085327148, + "rewards/rejected": -12.070067405700684, + "step": 1599 + }, + { + "epoch": 2.09, + "learning_rate": 4.52742933610102e-05, + "logits/chosen": -1.9109052419662476, + "logits/rejected": -1.8507012128829956, + "logps/chosen": -220.08070373535156, + "logps/rejected": -284.7148742675781, + "loss": 0.0936, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.1194047927856445, + "rewards/margins": 7.436119079589844, + "rewards/rejected": -10.555523872375488, + "step": 1600 + }, + { + "epoch": 2.1, + "learning_rate": 4.526819710605205e-05, + "logits/chosen": -2.1238362789154053, + "logits/rejected": -2.0772507190704346, + "logps/chosen": -201.04638671875, + "logps/rejected": -280.64678955078125, + "loss": 0.1025, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.489070415496826, + "rewards/margins": 6.871294975280762, + "rewards/rejected": -10.360365867614746, + "step": 1601 + }, + { + "epoch": 2.1, + "learning_rate": 4.526209733247551e-05, + "logits/chosen": -1.8819748163223267, + "logits/rejected": -1.8802152872085571, + "logps/chosen": -240.85882568359375, + "logps/rejected": -351.0258483886719, + "loss": 0.0314, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.410835266113281, + "rewards/margins": 7.961416721343994, + "rewards/rejected": -12.372251510620117, + "step": 1602 + }, + { + "epoch": 2.1, + "learning_rate": 4.5255994041339536e-05, + "logits/chosen": -2.0957300662994385, + "logits/rejected": -2.22131609916687, + "logps/chosen": -240.03170776367188, + "logps/rejected": -405.8812255859375, + "loss": 0.0495, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.5040841102600098, + "rewards/margins": 10.249372482299805, + "rewards/rejected": -13.75345516204834, + "step": 1603 + }, + { + "epoch": 2.1, + "learning_rate": 4.524988723370367e-05, + "logits/chosen": -1.8505096435546875, + "logits/rejected": -1.8317203521728516, + "logps/chosen": -206.16671752929688, + "logps/rejected": -289.6446838378906, + "loss": 0.0843, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.2112929821014404, + "rewards/margins": 6.922733306884766, + "rewards/rejected": -10.134026527404785, + "step": 1604 + }, + { + "epoch": 2.1, + "learning_rate": 4.5243776910628076e-05, + "logits/chosen": -1.963594913482666, + "logits/rejected": -2.020098924636841, + "logps/chosen": -210.90228271484375, + "logps/rejected": -305.5580749511719, + "loss": 0.0871, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.103038787841797, + "rewards/margins": 7.604170322418213, + "rewards/rejected": -10.707208633422852, + "step": 1605 + }, + { + "epoch": 2.1, + "learning_rate": 4.523766307317353e-05, + "logits/chosen": -1.9862631559371948, + "logits/rejected": -2.030336380004883, + "logps/chosen": -219.91226196289062, + "logps/rejected": -288.57550048828125, + "loss": 0.0044, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.6835989952087402, + "rewards/margins": 7.093358993530273, + "rewards/rejected": -10.776958465576172, + "step": 1606 + }, + { + "epoch": 2.1, + "learning_rate": 4.5231545722401406e-05, + "logits/chosen": -2.2059803009033203, + "logits/rejected": -2.218961477279663, + "logps/chosen": -197.95693969726562, + "logps/rejected": -261.8663024902344, + "loss": 0.0883, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.4130430221557617, + "rewards/margins": 7.487000465393066, + "rewards/rejected": -9.900043487548828, + "step": 1607 + }, + { + "epoch": 2.1, + "learning_rate": 4.522542485937369e-05, + "logits/chosen": -2.0145654678344727, + "logits/rejected": -2.118668794631958, + "logps/chosen": -221.60939025878906, + "logps/rejected": -365.8251647949219, + "loss": 0.0007, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.232522487640381, + "rewards/margins": 9.566020965576172, + "rewards/rejected": -12.798542976379395, + "step": 1608 + }, + { + "epoch": 2.11, + "learning_rate": 4.5219300485152983e-05, + "logits/chosen": -2.0122945308685303, + "logits/rejected": -2.0334525108337402, + "logps/chosen": -199.4896697998047, + "logps/rejected": -276.1574401855469, + "loss": 0.0618, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.5203516483306885, + "rewards/margins": 7.354372978210449, + "rewards/rejected": -10.874725341796875, + "step": 1609 + }, + { + "epoch": 2.11, + "learning_rate": 4.52131726008025e-05, + "logits/chosen": -1.971169114112854, + "logits/rejected": -2.1381020545959473, + "logps/chosen": -266.3401184082031, + "logps/rejected": -325.87091064453125, + "loss": 0.0734, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.6472837924957275, + "rewards/margins": 7.2722978591918945, + "rewards/rejected": -10.91958236694336, + "step": 1610 + }, + { + "epoch": 2.11, + "learning_rate": 4.520704120738606e-05, + "logits/chosen": -1.8407872915267944, + "logits/rejected": -1.9220272302627563, + "logps/chosen": -185.66297912597656, + "logps/rejected": -286.11334228515625, + "loss": 0.0543, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.526944160461426, + "rewards/margins": 6.661890506744385, + "rewards/rejected": -10.188834190368652, + "step": 1611 + }, + { + "epoch": 2.11, + "learning_rate": 4.520090630596808e-05, + "logits/chosen": -1.508141040802002, + "logits/rejected": -1.6697499752044678, + "logps/chosen": -205.7053680419922, + "logps/rejected": -327.8975830078125, + "loss": 0.0521, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.4045472145080566, + "rewards/margins": 7.769204139709473, + "rewards/rejected": -11.173750877380371, + "step": 1612 + }, + { + "epoch": 2.11, + "learning_rate": 4.519476789761361e-05, + "logits/chosen": -2.0378808975219727, + "logits/rejected": -2.055891990661621, + "logps/chosen": -224.57933044433594, + "logps/rejected": -330.0787658691406, + "loss": 0.0511, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.4485056400299072, + "rewards/margins": 7.820329666137695, + "rewards/rejected": -11.268835067749023, + "step": 1613 + }, + { + "epoch": 2.11, + "learning_rate": 4.5188625983388275e-05, + "logits/chosen": -1.7743136882781982, + "logits/rejected": -1.8478460311889648, + "logps/chosen": -247.61187744140625, + "logps/rejected": -324.42425537109375, + "loss": 0.1324, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.284206390380859, + "rewards/margins": 6.411763668060303, + "rewards/rejected": -10.69597053527832, + "step": 1614 + }, + { + "epoch": 2.11, + "learning_rate": 4.5182480564358356e-05, + "logits/chosen": -1.9719691276550293, + "logits/rejected": -2.0874102115631104, + "logps/chosen": -214.29307556152344, + "logps/rejected": -352.97418212890625, + "loss": 0.0874, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.3708295822143555, + "rewards/margins": 9.124308586120605, + "rewards/rejected": -11.495138168334961, + "step": 1615 + }, + { + "epoch": 2.11, + "learning_rate": 4.517633164159071e-05, + "logits/chosen": -2.063558578491211, + "logits/rejected": -2.044651508331299, + "logps/chosen": -250.53836059570312, + "logps/rejected": -333.0967102050781, + "loss": 0.002, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.142364978790283, + "rewards/margins": 7.67378568649292, + "rewards/rejected": -11.816149711608887, + "step": 1616 + }, + { + "epoch": 2.12, + "learning_rate": 4.5170179216152785e-05, + "logits/chosen": -2.05293607711792, + "logits/rejected": -2.0275938510894775, + "logps/chosen": -251.1569061279297, + "logps/rejected": -274.8096923828125, + "loss": 0.0885, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.5648229122161865, + "rewards/margins": 6.097499370574951, + "rewards/rejected": -9.662322044372559, + "step": 1617 + }, + { + "epoch": 2.12, + "learning_rate": 4.516402328911269e-05, + "logits/chosen": -2.0880846977233887, + "logits/rejected": -2.10282301902771, + "logps/chosen": -210.91859436035156, + "logps/rejected": -306.0312805175781, + "loss": 0.0452, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.1213300228118896, + "rewards/margins": 8.73926830291748, + "rewards/rejected": -11.86059856414795, + "step": 1618 + }, + { + "epoch": 2.12, + "learning_rate": 4.5157863861539093e-05, + "logits/chosen": -2.01450777053833, + "logits/rejected": -1.9900140762329102, + "logps/chosen": -247.73287963867188, + "logps/rejected": -339.9057312011719, + "loss": 0.0035, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.3483333587646484, + "rewards/margins": 9.037501335144043, + "rewards/rejected": -12.385833740234375, + "step": 1619 + }, + { + "epoch": 2.12, + "learning_rate": 4.515170093450129e-05, + "logits/chosen": -1.9653971195220947, + "logits/rejected": -2.040536642074585, + "logps/chosen": -239.28778076171875, + "logps/rejected": -357.8448486328125, + "loss": 0.017, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.8111658096313477, + "rewards/margins": 8.509544372558594, + "rewards/rejected": -12.320709228515625, + "step": 1620 + }, + { + "epoch": 2.12, + "learning_rate": 4.5145534509069196e-05, + "logits/chosen": -1.8105957508087158, + "logits/rejected": -1.7471611499786377, + "logps/chosen": -196.7708740234375, + "logps/rejected": -298.52545166015625, + "loss": 0.046, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.7363200187683105, + "rewards/margins": 7.920873641967773, + "rewards/rejected": -10.657193183898926, + "step": 1621 + }, + { + "epoch": 2.12, + "learning_rate": 4.513936458631332e-05, + "logits/chosen": -2.146426200866699, + "logits/rejected": -2.1980879306793213, + "logps/chosen": -262.87750244140625, + "logps/rejected": -332.66558837890625, + "loss": 0.0117, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.306097984313965, + "rewards/margins": 8.095102310180664, + "rewards/rejected": -12.401199340820312, + "step": 1622 + }, + { + "epoch": 2.12, + "learning_rate": 4.5133191167304754e-05, + "logits/chosen": -1.9258334636688232, + "logits/rejected": -1.9266880750656128, + "logps/chosen": -192.65664672851562, + "logps/rejected": -287.0689697265625, + "loss": 0.0043, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.7276978492736816, + "rewards/margins": 8.794516563415527, + "rewards/rejected": -11.522215843200684, + "step": 1623 + }, + { + "epoch": 2.13, + "learning_rate": 4.5127014253115246e-05, + "logits/chosen": -1.9906028509140015, + "logits/rejected": -2.0530624389648438, + "logps/chosen": -270.1056213378906, + "logps/rejected": -329.58441162109375, + "loss": 0.0439, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.131108283996582, + "rewards/margins": 8.448394775390625, + "rewards/rejected": -12.579503059387207, + "step": 1624 + }, + { + "epoch": 2.13, + "learning_rate": 4.5120833844817116e-05, + "logits/chosen": -1.2688982486724854, + "logits/rejected": -1.2891889810562134, + "logps/chosen": -228.94691467285156, + "logps/rejected": -296.3216552734375, + "loss": 0.0514, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.573295593261719, + "rewards/margins": 6.978508472442627, + "rewards/rejected": -11.551802635192871, + "step": 1625 + }, + { + "epoch": 2.13, + "learning_rate": 4.5114649943483306e-05, + "logits/chosen": -1.7697856426239014, + "logits/rejected": -1.8215487003326416, + "logps/chosen": -194.67367553710938, + "logps/rejected": -292.50726318359375, + "loss": 0.1312, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.337482213973999, + "rewards/margins": 6.588993072509766, + "rewards/rejected": -9.926475524902344, + "step": 1626 + }, + { + "epoch": 2.13, + "learning_rate": 4.510846255018736e-05, + "logits/chosen": -2.0337822437286377, + "logits/rejected": -2.033005714416504, + "logps/chosen": -247.71368408203125, + "logps/rejected": -357.8754577636719, + "loss": 0.023, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.704446315765381, + "rewards/margins": 8.37730884552002, + "rewards/rejected": -13.081753730773926, + "step": 1627 + }, + { + "epoch": 2.13, + "learning_rate": 4.5102271666003417e-05, + "logits/chosen": -1.8250856399536133, + "logits/rejected": -1.890662670135498, + "logps/chosen": -309.4023132324219, + "logps/rejected": -368.8353576660156, + "loss": 0.0464, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.654391288757324, + "rewards/margins": 7.280395984649658, + "rewards/rejected": -11.93478775024414, + "step": 1628 + }, + { + "epoch": 2.13, + "learning_rate": 4.509607729200625e-05, + "logits/chosen": -1.5864503383636475, + "logits/rejected": -1.6689658164978027, + "logps/chosen": -224.78378295898438, + "logps/rejected": -361.28692626953125, + "loss": 0.0471, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.2646989822387695, + "rewards/margins": 9.243759155273438, + "rewards/rejected": -12.508459091186523, + "step": 1629 + }, + { + "epoch": 2.13, + "learning_rate": 4.50898794292712e-05, + "logits/chosen": -1.9407392740249634, + "logits/rejected": -1.9865281581878662, + "logps/chosen": -327.77001953125, + "logps/rejected": -436.6799011230469, + "loss": 0.0749, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.880475044250488, + "rewards/margins": 8.208063125610352, + "rewards/rejected": -14.08853816986084, + "step": 1630 + }, + { + "epoch": 2.13, + "learning_rate": 4.508367807887425e-05, + "logits/chosen": -1.7678276300430298, + "logits/rejected": -1.7630263566970825, + "logps/chosen": -344.8454284667969, + "logps/rejected": -394.4046325683594, + "loss": 0.1963, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.344852924346924, + "rewards/margins": 6.4874467849731445, + "rewards/rejected": -12.83229923248291, + "step": 1631 + }, + { + "epoch": 2.14, + "learning_rate": 4.5077473241891965e-05, + "logits/chosen": -1.511955738067627, + "logits/rejected": -1.4905922412872314, + "logps/chosen": -220.30099487304688, + "logps/rejected": -281.9308776855469, + "loss": 0.2282, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.048691272735596, + "rewards/margins": 5.856573581695557, + "rewards/rejected": -9.905265808105469, + "step": 1632 + }, + { + "epoch": 2.14, + "learning_rate": 4.507126491940153e-05, + "logits/chosen": -1.8332291841506958, + "logits/rejected": -1.8841722011566162, + "logps/chosen": -225.41574096679688, + "logps/rejected": -307.5355529785156, + "loss": 0.0921, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.2159814834594727, + "rewards/margins": 7.965794086456299, + "rewards/rejected": -11.181775093078613, + "step": 1633 + }, + { + "epoch": 2.14, + "learning_rate": 4.506505311248072e-05, + "logits/chosen": -1.708308219909668, + "logits/rejected": -1.7283846139907837, + "logps/chosen": -204.90386962890625, + "logps/rejected": -330.06927490234375, + "loss": 0.0012, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.275350570678711, + "rewards/margins": 9.99798583984375, + "rewards/rejected": -14.273336410522461, + "step": 1634 + }, + { + "epoch": 2.14, + "learning_rate": 4.505883782220793e-05, + "logits/chosen": -1.7285094261169434, + "logits/rejected": -1.7134628295898438, + "logps/chosen": -236.44061279296875, + "logps/rejected": -302.3081359863281, + "loss": 0.1065, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.248753547668457, + "rewards/margins": 6.385387420654297, + "rewards/rejected": -11.634140014648438, + "step": 1635 + }, + { + "epoch": 2.14, + "learning_rate": 4.505261904966214e-05, + "logits/chosen": -1.644340991973877, + "logits/rejected": -1.7456663846969604, + "logps/chosen": -234.6942138671875, + "logps/rejected": -330.1268615722656, + "loss": 0.0055, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.138976573944092, + "rewards/margins": 7.677927017211914, + "rewards/rejected": -12.816903114318848, + "step": 1636 + }, + { + "epoch": 2.14, + "learning_rate": 4.5046396795922964e-05, + "logits/chosen": -1.590599775314331, + "logits/rejected": -1.6563043594360352, + "logps/chosen": -243.24896240234375, + "logps/rejected": -286.80499267578125, + "loss": 0.0517, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.77515983581543, + "rewards/margins": 6.981686115264893, + "rewards/rejected": -11.75684642791748, + "step": 1637 + }, + { + "epoch": 2.14, + "learning_rate": 4.504017106207059e-05, + "logits/chosen": -1.8463704586029053, + "logits/rejected": -1.9400237798690796, + "logps/chosen": -263.376953125, + "logps/rejected": -324.1238708496094, + "loss": 0.0479, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.044637203216553, + "rewards/margins": 8.752671241760254, + "rewards/rejected": -13.797307968139648, + "step": 1638 + }, + { + "epoch": 2.14, + "learning_rate": 4.5033941849185835e-05, + "logits/chosen": -2.0151426792144775, + "logits/rejected": -2.0619280338287354, + "logps/chosen": -239.4160919189453, + "logps/rejected": -311.9981384277344, + "loss": 0.0459, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.772345542907715, + "rewards/margins": 7.730282783508301, + "rewards/rejected": -12.502628326416016, + "step": 1639 + }, + { + "epoch": 2.15, + "learning_rate": 4.50277091583501e-05, + "logits/chosen": -1.5663697719573975, + "logits/rejected": -1.6064140796661377, + "logps/chosen": -182.79518127441406, + "logps/rejected": -268.147216796875, + "loss": 0.1124, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.8507080078125, + "rewards/margins": 7.0118231773376465, + "rewards/rejected": -10.862531661987305, + "step": 1640 + }, + { + "epoch": 2.15, + "learning_rate": 4.5021472990645404e-05, + "logits/chosen": -1.7681742906570435, + "logits/rejected": -1.8470631837844849, + "logps/chosen": -315.16131591796875, + "logps/rejected": -430.01513671875, + "loss": 0.088, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.021973133087158, + "rewards/margins": 8.149192810058594, + "rewards/rejected": -13.171164512634277, + "step": 1641 + }, + { + "epoch": 2.15, + "learning_rate": 4.501523334715436e-05, + "logits/chosen": -1.843216896057129, + "logits/rejected": -1.9197378158569336, + "logps/chosen": -242.83187866210938, + "logps/rejected": -410.14581298828125, + "loss": 0.0874, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.327381610870361, + "rewards/margins": 8.265216827392578, + "rewards/rejected": -12.592597961425781, + "step": 1642 + }, + { + "epoch": 2.15, + "learning_rate": 4.5008990228960176e-05, + "logits/chosen": -1.5981818437576294, + "logits/rejected": -1.7583653926849365, + "logps/chosen": -209.43771362304688, + "logps/rejected": -317.85296630859375, + "loss": 0.1179, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.252575397491455, + "rewards/margins": 8.589457511901855, + "rewards/rejected": -12.842031478881836, + "step": 1643 + }, + { + "epoch": 2.15, + "learning_rate": 4.500274363714671e-05, + "logits/chosen": -1.8431150913238525, + "logits/rejected": -2.0487449169158936, + "logps/chosen": -255.32421875, + "logps/rejected": -387.9079895019531, + "loss": 0.0803, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.5627946853637695, + "rewards/margins": 8.269213676452637, + "rewards/rejected": -12.832008361816406, + "step": 1644 + }, + { + "epoch": 2.15, + "learning_rate": 4.4996493572798356e-05, + "logits/chosen": -2.1761112213134766, + "logits/rejected": -2.2496793270111084, + "logps/chosen": -270.3349914550781, + "logps/rejected": -391.78509521484375, + "loss": 0.087, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.312464714050293, + "rewards/margins": 7.775787353515625, + "rewards/rejected": -12.088252067565918, + "step": 1645 + }, + { + "epoch": 2.15, + "learning_rate": 4.499024003700015e-05, + "logits/chosen": -1.5933220386505127, + "logits/rejected": -1.6437098979949951, + "logps/chosen": -202.7498016357422, + "logps/rejected": -268.6524353027344, + "loss": 0.102, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.125113010406494, + "rewards/margins": 7.220837116241455, + "rewards/rejected": -10.34595012664795, + "step": 1646 + }, + { + "epoch": 2.16, + "learning_rate": 4.498398303083774e-05, + "logits/chosen": -1.565222144126892, + "logits/rejected": -1.5468480587005615, + "logps/chosen": -231.1023712158203, + "logps/rejected": -317.9393005371094, + "loss": 0.0982, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.608382701873779, + "rewards/margins": 6.992830276489258, + "rewards/rejected": -11.601211547851562, + "step": 1647 + }, + { + "epoch": 2.16, + "learning_rate": 4.497772255539735e-05, + "logits/chosen": -1.8142828941345215, + "logits/rejected": -1.849523901939392, + "logps/chosen": -186.3400115966797, + "logps/rejected": -267.7499084472656, + "loss": 0.064, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.7400169372558594, + "rewards/margins": 6.862453937530518, + "rewards/rejected": -10.602472305297852, + "step": 1648 + }, + { + "epoch": 2.16, + "learning_rate": 4.497145861176581e-05, + "logits/chosen": -1.8521313667297363, + "logits/rejected": -1.8114895820617676, + "logps/chosen": -211.46078491210938, + "logps/rejected": -263.2080993652344, + "loss": 0.0491, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.9873476028442383, + "rewards/margins": 6.93930721282959, + "rewards/rejected": -10.926654815673828, + "step": 1649 + }, + { + "epoch": 2.16, + "learning_rate": 4.496519120103057e-05, + "logits/chosen": -1.874268889427185, + "logits/rejected": -2.000852584838867, + "logps/chosen": -219.22030639648438, + "logps/rejected": -360.3320617675781, + "loss": 0.0694, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.9966840744018555, + "rewards/margins": 7.829019546508789, + "rewards/rejected": -11.825703620910645, + "step": 1650 + }, + { + "epoch": 2.16, + "learning_rate": 4.495892032427966e-05, + "logits/chosen": -2.003627061843872, + "logits/rejected": -2.088552236557007, + "logps/chosen": -213.91397094726562, + "logps/rejected": -319.4725646972656, + "loss": 0.0451, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.4441897869110107, + "rewards/margins": 8.1718111038208, + "rewards/rejected": -11.61600112915039, + "step": 1651 + }, + { + "epoch": 2.16, + "learning_rate": 4.495264598260174e-05, + "logits/chosen": -1.4233427047729492, + "logits/rejected": -1.4113380908966064, + "logps/chosen": -239.0942840576172, + "logps/rejected": -325.3289489746094, + "loss": 0.1248, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.8961760997772217, + "rewards/margins": 7.048993110656738, + "rewards/rejected": -10.945169448852539, + "step": 1652 + }, + { + "epoch": 2.16, + "learning_rate": 4.4946368177086034e-05, + "logits/chosen": -1.8109068870544434, + "logits/rejected": -1.8243604898452759, + "logps/chosen": -223.3125, + "logps/rejected": -280.24566650390625, + "loss": 0.0913, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.3544464111328125, + "rewards/margins": 7.040537357330322, + "rewards/rejected": -10.394983291625977, + "step": 1653 + }, + { + "epoch": 2.16, + "learning_rate": 4.49400869088224e-05, + "logits/chosen": -1.8591406345367432, + "logits/rejected": -1.917752742767334, + "logps/chosen": -265.19586181640625, + "logps/rejected": -344.5182800292969, + "loss": 0.0802, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.180366039276123, + "rewards/margins": 7.217251300811768, + "rewards/rejected": -11.39761734008789, + "step": 1654 + }, + { + "epoch": 2.17, + "learning_rate": 4.493380217890128e-05, + "logits/chosen": -2.016444683074951, + "logits/rejected": -2.066431999206543, + "logps/chosen": -249.1741180419922, + "logps/rejected": -367.8361511230469, + "loss": 0.0872, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.942394733428955, + "rewards/margins": 7.774467468261719, + "rewards/rejected": -11.716863632202148, + "step": 1655 + }, + { + "epoch": 2.17, + "learning_rate": 4.492751398841372e-05, + "logits/chosen": -1.8434172868728638, + "logits/rejected": -1.943632960319519, + "logps/chosen": -183.25062561035156, + "logps/rejected": -272.4522705078125, + "loss": 0.0879, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.8946924209594727, + "rewards/margins": 7.5939788818359375, + "rewards/rejected": -11.488670349121094, + "step": 1656 + }, + { + "epoch": 2.17, + "learning_rate": 4.492122233845138e-05, + "logits/chosen": -1.9134418964385986, + "logits/rejected": -1.9178599119186401, + "logps/chosen": -230.68223571777344, + "logps/rejected": -401.3302307128906, + "loss": 0.0028, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.2163848876953125, + "rewards/margins": 8.244821548461914, + "rewards/rejected": -12.461207389831543, + "step": 1657 + }, + { + "epoch": 2.17, + "learning_rate": 4.4914927230106495e-05, + "logits/chosen": -1.8793325424194336, + "logits/rejected": -2.0541069507598877, + "logps/chosen": -225.55633544921875, + "logps/rejected": -350.17144775390625, + "loss": 0.0102, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.772604942321777, + "rewards/margins": 7.690614223480225, + "rewards/rejected": -12.463218688964844, + "step": 1658 + }, + { + "epoch": 2.17, + "learning_rate": 4.490862866447191e-05, + "logits/chosen": -1.852423906326294, + "logits/rejected": -1.8437042236328125, + "logps/chosen": -193.12823486328125, + "logps/rejected": -297.361572265625, + "loss": 0.0886, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.9329285621643066, + "rewards/margins": 7.062554836273193, + "rewards/rejected": -10.995484352111816, + "step": 1659 + }, + { + "epoch": 2.17, + "learning_rate": 4.4902326642641095e-05, + "logits/chosen": -2.0002667903900146, + "logits/rejected": -1.9759914875030518, + "logps/chosen": -230.4566650390625, + "logps/rejected": -313.8236389160156, + "loss": 0.1312, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.2740044593811035, + "rewards/margins": 7.39484167098999, + "rewards/rejected": -10.668846130371094, + "step": 1660 + }, + { + "epoch": 2.17, + "learning_rate": 4.489602116570809e-05, + "logits/chosen": -1.8080790042877197, + "logits/rejected": -1.86578369140625, + "logps/chosen": -199.6761016845703, + "logps/rejected": -272.2088623046875, + "loss": 0.0626, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.074660301208496, + "rewards/margins": 6.800763130187988, + "rewards/rejected": -10.875423431396484, + "step": 1661 + }, + { + "epoch": 2.18, + "learning_rate": 4.488971223476753e-05, + "logits/chosen": -2.065697431564331, + "logits/rejected": -2.094249725341797, + "logps/chosen": -219.26934814453125, + "logps/rejected": -289.54949951171875, + "loss": 0.0055, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.626599073410034, + "rewards/margins": 7.229710102081299, + "rewards/rejected": -10.85630989074707, + "step": 1662 + }, + { + "epoch": 2.18, + "learning_rate": 4.4883399850914684e-05, + "logits/chosen": -2.0257272720336914, + "logits/rejected": -2.0586395263671875, + "logps/chosen": -210.08489990234375, + "logps/rejected": -290.2291259765625, + "loss": 0.1383, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.094170570373535, + "rewards/margins": 6.843661308288574, + "rewards/rejected": -10.93783187866211, + "step": 1663 + }, + { + "epoch": 2.18, + "learning_rate": 4.4877084015245394e-05, + "logits/chosen": -2.095829963684082, + "logits/rejected": -2.090388059616089, + "logps/chosen": -216.79452514648438, + "logps/rejected": -315.7300720214844, + "loss": 0.0903, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.351548194885254, + "rewards/margins": 7.777657508850098, + "rewards/rejected": -12.129206657409668, + "step": 1664 + }, + { + "epoch": 2.18, + "learning_rate": 4.48707647288561e-05, + "logits/chosen": -1.8059430122375488, + "logits/rejected": -1.9512474536895752, + "logps/chosen": -220.30703735351562, + "logps/rejected": -398.6494445800781, + "loss": 0.0036, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.140538215637207, + "rewards/margins": 9.172012329101562, + "rewards/rejected": -13.31255054473877, + "step": 1665 + }, + { + "epoch": 2.18, + "learning_rate": 4.486444199284386e-05, + "logits/chosen": -1.8825180530548096, + "logits/rejected": -1.9374803304672241, + "logps/chosen": -256.9749755859375, + "logps/rejected": -345.34686279296875, + "loss": 0.0065, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.18184757232666, + "rewards/margins": 9.528462409973145, + "rewards/rejected": -13.710309982299805, + "step": 1666 + }, + { + "epoch": 2.18, + "learning_rate": 4.4858115808306314e-05, + "logits/chosen": -1.770180106163025, + "logits/rejected": -1.740568995475769, + "logps/chosen": -235.30775451660156, + "logps/rejected": -320.1776428222656, + "loss": 0.0181, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.58617639541626, + "rewards/margins": 7.5318989753723145, + "rewards/rejected": -12.118075370788574, + "step": 1667 + }, + { + "epoch": 2.18, + "learning_rate": 4.48517861763417e-05, + "logits/chosen": -1.7661123275756836, + "logits/rejected": -1.7018529176712036, + "logps/chosen": -224.44650268554688, + "logps/rejected": -286.1048278808594, + "loss": 0.0454, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.588602066040039, + "rewards/margins": 7.2582621574401855, + "rewards/rejected": -10.846864700317383, + "step": 1668 + }, + { + "epoch": 2.18, + "learning_rate": 4.4845453098048876e-05, + "logits/chosen": -1.9162724018096924, + "logits/rejected": -1.9782600402832031, + "logps/chosen": -252.06277465820312, + "logps/rejected": -341.20379638671875, + "loss": 0.1005, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.865740776062012, + "rewards/margins": 7.253775119781494, + "rewards/rejected": -12.119516372680664, + "step": 1669 + }, + { + "epoch": 2.19, + "learning_rate": 4.483911657452726e-05, + "logits/chosen": -2.012774705886841, + "logits/rejected": -2.090327501296997, + "logps/chosen": -203.31741333007812, + "logps/rejected": -287.6564025878906, + "loss": 0.1608, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.810271739959717, + "rewards/margins": 6.6673126220703125, + "rewards/rejected": -10.477583885192871, + "step": 1670 + }, + { + "epoch": 2.19, + "learning_rate": 4.483277660687692e-05, + "logits/chosen": -1.927455186843872, + "logits/rejected": -1.9408812522888184, + "logps/chosen": -276.76416015625, + "logps/rejected": -364.72320556640625, + "loss": 0.0474, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.065609931945801, + "rewards/margins": 7.9829230308532715, + "rewards/rejected": -12.04853343963623, + "step": 1671 + }, + { + "epoch": 2.19, + "learning_rate": 4.482643319619847e-05, + "logits/chosen": -1.7971769571304321, + "logits/rejected": -1.834416389465332, + "logps/chosen": -247.7070770263672, + "logps/rejected": -356.5272216796875, + "loss": 0.0897, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.6214561462402344, + "rewards/margins": 8.43688678741455, + "rewards/rejected": -12.058342933654785, + "step": 1672 + }, + { + "epoch": 2.19, + "learning_rate": 4.482008634359315e-05, + "logits/chosen": -1.9927940368652344, + "logits/rejected": -2.031109094619751, + "logps/chosen": -237.0690460205078, + "logps/rejected": -314.8182373046875, + "loss": 0.0932, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.718445777893066, + "rewards/margins": 7.051286220550537, + "rewards/rejected": -11.769731521606445, + "step": 1673 + }, + { + "epoch": 2.19, + "learning_rate": 4.48137360501628e-05, + "logits/chosen": -1.9173270463943481, + "logits/rejected": -1.9167189598083496, + "logps/chosen": -203.08154296875, + "logps/rejected": -287.092041015625, + "loss": 0.0885, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.5145535469055176, + "rewards/margins": 7.844388484954834, + "rewards/rejected": -11.358942985534668, + "step": 1674 + }, + { + "epoch": 2.19, + "learning_rate": 4.480738231700985e-05, + "logits/chosen": -2.1176838874816895, + "logits/rejected": -2.0974013805389404, + "logps/chosen": -266.84588623046875, + "logps/rejected": -359.0920104980469, + "loss": 0.0445, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.568906784057617, + "rewards/margins": 9.128886222839355, + "rewards/rejected": -13.697793006896973, + "step": 1675 + }, + { + "epoch": 2.19, + "learning_rate": 4.4801025145237306e-05, + "logits/chosen": -1.902029037475586, + "logits/rejected": -1.8923699855804443, + "logps/chosen": -227.2543487548828, + "logps/rejected": -288.05206298828125, + "loss": 0.1307, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.330000400543213, + "rewards/margins": 7.22145938873291, + "rewards/rejected": -10.551460266113281, + "step": 1676 + }, + { + "epoch": 2.19, + "learning_rate": 4.4794664535948816e-05, + "logits/chosen": -1.6977206468582153, + "logits/rejected": -1.8270032405853271, + "logps/chosen": -223.025390625, + "logps/rejected": -359.58563232421875, + "loss": 0.0053, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.561017036437988, + "rewards/margins": 8.420669555664062, + "rewards/rejected": -12.981685638427734, + "step": 1677 + }, + { + "epoch": 2.2, + "learning_rate": 4.478830049024859e-05, + "logits/chosen": -2.022806167602539, + "logits/rejected": -1.9440600872039795, + "logps/chosen": -224.28826904296875, + "logps/rejected": -275.18988037109375, + "loss": 0.0473, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.236040115356445, + "rewards/margins": 7.757159233093262, + "rewards/rejected": -11.993199348449707, + "step": 1678 + }, + { + "epoch": 2.2, + "learning_rate": 4.478193300924145e-05, + "logits/chosen": -2.131181001663208, + "logits/rejected": -2.179513931274414, + "logps/chosen": -221.23886108398438, + "logps/rejected": -285.28485107421875, + "loss": 0.0884, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.4926037788391113, + "rewards/margins": 7.22347354888916, + "rewards/rejected": -10.716076850891113, + "step": 1679 + }, + { + "epoch": 2.2, + "learning_rate": 4.4775562094032795e-05, + "logits/chosen": -1.5273998975753784, + "logits/rejected": -1.7143009901046753, + "logps/chosen": -271.4801025390625, + "logps/rejected": -375.77069091796875, + "loss": 0.1006, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.011230945587158, + "rewards/margins": 7.899630069732666, + "rewards/rejected": -13.910860061645508, + "step": 1680 + }, + { + "epoch": 2.2, + "learning_rate": 4.476918774572865e-05, + "logits/chosen": -1.828747272491455, + "logits/rejected": -1.9121427536010742, + "logps/chosen": -214.4183807373047, + "logps/rejected": -285.906982421875, + "loss": 0.0648, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.341349124908447, + "rewards/margins": 6.82589054107666, + "rewards/rejected": -12.16723918914795, + "step": 1681 + }, + { + "epoch": 2.2, + "learning_rate": 4.476280996543561e-05, + "logits/chosen": -1.9479432106018066, + "logits/rejected": -2.016745090484619, + "logps/chosen": -272.9812316894531, + "logps/rejected": -366.86865234375, + "loss": 0.0759, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.882082462310791, + "rewards/margins": 8.156986236572266, + "rewards/rejected": -12.039068222045898, + "step": 1682 + }, + { + "epoch": 2.2, + "learning_rate": 4.47564287542609e-05, + "logits/chosen": -1.7548673152923584, + "logits/rejected": -1.747685432434082, + "logps/chosen": -233.76705932617188, + "logps/rejected": -317.80145263671875, + "loss": 0.002, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.551597595214844, + "rewards/margins": 8.004530906677246, + "rewards/rejected": -12.556129455566406, + "step": 1683 + }, + { + "epoch": 2.2, + "learning_rate": 4.4750044113312284e-05, + "logits/chosen": -1.8895262479782104, + "logits/rejected": -1.8817856311798096, + "logps/chosen": -243.14794921875, + "logps/rejected": -309.8323974609375, + "loss": 0.0467, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.921259880065918, + "rewards/margins": 7.604479789733887, + "rewards/rejected": -12.525739669799805, + "step": 1684 + }, + { + "epoch": 2.21, + "learning_rate": 4.474365604369817e-05, + "logits/chosen": -1.8540509939193726, + "logits/rejected": -1.894709587097168, + "logps/chosen": -225.369384765625, + "logps/rejected": -333.721923828125, + "loss": 0.096, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.20390510559082, + "rewards/margins": 7.378586769104004, + "rewards/rejected": -12.582491874694824, + "step": 1685 + }, + { + "epoch": 2.21, + "learning_rate": 4.473726454652755e-05, + "logits/chosen": -1.8880846500396729, + "logits/rejected": -1.8266561031341553, + "logps/chosen": -197.33335876464844, + "logps/rejected": -272.0766296386719, + "loss": 0.0456, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.894861936569214, + "rewards/margins": 7.535285472869873, + "rewards/rejected": -11.430148124694824, + "step": 1686 + }, + { + "epoch": 2.21, + "learning_rate": 4.473086962290999e-05, + "logits/chosen": -2.0629053115844727, + "logits/rejected": -2.107692003250122, + "logps/chosen": -267.00848388671875, + "logps/rejected": -336.1997985839844, + "loss": 0.0952, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.5595641136169434, + "rewards/margins": 8.624744415283203, + "rewards/rejected": -12.184309005737305, + "step": 1687 + }, + { + "epoch": 2.21, + "learning_rate": 4.4724471273955686e-05, + "logits/chosen": -2.018510341644287, + "logits/rejected": -2.0894618034362793, + "logps/chosen": -231.053466796875, + "logps/rejected": -336.80023193359375, + "loss": 0.0048, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.892425537109375, + "rewards/margins": 8.695682525634766, + "rewards/rejected": -13.58810806274414, + "step": 1688 + }, + { + "epoch": 2.21, + "learning_rate": 4.471806950077541e-05, + "logits/chosen": -1.6260873079299927, + "logits/rejected": -1.654736876487732, + "logps/chosen": -198.38699340820312, + "logps/rejected": -290.44287109375, + "loss": 0.0025, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.7985918521881104, + "rewards/margins": 8.221394538879395, + "rewards/rejected": -12.019987106323242, + "step": 1689 + }, + { + "epoch": 2.21, + "learning_rate": 4.47116643044805e-05, + "logits/chosen": -1.8681111335754395, + "logits/rejected": -1.9466384649276733, + "logps/chosen": -228.25901794433594, + "logps/rejected": -361.3291320800781, + "loss": 0.0038, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.587683200836182, + "rewards/margins": 9.266342163085938, + "rewards/rejected": -13.854026794433594, + "step": 1690 + }, + { + "epoch": 2.21, + "learning_rate": 4.470525568618296e-05, + "logits/chosen": -1.870570182800293, + "logits/rejected": -1.9170101881027222, + "logps/chosen": -257.67340087890625, + "logps/rejected": -370.72943115234375, + "loss": 0.0454, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.8660976886749268, + "rewards/margins": 11.009435653686523, + "rewards/rejected": -14.875534057617188, + "step": 1691 + }, + { + "epoch": 2.21, + "learning_rate": 4.469884364699531e-05, + "logits/chosen": -1.875512957572937, + "logits/rejected": -2.05307674407959, + "logps/chosen": -236.80178833007812, + "logps/rejected": -374.8936767578125, + "loss": 0.0879, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.758052825927734, + "rewards/margins": 9.715606689453125, + "rewards/rejected": -14.47365951538086, + "step": 1692 + }, + { + "epoch": 2.22, + "learning_rate": 4.469242818803072e-05, + "logits/chosen": -1.8573062419891357, + "logits/rejected": -1.7853858470916748, + "logps/chosen": -311.140625, + "logps/rejected": -379.6907653808594, + "loss": 0.0916, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.95181941986084, + "rewards/margins": 8.339434623718262, + "rewards/rejected": -12.291254043579102, + "step": 1693 + }, + { + "epoch": 2.22, + "learning_rate": 4.4686009310402925e-05, + "logits/chosen": -1.7488088607788086, + "logits/rejected": -1.765557885169983, + "logps/chosen": -228.75222778320312, + "logps/rejected": -324.71331787109375, + "loss": 0.0484, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.402834415435791, + "rewards/margins": 8.772923469543457, + "rewards/rejected": -13.175758361816406, + "step": 1694 + }, + { + "epoch": 2.22, + "learning_rate": 4.4679587015226253e-05, + "logits/chosen": -1.8654567003250122, + "logits/rejected": -1.9250493049621582, + "logps/chosen": -239.89849853515625, + "logps/rejected": -349.3185119628906, + "loss": 0.0413, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.728438854217529, + "rewards/margins": 8.32399845123291, + "rewards/rejected": -14.052436828613281, + "step": 1695 + }, + { + "epoch": 2.22, + "learning_rate": 4.467316130361565e-05, + "logits/chosen": -1.8583791255950928, + "logits/rejected": -1.8942469358444214, + "logps/chosen": -237.06396484375, + "logps/rejected": -343.1637878417969, + "loss": 0.0162, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.936460018157959, + "rewards/margins": 8.794307708740234, + "rewards/rejected": -13.730766296386719, + "step": 1696 + }, + { + "epoch": 2.22, + "learning_rate": 4.466673217668662e-05, + "logits/chosen": -2.079540252685547, + "logits/rejected": -2.164118528366089, + "logps/chosen": -279.9512023925781, + "logps/rejected": -423.8478698730469, + "loss": 0.0436, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.133212566375732, + "rewards/margins": 10.697543144226074, + "rewards/rejected": -15.830756187438965, + "step": 1697 + }, + { + "epoch": 2.22, + "learning_rate": 4.466029963555529e-05, + "logits/chosen": -1.917163372039795, + "logits/rejected": -1.8837778568267822, + "logps/chosen": -233.43521118164062, + "logps/rejected": -284.9217834472656, + "loss": 0.0964, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.6208252906799316, + "rewards/margins": 6.869645595550537, + "rewards/rejected": -10.490470886230469, + "step": 1698 + }, + { + "epoch": 2.22, + "learning_rate": 4.465386368133837e-05, + "logits/chosen": -1.918818473815918, + "logits/rejected": -1.984576940536499, + "logps/chosen": -248.1490936279297, + "logps/rejected": -338.5713806152344, + "loss": 0.0441, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.999472141265869, + "rewards/margins": 7.363437652587891, + "rewards/rejected": -13.362909317016602, + "step": 1699 + }, + { + "epoch": 2.22, + "learning_rate": 4.464742431515315e-05, + "logits/chosen": -1.8976091146469116, + "logits/rejected": -1.9875017404556274, + "logps/chosen": -208.65817260742188, + "logps/rejected": -305.20428466796875, + "loss": 0.1059, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.453165531158447, + "rewards/margins": 9.020037651062012, + "rewards/rejected": -13.473203659057617, + "step": 1700 + }, + { + "epoch": 2.23, + "learning_rate": 4.4640981538117534e-05, + "logits/chosen": -1.6591687202453613, + "logits/rejected": -1.6958729028701782, + "logps/chosen": -226.06491088867188, + "logps/rejected": -328.7708740234375, + "loss": 0.048, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.8626770973205566, + "rewards/margins": 9.48762321472168, + "rewards/rejected": -13.350301742553711, + "step": 1701 + }, + { + "epoch": 2.23, + "learning_rate": 4.4634535351349995e-05, + "logits/chosen": -1.8268656730651855, + "logits/rejected": -1.9317461252212524, + "logps/chosen": -192.97369384765625, + "logps/rejected": -290.1656494140625, + "loss": 0.0599, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.650546550750732, + "rewards/margins": 7.853985786437988, + "rewards/rejected": -12.504531860351562, + "step": 1702 + }, + { + "epoch": 2.23, + "learning_rate": 4.462808575596962e-05, + "logits/chosen": -1.9132554531097412, + "logits/rejected": -1.965674877166748, + "logps/chosen": -218.21656799316406, + "logps/rejected": -364.0094909667969, + "loss": 0.0013, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.798173427581787, + "rewards/margins": 9.15748119354248, + "rewards/rejected": -13.955655097961426, + "step": 1703 + }, + { + "epoch": 2.23, + "learning_rate": 4.4621632753096066e-05, + "logits/chosen": -1.9854216575622559, + "logits/rejected": -2.0571200847625732, + "logps/chosen": -252.40103149414062, + "logps/rejected": -309.4356689453125, + "loss": 0.071, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.203364372253418, + "rewards/margins": 6.705964088439941, + "rewards/rejected": -11.90932846069336, + "step": 1704 + }, + { + "epoch": 2.23, + "learning_rate": 4.461517634384961e-05, + "logits/chosen": -1.6782927513122559, + "logits/rejected": -1.6738965511322021, + "logps/chosen": -216.7491912841797, + "logps/rejected": -303.1639709472656, + "loss": 0.1341, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.4354166984558105, + "rewards/margins": 7.658374786376953, + "rewards/rejected": -11.093792915344238, + "step": 1705 + }, + { + "epoch": 2.23, + "learning_rate": 4.460871652935109e-05, + "logits/chosen": -1.87294602394104, + "logits/rejected": -1.9136104583740234, + "logps/chosen": -267.9233093261719, + "logps/rejected": -348.4906921386719, + "loss": 0.1321, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.544414043426514, + "rewards/margins": 7.048198699951172, + "rewards/rejected": -11.59261417388916, + "step": 1706 + }, + { + "epoch": 2.23, + "learning_rate": 4.4602253310721964e-05, + "logits/chosen": -1.7989273071289062, + "logits/rejected": -1.8650834560394287, + "logps/chosen": -252.19677734375, + "logps/rejected": -340.9962158203125, + "loss": 0.0045, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.7554030418396, + "rewards/margins": 8.529303550720215, + "rewards/rejected": -13.284706115722656, + "step": 1707 + }, + { + "epoch": 2.24, + "learning_rate": 4.4595786689084246e-05, + "logits/chosen": -2.063161611557007, + "logits/rejected": -2.0948920249938965, + "logps/chosen": -233.23056030273438, + "logps/rejected": -325.5529479980469, + "loss": 0.0056, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.490490674972534, + "rewards/margins": 10.117737770080566, + "rewards/rejected": -13.608229637145996, + "step": 1708 + }, + { + "epoch": 2.24, + "learning_rate": 4.4589316665560564e-05, + "logits/chosen": -1.9486212730407715, + "logits/rejected": -1.88687264919281, + "logps/chosen": -288.6056213378906, + "logps/rejected": -419.3163757324219, + "loss": 0.0019, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.242236137390137, + "rewards/margins": 11.447809219360352, + "rewards/rejected": -15.690044403076172, + "step": 1709 + }, + { + "epoch": 2.24, + "learning_rate": 4.458284324127415e-05, + "logits/chosen": -1.4820352792739868, + "logits/rejected": -1.5303387641906738, + "logps/chosen": -222.28982543945312, + "logps/rejected": -320.25238037109375, + "loss": 0.0892, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.2132415771484375, + "rewards/margins": 8.695575714111328, + "rewards/rejected": -12.908817291259766, + "step": 1710 + }, + { + "epoch": 2.24, + "learning_rate": 4.457636641734879e-05, + "logits/chosen": -1.919441819190979, + "logits/rejected": -1.820131540298462, + "logps/chosen": -202.3666534423828, + "logps/rejected": -269.91851806640625, + "loss": 0.0526, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.528416633605957, + "rewards/margins": 9.030194282531738, + "rewards/rejected": -12.558610916137695, + "step": 1711 + }, + { + "epoch": 2.24, + "learning_rate": 4.456988619490889e-05, + "logits/chosen": -1.9204466342926025, + "logits/rejected": -1.8682175874710083, + "logps/chosen": -261.3448181152344, + "logps/rejected": -361.8304443359375, + "loss": 0.0871, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.659709930419922, + "rewards/margins": 10.16419506072998, + "rewards/rejected": -14.823904991149902, + "step": 1712 + }, + { + "epoch": 2.24, + "learning_rate": 4.456340257507944e-05, + "logits/chosen": -1.9544986486434937, + "logits/rejected": -2.006072759628296, + "logps/chosen": -173.63169860839844, + "logps/rejected": -272.7809143066406, + "loss": 0.0544, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.102273941040039, + "rewards/margins": 7.849088668823242, + "rewards/rejected": -10.951361656188965, + "step": 1713 + }, + { + "epoch": 2.24, + "learning_rate": 4.4556915558986e-05, + "logits/chosen": -2.0595617294311523, + "logits/rejected": -2.0527684688568115, + "logps/chosen": -249.13677978515625, + "logps/rejected": -318.8485107421875, + "loss": 0.0058, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.425116062164307, + "rewards/margins": 9.156496047973633, + "rewards/rejected": -13.581611633300781, + "step": 1714 + }, + { + "epoch": 2.24, + "learning_rate": 4.455042514775475e-05, + "logits/chosen": -2.034576416015625, + "logits/rejected": -2.082951784133911, + "logps/chosen": -238.15553283691406, + "logps/rejected": -313.01251220703125, + "loss": 0.0878, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.427706241607666, + "rewards/margins": 7.517872333526611, + "rewards/rejected": -11.945578575134277, + "step": 1715 + }, + { + "epoch": 2.25, + "learning_rate": 4.454393134251243e-05, + "logits/chosen": -1.9722774028778076, + "logits/rejected": -2.0078694820404053, + "logps/chosen": -214.91195678710938, + "logps/rejected": -305.921630859375, + "loss": 0.0893, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.9531149864196777, + "rewards/margins": 8.302417755126953, + "rewards/rejected": -12.255531311035156, + "step": 1716 + }, + { + "epoch": 2.25, + "learning_rate": 4.4537434144386406e-05, + "logits/chosen": -2.0352540016174316, + "logits/rejected": -2.0985212326049805, + "logps/chosen": -213.61550903320312, + "logps/rejected": -310.8976745605469, + "loss": 0.0445, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.662770748138428, + "rewards/margins": 8.683988571166992, + "rewards/rejected": -13.346759796142578, + "step": 1717 + }, + { + "epoch": 2.25, + "learning_rate": 4.4530933554504584e-05, + "logits/chosen": -1.7264947891235352, + "logits/rejected": -1.7682827711105347, + "logps/chosen": -173.7081298828125, + "logps/rejected": -313.7381286621094, + "loss": 0.0953, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.215333461761475, + "rewards/margins": 8.603334426879883, + "rewards/rejected": -12.8186674118042, + "step": 1718 + }, + { + "epoch": 2.25, + "learning_rate": 4.45244295739955e-05, + "logits/chosen": -1.9386180639266968, + "logits/rejected": -2.0219626426696777, + "logps/chosen": -310.0873107910156, + "logps/rejected": -405.73297119140625, + "loss": 0.001, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.907812595367432, + "rewards/margins": 11.40237045288086, + "rewards/rejected": -16.310182571411133, + "step": 1719 + }, + { + "epoch": 2.25, + "learning_rate": 4.451792220398826e-05, + "logits/chosen": -1.8895277976989746, + "logits/rejected": -1.7938296794891357, + "logps/chosen": -282.5625305175781, + "logps/rejected": -364.5167236328125, + "loss": 0.0722, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.161428928375244, + "rewards/margins": 8.818914413452148, + "rewards/rejected": -13.98034381866455, + "step": 1720 + }, + { + "epoch": 2.25, + "learning_rate": 4.4511411445612574e-05, + "logits/chosen": -2.0378122329711914, + "logits/rejected": -2.16402006149292, + "logps/chosen": -252.35549926757812, + "logps/rejected": -356.19903564453125, + "loss": 0.1538, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.731842994689941, + "rewards/margins": 6.997723579406738, + "rewards/rejected": -12.729567527770996, + "step": 1721 + }, + { + "epoch": 2.25, + "learning_rate": 4.450489729999871e-05, + "logits/chosen": -1.8221724033355713, + "logits/rejected": -1.8675659894943237, + "logps/chosen": -201.3778076171875, + "logps/rejected": -327.18841552734375, + "loss": 0.0127, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.0520832538604736, + "rewards/margins": 10.92273998260498, + "rewards/rejected": -13.974822044372559, + "step": 1722 + }, + { + "epoch": 2.25, + "learning_rate": 4.4498379768277555e-05, + "logits/chosen": -1.6914008855819702, + "logits/rejected": -1.7110141515731812, + "logps/chosen": -222.4412841796875, + "logps/rejected": -326.3401184082031, + "loss": 0.0122, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.712990760803223, + "rewards/margins": 7.869570732116699, + "rewards/rejected": -12.582560539245605, + "step": 1723 + }, + { + "epoch": 2.26, + "learning_rate": 4.449185885158056e-05, + "logits/chosen": -1.650061845779419, + "logits/rejected": -1.771134614944458, + "logps/chosen": -200.29811096191406, + "logps/rejected": -301.3843688964844, + "loss": 0.0467, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.501410961151123, + "rewards/margins": 8.028314590454102, + "rewards/rejected": -12.529725074768066, + "step": 1724 + }, + { + "epoch": 2.26, + "learning_rate": 4.448533455103979e-05, + "logits/chosen": -1.8757323026657104, + "logits/rejected": -1.9865539073944092, + "logps/chosen": -233.38584899902344, + "logps/rejected": -327.2201232910156, + "loss": 0.0065, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.171833038330078, + "rewards/margins": 7.656855583190918, + "rewards/rejected": -12.82868766784668, + "step": 1725 + }, + { + "epoch": 2.26, + "learning_rate": 4.447880686778788e-05, + "logits/chosen": -1.9543880224227905, + "logits/rejected": -1.994931697845459, + "logps/chosen": -211.9916229248047, + "logps/rejected": -283.35003662109375, + "loss": 0.0667, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.0559065341949463, + "rewards/margins": 8.326825141906738, + "rewards/rejected": -11.382730484008789, + "step": 1726 + }, + { + "epoch": 2.26, + "learning_rate": 4.447227580295805e-05, + "logits/chosen": -2.1561076641082764, + "logits/rejected": -2.169799327850342, + "logps/chosen": -227.1396484375, + "logps/rejected": -322.3929443359375, + "loss": 0.1708, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.8819479942321777, + "rewards/margins": 8.353205680847168, + "rewards/rejected": -12.235153198242188, + "step": 1727 + }, + { + "epoch": 2.26, + "learning_rate": 4.446574135768411e-05, + "logits/chosen": -1.7280957698822021, + "logits/rejected": -1.8162102699279785, + "logps/chosen": -219.45413208007812, + "logps/rejected": -305.6361083984375, + "loss": 0.0064, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.295023441314697, + "rewards/margins": 9.003251075744629, + "rewards/rejected": -13.298274993896484, + "step": 1728 + }, + { + "epoch": 2.26, + "learning_rate": 4.445920353310046e-05, + "logits/chosen": -2.0517849922180176, + "logits/rejected": -2.169731378555298, + "logps/chosen": -244.9127655029297, + "logps/rejected": -339.7769470214844, + "loss": 0.0464, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.833156585693359, + "rewards/margins": 8.40161418914795, + "rewards/rejected": -13.234771728515625, + "step": 1729 + }, + { + "epoch": 2.26, + "learning_rate": 4.445266233034209e-05, + "logits/chosen": -1.9063819646835327, + "logits/rejected": -1.9939993619918823, + "logps/chosen": -210.46400451660156, + "logps/rejected": -321.38134765625, + "loss": 0.0088, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.046882629394531, + "rewards/margins": 9.02386474609375, + "rewards/rejected": -13.070748329162598, + "step": 1730 + }, + { + "epoch": 2.27, + "learning_rate": 4.444611775054457e-05, + "logits/chosen": -2.016284942626953, + "logits/rejected": -2.0195984840393066, + "logps/chosen": -220.59742736816406, + "logps/rejected": -260.17816162109375, + "loss": 0.0771, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.4410529136657715, + "rewards/margins": 6.340338706970215, + "rewards/rejected": -9.781391143798828, + "step": 1731 + }, + { + "epoch": 2.27, + "learning_rate": 4.443956979484406e-05, + "logits/chosen": -1.9992839097976685, + "logits/rejected": -1.943981647491455, + "logps/chosen": -236.16993713378906, + "logps/rejected": -313.77069091796875, + "loss": 0.0151, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.058987617492676, + "rewards/margins": 8.646703720092773, + "rewards/rejected": -12.70569133758545, + "step": 1732 + }, + { + "epoch": 2.27, + "learning_rate": 4.44330184643773e-05, + "logits/chosen": -1.8650450706481934, + "logits/rejected": -1.7884596586227417, + "logps/chosen": -224.2449188232422, + "logps/rejected": -271.3692321777344, + "loss": 0.1366, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.8722286224365234, + "rewards/margins": 6.974151134490967, + "rewards/rejected": -10.846380233764648, + "step": 1733 + }, + { + "epoch": 2.27, + "learning_rate": 4.442646376028162e-05, + "logits/chosen": -1.7677942514419556, + "logits/rejected": -1.826995611190796, + "logps/chosen": -224.70472717285156, + "logps/rejected": -334.978271484375, + "loss": 0.1076, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.9479570388793945, + "rewards/margins": 8.14836311340332, + "rewards/rejected": -13.096320152282715, + "step": 1734 + }, + { + "epoch": 2.27, + "learning_rate": 4.441990568369494e-05, + "logits/chosen": -1.9324272871017456, + "logits/rejected": -2.0757665634155273, + "logps/chosen": -217.09535217285156, + "logps/rejected": -313.4742736816406, + "loss": 0.0459, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.570109844207764, + "rewards/margins": 8.275823593139648, + "rewards/rejected": -12.84593391418457, + "step": 1735 + }, + { + "epoch": 2.27, + "learning_rate": 4.441334423575576e-05, + "logits/chosen": -1.9709575176239014, + "logits/rejected": -1.8980768918991089, + "logps/chosen": -233.23419189453125, + "logps/rejected": -291.5000305175781, + "loss": 0.0918, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.155413627624512, + "rewards/margins": 5.9400105476379395, + "rewards/rejected": -10.09542465209961, + "step": 1736 + }, + { + "epoch": 2.27, + "learning_rate": 4.440677941760316e-05, + "logits/chosen": -1.8670390844345093, + "logits/rejected": -1.864248275756836, + "logps/chosen": -177.4266815185547, + "logps/rejected": -256.11529541015625, + "loss": 0.0491, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.6511425971984863, + "rewards/margins": 6.640891075134277, + "rewards/rejected": -10.292034149169922, + "step": 1737 + }, + { + "epoch": 2.27, + "learning_rate": 4.4400211230376826e-05, + "logits/chosen": -2.05721378326416, + "logits/rejected": -2.158565044403076, + "logps/chosen": -260.0497741699219, + "logps/rejected": -376.5098571777344, + "loss": 0.0016, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.747610569000244, + "rewards/margins": 9.646328926086426, + "rewards/rejected": -14.393939971923828, + "step": 1738 + }, + { + "epoch": 2.28, + "learning_rate": 4.4393639675217004e-05, + "logits/chosen": -1.7898157835006714, + "logits/rejected": -1.9009355306625366, + "logps/chosen": -203.9080352783203, + "logps/rejected": -296.2388610839844, + "loss": 0.095, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.05819034576416, + "rewards/margins": 6.971330165863037, + "rewards/rejected": -11.029520034790039, + "step": 1739 + }, + { + "epoch": 2.28, + "learning_rate": 4.438706475326453e-05, + "logits/chosen": -1.7593650817871094, + "logits/rejected": -1.8225117921829224, + "logps/chosen": -205.5297088623047, + "logps/rejected": -274.93328857421875, + "loss": 0.0872, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.9774317741394043, + "rewards/margins": 7.377807140350342, + "rewards/rejected": -11.355239868164062, + "step": 1740 + }, + { + "epoch": 2.28, + "learning_rate": 4.438048646566084e-05, + "logits/chosen": -1.8858990669250488, + "logits/rejected": -1.9695152044296265, + "logps/chosen": -215.17010498046875, + "logps/rejected": -334.5107727050781, + "loss": 0.0057, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.608569622039795, + "rewards/margins": 8.680890083312988, + "rewards/rejected": -12.289458274841309, + "step": 1741 + }, + { + "epoch": 2.28, + "learning_rate": 4.437390481354795e-05, + "logits/chosen": -2.0719106197357178, + "logits/rejected": -2.1239736080169678, + "logps/chosen": -251.9342041015625, + "logps/rejected": -374.99908447265625, + "loss": 0.0016, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.334467887878418, + "rewards/margins": 9.548616409301758, + "rewards/rejected": -13.883084297180176, + "step": 1742 + }, + { + "epoch": 2.28, + "learning_rate": 4.436731979806844e-05, + "logits/chosen": -1.8284627199172974, + "logits/rejected": -1.918864369392395, + "logps/chosen": -195.5998077392578, + "logps/rejected": -328.7000732421875, + "loss": 0.0021, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.969945192337036, + "rewards/margins": 9.359249114990234, + "rewards/rejected": -13.329195022583008, + "step": 1743 + }, + { + "epoch": 2.28, + "learning_rate": 4.4360731420365486e-05, + "logits/chosen": -1.6309916973114014, + "logits/rejected": -1.718712329864502, + "logps/chosen": -202.272705078125, + "logps/rejected": -307.3332214355469, + "loss": 0.0657, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.2723641395568848, + "rewards/margins": 8.126914024353027, + "rewards/rejected": -11.39927864074707, + "step": 1744 + }, + { + "epoch": 2.28, + "learning_rate": 4.4354139681582865e-05, + "logits/chosen": -1.9175488948822021, + "logits/rejected": -1.9487149715423584, + "logps/chosen": -284.1695556640625, + "logps/rejected": -303.47442626953125, + "loss": 0.0522, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.2527360916137695, + "rewards/margins": 7.352540016174316, + "rewards/rejected": -11.605276107788086, + "step": 1745 + }, + { + "epoch": 2.28, + "learning_rate": 4.434754458286491e-05, + "logits/chosen": -2.0459342002868652, + "logits/rejected": -2.0865938663482666, + "logps/chosen": -235.94528198242188, + "logps/rejected": -334.06109619140625, + "loss": 0.0877, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.6131935119628906, + "rewards/margins": 8.025009155273438, + "rewards/rejected": -11.638202667236328, + "step": 1746 + }, + { + "epoch": 2.29, + "learning_rate": 4.434094612535655e-05, + "logits/chosen": -2.1044609546661377, + "logits/rejected": -2.0698766708374023, + "logps/chosen": -233.37014770507812, + "logps/rejected": -340.31756591796875, + "loss": 0.0065, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.785109043121338, + "rewards/margins": 9.152952194213867, + "rewards/rejected": -13.938060760498047, + "step": 1747 + }, + { + "epoch": 2.29, + "learning_rate": 4.433434431020331e-05, + "logits/chosen": -1.820961594581604, + "logits/rejected": -1.9584262371063232, + "logps/chosen": -175.23321533203125, + "logps/rejected": -304.24334716796875, + "loss": 0.1015, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.8073019981384277, + "rewards/margins": 7.9026360511779785, + "rewards/rejected": -10.709938049316406, + "step": 1748 + }, + { + "epoch": 2.29, + "learning_rate": 4.432773913855127e-05, + "logits/chosen": -1.9037257432937622, + "logits/rejected": -1.7631274461746216, + "logps/chosen": -224.0203857421875, + "logps/rejected": -311.80560302734375, + "loss": 0.0099, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.7373249530792236, + "rewards/margins": 8.073929786682129, + "rewards/rejected": -11.811254501342773, + "step": 1749 + }, + { + "epoch": 2.29, + "learning_rate": 4.4321130611547116e-05, + "logits/chosen": -1.9026196002960205, + "logits/rejected": -1.9815964698791504, + "logps/chosen": -212.11672973632812, + "logps/rejected": -305.43682861328125, + "loss": 0.166, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.651831865310669, + "rewards/margins": 6.873808860778809, + "rewards/rejected": -10.525641441345215, + "step": 1750 + }, + { + "epoch": 2.29, + "learning_rate": 4.431451873033809e-05, + "logits/chosen": -1.6320197582244873, + "logits/rejected": -1.6567957401275635, + "logps/chosen": -198.2225799560547, + "logps/rejected": -279.5611877441406, + "loss": 0.1458, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.1111578941345215, + "rewards/margins": 5.316527366638184, + "rewards/rejected": -9.427684783935547, + "step": 1751 + }, + { + "epoch": 2.29, + "learning_rate": 4.4307903496072053e-05, + "logits/chosen": -1.90899658203125, + "logits/rejected": -1.9217326641082764, + "logps/chosen": -170.5017547607422, + "logps/rejected": -242.19381713867188, + "loss": 0.1412, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.4766669273376465, + "rewards/margins": 6.857580184936523, + "rewards/rejected": -10.334247589111328, + "step": 1752 + }, + { + "epoch": 2.29, + "learning_rate": 4.430128490989742e-05, + "logits/chosen": -2.097954273223877, + "logits/rejected": -2.082620859146118, + "logps/chosen": -204.95272827148438, + "logps/rejected": -305.71075439453125, + "loss": 0.0723, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.122200012207031, + "rewards/margins": 7.713401794433594, + "rewards/rejected": -12.835600852966309, + "step": 1753 + }, + { + "epoch": 2.3, + "learning_rate": 4.429466297296321e-05, + "logits/chosen": -1.9181946516036987, + "logits/rejected": -2.054036855697632, + "logps/chosen": -254.34112548828125, + "logps/rejected": -373.45904541015625, + "loss": 0.0446, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.675026893615723, + "rewards/margins": 8.775556564331055, + "rewards/rejected": -13.450584411621094, + "step": 1754 + }, + { + "epoch": 2.3, + "learning_rate": 4.4288037686418995e-05, + "logits/chosen": -1.8053514957427979, + "logits/rejected": -1.8212918043136597, + "logps/chosen": -294.88299560546875, + "logps/rejected": -375.58001708984375, + "loss": 0.051, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.989837169647217, + "rewards/margins": 7.898305892944336, + "rewards/rejected": -12.888142585754395, + "step": 1755 + }, + { + "epoch": 2.3, + "learning_rate": 4.4281409051414944e-05, + "logits/chosen": -1.6933479309082031, + "logits/rejected": -1.8062795400619507, + "logps/chosen": -231.76593017578125, + "logps/rejected": -350.6939392089844, + "loss": 0.0874, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.724734306335449, + "rewards/margins": 8.08170223236084, + "rewards/rejected": -12.806437492370605, + "step": 1756 + }, + { + "epoch": 2.3, + "learning_rate": 4.427477706910182e-05, + "logits/chosen": -1.7838891744613647, + "logits/rejected": -1.9130263328552246, + "logps/chosen": -270.8051452636719, + "logps/rejected": -399.8632507324219, + "loss": 0.0112, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.593284606933594, + "rewards/margins": 9.894659042358398, + "rewards/rejected": -15.487942695617676, + "step": 1757 + }, + { + "epoch": 2.3, + "learning_rate": 4.426814174063094e-05, + "logits/chosen": -1.8285061120986938, + "logits/rejected": -1.9469319581985474, + "logps/chosen": -194.19351196289062, + "logps/rejected": -354.72576904296875, + "loss": 0.0081, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.976185321807861, + "rewards/margins": 7.8145551681518555, + "rewards/rejected": -12.790740013122559, + "step": 1758 + }, + { + "epoch": 2.3, + "learning_rate": 4.4261503067154234e-05, + "logits/chosen": -1.8068652153015137, + "logits/rejected": -1.9476265907287598, + "logps/chosen": -279.4858703613281, + "logps/rejected": -366.8956298828125, + "loss": 0.0441, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.46669864654541, + "rewards/margins": 9.39613151550293, + "rewards/rejected": -14.86283016204834, + "step": 1759 + }, + { + "epoch": 2.3, + "learning_rate": 4.425486104982418e-05, + "logits/chosen": -1.9092851877212524, + "logits/rejected": -1.862753987312317, + "logps/chosen": -266.92572021484375, + "logps/rejected": -353.6373291015625, + "loss": 0.0009, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.8829591274261475, + "rewards/margins": 10.231354713439941, + "rewards/rejected": -14.114313125610352, + "step": 1760 + }, + { + "epoch": 2.3, + "learning_rate": 4.424821568979386e-05, + "logits/chosen": -1.883470058441162, + "logits/rejected": -1.826371669769287, + "logps/chosen": -256.5468444824219, + "logps/rejected": -334.00390625, + "loss": 0.0872, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.85820198059082, + "rewards/margins": 7.960251331329346, + "rewards/rejected": -12.818452835083008, + "step": 1761 + }, + { + "epoch": 2.31, + "learning_rate": 4.424156698821693e-05, + "logits/chosen": -1.6825509071350098, + "logits/rejected": -1.700924038887024, + "logps/chosen": -253.4066925048828, + "logps/rejected": -345.5010070800781, + "loss": 0.162, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.353029251098633, + "rewards/margins": 8.234490394592285, + "rewards/rejected": -13.587520599365234, + "step": 1762 + }, + { + "epoch": 2.31, + "learning_rate": 4.423491494624761e-05, + "logits/chosen": -2.003420829772949, + "logits/rejected": -2.0149574279785156, + "logps/chosen": -255.87901306152344, + "logps/rejected": -313.81005859375, + "loss": 0.0876, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.805246829986572, + "rewards/margins": 7.625028610229492, + "rewards/rejected": -12.430275917053223, + "step": 1763 + }, + { + "epoch": 2.31, + "learning_rate": 4.422825956504072e-05, + "logits/chosen": -1.9779905080795288, + "logits/rejected": -1.8467421531677246, + "logps/chosen": -302.4613037109375, + "logps/rejected": -364.8889465332031, + "loss": 0.0454, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.031822681427002, + "rewards/margins": 8.41565990447998, + "rewards/rejected": -12.447482109069824, + "step": 1764 + }, + { + "epoch": 2.31, + "learning_rate": 4.422160084575167e-05, + "logits/chosen": -1.8664950132369995, + "logits/rejected": -1.8366044759750366, + "logps/chosen": -229.32427978515625, + "logps/rejected": -269.58538818359375, + "loss": 0.09, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.961425304412842, + "rewards/margins": 6.573773384094238, + "rewards/rejected": -11.535199165344238, + "step": 1765 + }, + { + "epoch": 2.31, + "learning_rate": 4.421493878953641e-05, + "logits/chosen": -1.8571369647979736, + "logits/rejected": -1.9223442077636719, + "logps/chosen": -249.59237670898438, + "logps/rejected": -351.52716064453125, + "loss": 0.0433, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.7298736572265625, + "rewards/margins": 8.822587013244629, + "rewards/rejected": -14.552459716796875, + "step": 1766 + }, + { + "epoch": 2.31, + "learning_rate": 4.42082733975515e-05, + "logits/chosen": -1.4088032245635986, + "logits/rejected": -1.5760774612426758, + "logps/chosen": -260.4433898925781, + "logps/rejected": -356.95819091796875, + "loss": 0.0039, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.6229753494262695, + "rewards/margins": 8.967402458190918, + "rewards/rejected": -14.590376853942871, + "step": 1767 + }, + { + "epoch": 2.31, + "learning_rate": 4.420160467095408e-05, + "logits/chosen": -2.181647539138794, + "logits/rejected": -2.090801954269409, + "logps/chosen": -221.33909606933594, + "logps/rejected": -306.36138916015625, + "loss": 0.0528, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.981787919998169, + "rewards/margins": 9.448159217834473, + "rewards/rejected": -13.429945945739746, + "step": 1768 + }, + { + "epoch": 2.32, + "learning_rate": 4.419493261090186e-05, + "logits/chosen": -1.6997369527816772, + "logits/rejected": -1.7968833446502686, + "logps/chosen": -195.84988403320312, + "logps/rejected": -321.94659423828125, + "loss": 0.0473, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.023075103759766, + "rewards/margins": 8.610339164733887, + "rewards/rejected": -13.633415222167969, + "step": 1769 + }, + { + "epoch": 2.32, + "learning_rate": 4.418825721855311e-05, + "logits/chosen": -2.084487199783325, + "logits/rejected": -2.1054601669311523, + "logps/chosen": -272.1376037597656, + "logps/rejected": -358.4344482421875, + "loss": 0.0478, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.109389781951904, + "rewards/margins": 7.3886823654174805, + "rewards/rejected": -12.498072624206543, + "step": 1770 + }, + { + "epoch": 2.32, + "learning_rate": 4.418157849506672e-05, + "logits/chosen": -1.8278807401657104, + "logits/rejected": -1.8396012783050537, + "logps/chosen": -218.77023315429688, + "logps/rejected": -292.850341796875, + "loss": 0.142, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.907233715057373, + "rewards/margins": 7.580648422241211, + "rewards/rejected": -12.487882614135742, + "step": 1771 + }, + { + "epoch": 2.32, + "learning_rate": 4.417489644160213e-05, + "logits/chosen": -2.0319652557373047, + "logits/rejected": -2.0633068084716797, + "logps/chosen": -260.70452880859375, + "logps/rejected": -345.8565368652344, + "loss": 0.0033, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.086930274963379, + "rewards/margins": 9.484430313110352, + "rewards/rejected": -14.571361541748047, + "step": 1772 + }, + { + "epoch": 2.32, + "learning_rate": 4.4168211059319356e-05, + "logits/chosen": -1.8316105604171753, + "logits/rejected": -1.7764731645584106, + "logps/chosen": -230.56332397460938, + "logps/rejected": -325.6812744140625, + "loss": 0.0442, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.026920318603516, + "rewards/margins": 8.883038520812988, + "rewards/rejected": -13.909958839416504, + "step": 1773 + }, + { + "epoch": 2.32, + "learning_rate": 4.4161522349379e-05, + "logits/chosen": -1.7927948236465454, + "logits/rejected": -1.8635965585708618, + "logps/chosen": -197.43812561035156, + "logps/rejected": -288.6591491699219, + "loss": 0.0454, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.8581838607788086, + "rewards/margins": 8.61027717590332, + "rewards/rejected": -12.468461036682129, + "step": 1774 + }, + { + "epoch": 2.32, + "learning_rate": 4.415483031294225e-05, + "logits/chosen": -1.9382948875427246, + "logits/rejected": -1.9620851278305054, + "logps/chosen": -232.10604858398438, + "logps/rejected": -324.19183349609375, + "loss": 0.0445, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.678308963775635, + "rewards/margins": 9.27111530303955, + "rewards/rejected": -13.949423789978027, + "step": 1775 + }, + { + "epoch": 2.32, + "learning_rate": 4.414813495117087e-05, + "logits/chosen": -1.6935954093933105, + "logits/rejected": -1.6843981742858887, + "logps/chosen": -298.6250305175781, + "logps/rejected": -375.5038146972656, + "loss": 0.0448, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.873818397521973, + "rewards/margins": 8.789769172668457, + "rewards/rejected": -14.66358757019043, + "step": 1776 + }, + { + "epoch": 2.33, + "learning_rate": 4.4141436265227174e-05, + "logits/chosen": -1.7664049863815308, + "logits/rejected": -1.801297903060913, + "logps/chosen": -217.70216369628906, + "logps/rejected": -327.42449951171875, + "loss": 0.1323, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.481310844421387, + "rewards/margins": 7.413915634155273, + "rewards/rejected": -12.89522647857666, + "step": 1777 + }, + { + "epoch": 2.33, + "learning_rate": 4.413473425627408e-05, + "logits/chosen": -1.9183545112609863, + "logits/rejected": -1.9855916500091553, + "logps/chosen": -220.51705932617188, + "logps/rejected": -372.94830322265625, + "loss": 0.0546, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.821767807006836, + "rewards/margins": 10.212467193603516, + "rewards/rejected": -15.034236907958984, + "step": 1778 + }, + { + "epoch": 2.33, + "learning_rate": 4.4128028925475095e-05, + "logits/chosen": -2.1654553413391113, + "logits/rejected": -2.2233729362487793, + "logps/chosen": -315.9672546386719, + "logps/rejected": -419.172607421875, + "loss": 0.0869, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.519159317016602, + "rewards/margins": 9.854990005493164, + "rewards/rejected": -15.374149322509766, + "step": 1779 + }, + { + "epoch": 2.33, + "learning_rate": 4.412132027399426e-05, + "logits/chosen": -2.056398630142212, + "logits/rejected": -2.127978801727295, + "logps/chosen": -201.8386993408203, + "logps/rejected": -338.69744873046875, + "loss": 0.0939, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.55429744720459, + "rewards/margins": 8.227093696594238, + "rewards/rejected": -12.781391143798828, + "step": 1780 + }, + { + "epoch": 2.33, + "learning_rate": 4.411460830299622e-05, + "logits/chosen": -1.7686713933944702, + "logits/rejected": -1.9483716487884521, + "logps/chosen": -258.9558410644531, + "logps/rejected": -341.24285888671875, + "loss": 0.1308, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.698920726776123, + "rewards/margins": 7.26412296295166, + "rewards/rejected": -13.963045120239258, + "step": 1781 + }, + { + "epoch": 2.33, + "learning_rate": 4.410789301364621e-05, + "logits/chosen": -2.030712604522705, + "logits/rejected": -2.0665645599365234, + "logps/chosen": -258.18218994140625, + "logps/rejected": -353.1686096191406, + "loss": 0.0008, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.262081623077393, + "rewards/margins": 9.589116096496582, + "rewards/rejected": -13.8511962890625, + "step": 1782 + }, + { + "epoch": 2.33, + "learning_rate": 4.4101174407110014e-05, + "logits/chosen": -1.7826972007751465, + "logits/rejected": -1.9188902378082275, + "logps/chosen": -206.72190856933594, + "logps/rejected": -328.41650390625, + "loss": 0.0468, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.051647186279297, + "rewards/margins": 8.272088050842285, + "rewards/rejected": -13.323734283447266, + "step": 1783 + }, + { + "epoch": 2.33, + "learning_rate": 4.409445248455399e-05, + "logits/chosen": -1.9735987186431885, + "logits/rejected": -2.0791704654693604, + "logps/chosen": -252.66329956054688, + "logps/rejected": -400.55242919921875, + "loss": 0.0439, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.900476455688477, + "rewards/margins": 9.93791675567627, + "rewards/rejected": -14.838394165039062, + "step": 1784 + }, + { + "epoch": 2.34, + "learning_rate": 4.408772724714509e-05, + "logits/chosen": -1.9090800285339355, + "logits/rejected": -1.964564561843872, + "logps/chosen": -216.352783203125, + "logps/rejected": -327.7049560546875, + "loss": 0.0466, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.770293712615967, + "rewards/margins": 8.165373802185059, + "rewards/rejected": -12.935667037963867, + "step": 1785 + }, + { + "epoch": 2.34, + "learning_rate": 4.408099869605085e-05, + "logits/chosen": -2.079120397567749, + "logits/rejected": -2.0540542602539062, + "logps/chosen": -324.5140075683594, + "logps/rejected": -396.20391845703125, + "loss": 0.0447, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.006938457489014, + "rewards/margins": 8.714357376098633, + "rewards/rejected": -14.721295356750488, + "step": 1786 + }, + { + "epoch": 2.34, + "learning_rate": 4.4074266832439355e-05, + "logits/chosen": -1.7295905351638794, + "logits/rejected": -1.7185676097869873, + "logps/chosen": -233.32333374023438, + "logps/rejected": -330.0339050292969, + "loss": 0.1225, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.257805347442627, + "rewards/margins": 8.65319538116455, + "rewards/rejected": -13.911002159118652, + "step": 1787 + }, + { + "epoch": 2.34, + "learning_rate": 4.406753165747927e-05, + "logits/chosen": -1.5425264835357666, + "logits/rejected": -1.580695629119873, + "logps/chosen": -213.93472290039062, + "logps/rejected": -363.4765930175781, + "loss": 0.0896, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.49269962310791, + "rewards/margins": 9.06192684173584, + "rewards/rejected": -14.554627418518066, + "step": 1788 + }, + { + "epoch": 2.34, + "learning_rate": 4.4060793172339855e-05, + "logits/chosen": -1.637101173400879, + "logits/rejected": -1.7731558084487915, + "logps/chosen": -305.5036315917969, + "logps/rejected": -429.70941162109375, + "loss": 0.0523, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.886563301086426, + "rewards/margins": 10.07012939453125, + "rewards/rejected": -15.95669174194336, + "step": 1789 + }, + { + "epoch": 2.34, + "learning_rate": 4.4054051378190915e-05, + "logits/chosen": -1.8037257194519043, + "logits/rejected": -1.937896728515625, + "logps/chosen": -242.22866821289062, + "logps/rejected": -403.0519104003906, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.685405254364014, + "rewards/margins": 10.16978645324707, + "rewards/rejected": -16.855192184448242, + "step": 1790 + }, + { + "epoch": 2.34, + "learning_rate": 4.404730627620285e-05, + "logits/chosen": -1.89559006690979, + "logits/rejected": -1.9932374954223633, + "logps/chosen": -253.1778564453125, + "logps/rejected": -392.6427917480469, + "loss": 0.0441, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.057366847991943, + "rewards/margins": 10.07391357421875, + "rewards/rejected": -16.13128089904785, + "step": 1791 + }, + { + "epoch": 2.35, + "learning_rate": 4.404055786754664e-05, + "logits/chosen": -2.106956958770752, + "logits/rejected": -2.170747995376587, + "logps/chosen": -253.71273803710938, + "logps/rejected": -377.1934509277344, + "loss": 0.045, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.209299087524414, + "rewards/margins": 8.318023681640625, + "rewards/rejected": -13.527321815490723, + "step": 1792 + }, + { + "epoch": 2.35, + "learning_rate": 4.403380615339382e-05, + "logits/chosen": -1.9337446689605713, + "logits/rejected": -1.9458836317062378, + "logps/chosen": -254.76597595214844, + "logps/rejected": -364.29302978515625, + "loss": 0.0012, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.060081958770752, + "rewards/margins": 11.015154838562012, + "rewards/rejected": -16.075237274169922, + "step": 1793 + }, + { + "epoch": 2.35, + "learning_rate": 4.402705113491651e-05, + "logits/chosen": -1.6449617147445679, + "logits/rejected": -1.5874007940292358, + "logps/chosen": -240.11338806152344, + "logps/rejected": -294.23553466796875, + "loss": 0.1034, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.366305351257324, + "rewards/margins": 6.1185784339904785, + "rewards/rejected": -11.484884262084961, + "step": 1794 + }, + { + "epoch": 2.35, + "learning_rate": 4.40202928132874e-05, + "logits/chosen": -1.9977463483810425, + "logits/rejected": -1.9549039602279663, + "logps/chosen": -257.6357116699219, + "logps/rejected": -340.7739562988281, + "loss": 0.0886, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.2195940017700195, + "rewards/margins": 8.912274360656738, + "rewards/rejected": -14.131868362426758, + "step": 1795 + }, + { + "epoch": 2.35, + "learning_rate": 4.4013531189679747e-05, + "logits/chosen": -1.8238599300384521, + "logits/rejected": -1.846430778503418, + "logps/chosen": -306.49176025390625, + "logps/rejected": -373.8945007324219, + "loss": 0.0444, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.066048622131348, + "rewards/margins": 9.407059669494629, + "rewards/rejected": -15.47310733795166, + "step": 1796 + }, + { + "epoch": 2.35, + "learning_rate": 4.40067662652674e-05, + "logits/chosen": -2.069310188293457, + "logits/rejected": -2.1010749340057373, + "logps/chosen": -248.91537475585938, + "logps/rejected": -339.01397705078125, + "loss": 0.1163, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.1216020584106445, + "rewards/margins": 9.208147048950195, + "rewards/rejected": -14.329750061035156, + "step": 1797 + }, + { + "epoch": 2.35, + "learning_rate": 4.399999804122477e-05, + "logits/chosen": -1.7326210737228394, + "logits/rejected": -1.844303011894226, + "logps/chosen": -211.9335174560547, + "logps/rejected": -317.3373107910156, + "loss": 0.0883, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.708733558654785, + "rewards/margins": 8.05502700805664, + "rewards/rejected": -13.763760566711426, + "step": 1798 + }, + { + "epoch": 2.35, + "learning_rate": 4.399322651872683e-05, + "logits/chosen": -1.9128247499465942, + "logits/rejected": -1.9641433954238892, + "logps/chosen": -238.70310974121094, + "logps/rejected": -336.80023193359375, + "loss": 0.0005, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.533979415893555, + "rewards/margins": 10.12472915649414, + "rewards/rejected": -14.658710479736328, + "step": 1799 + }, + { + "epoch": 2.36, + "learning_rate": 4.398645169894914e-05, + "logits/chosen": -1.973076343536377, + "logits/rejected": -2.023376226425171, + "logps/chosen": -255.61390686035156, + "logps/rejected": -356.170654296875, + "loss": 0.0529, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.700221538543701, + "rewards/margins": 7.843366622924805, + "rewards/rejected": -13.543588638305664, + "step": 1800 + }, + { + "epoch": 2.36, + "learning_rate": 4.3979673583067836e-05, + "logits/chosen": -1.8819239139556885, + "logits/rejected": -1.7926932573318481, + "logps/chosen": -210.09368896484375, + "logps/rejected": -265.912353515625, + "loss": 0.0504, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.568578720092773, + "rewards/margins": 7.491393089294434, + "rewards/rejected": -12.059972763061523, + "step": 1801 + }, + { + "epoch": 2.36, + "learning_rate": 4.3972892172259605e-05, + "logits/chosen": -1.8808557987213135, + "logits/rejected": -1.8767778873443604, + "logps/chosen": -215.65005493164062, + "logps/rejected": -278.6995849609375, + "loss": 0.0692, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.384488105773926, + "rewards/margins": 7.532333850860596, + "rewards/rejected": -11.916821479797363, + "step": 1802 + }, + { + "epoch": 2.36, + "learning_rate": 4.3966107467701735e-05, + "logits/chosen": -1.7873945236206055, + "logits/rejected": -1.922156572341919, + "logps/chosen": -216.10459899902344, + "logps/rejected": -328.5376281738281, + "loss": 0.0871, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.000890254974365, + "rewards/margins": 7.771942615509033, + "rewards/rejected": -11.772831916809082, + "step": 1803 + }, + { + "epoch": 2.36, + "learning_rate": 4.3959319470572064e-05, + "logits/chosen": -1.863304615020752, + "logits/rejected": -1.828956961631775, + "logps/chosen": -190.7486572265625, + "logps/rejected": -266.3214416503906, + "loss": 0.0772, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.7576091289520264, + "rewards/margins": 7.365079879760742, + "rewards/rejected": -11.122689247131348, + "step": 1804 + }, + { + "epoch": 2.36, + "learning_rate": 4.3952528182049e-05, + "logits/chosen": -2.0583343505859375, + "logits/rejected": -2.0666375160217285, + "logps/chosen": -203.62545776367188, + "logps/rejected": -301.2793884277344, + "loss": 0.0795, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.846843242645264, + "rewards/margins": 8.276459693908691, + "rewards/rejected": -13.123303413391113, + "step": 1805 + }, + { + "epoch": 2.36, + "learning_rate": 4.394573360331155e-05, + "logits/chosen": -2.133742332458496, + "logits/rejected": -2.118887424468994, + "logps/chosen": -209.5016632080078, + "logps/rejected": -298.78485107421875, + "loss": 0.1229, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.166501998901367, + "rewards/margins": 7.851699352264404, + "rewards/rejected": -13.01820182800293, + "step": 1806 + }, + { + "epoch": 2.36, + "learning_rate": 4.393893573553926e-05, + "logits/chosen": -1.800545334815979, + "logits/rejected": -1.7241178750991821, + "logps/chosen": -234.68531799316406, + "logps/rejected": -313.6758117675781, + "loss": 0.1319, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.9824187755584717, + "rewards/margins": 8.531006813049316, + "rewards/rejected": -12.51342487335205, + "step": 1807 + }, + { + "epoch": 2.37, + "learning_rate": 4.393213457991225e-05, + "logits/chosen": -1.7737388610839844, + "logits/rejected": -1.7564946413040161, + "logps/chosen": -242.74880981445312, + "logps/rejected": -319.61334228515625, + "loss": 0.0349, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.957961082458496, + "rewards/margins": 9.886978149414062, + "rewards/rejected": -14.844938278198242, + "step": 1808 + }, + { + "epoch": 2.37, + "learning_rate": 4.3925330137611236e-05, + "logits/chosen": -1.8337657451629639, + "logits/rejected": -1.7229583263397217, + "logps/chosen": -302.4295654296875, + "logps/rejected": -342.783935546875, + "loss": 0.005, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.371170520782471, + "rewards/margins": 9.544804573059082, + "rewards/rejected": -13.915973663330078, + "step": 1809 + }, + { + "epoch": 2.37, + "learning_rate": 4.391852240981749e-05, + "logits/chosen": -1.9108226299285889, + "logits/rejected": -2.050128936767578, + "logps/chosen": -248.89309692382812, + "logps/rejected": -343.4649963378906, + "loss": 0.0439, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.320683479309082, + "rewards/margins": 8.538372039794922, + "rewards/rejected": -12.85905647277832, + "step": 1810 + }, + { + "epoch": 2.37, + "learning_rate": 4.391171139771284e-05, + "logits/chosen": -2.0633928775787354, + "logits/rejected": -2.028679132461548, + "logps/chosen": -276.5841064453125, + "logps/rejected": -366.01812744140625, + "loss": 0.0029, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.392476558685303, + "rewards/margins": 8.86742877960205, + "rewards/rejected": -14.259904861450195, + "step": 1811 + }, + { + "epoch": 2.37, + "learning_rate": 4.390489710247971e-05, + "logits/chosen": -1.4684603214263916, + "logits/rejected": -1.5528146028518677, + "logps/chosen": -233.13583374023438, + "logps/rejected": -316.0859680175781, + "loss": 0.0902, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.470398426055908, + "rewards/margins": 7.3788886070251465, + "rewards/rejected": -11.849287033081055, + "step": 1812 + }, + { + "epoch": 2.37, + "learning_rate": 4.389807952530107e-05, + "logits/chosen": -1.895721435546875, + "logits/rejected": -1.9583930969238281, + "logps/chosen": -232.15414428710938, + "logps/rejected": -302.6424865722656, + "loss": 0.0896, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.65089750289917, + "rewards/margins": 8.486051559448242, + "rewards/rejected": -12.13694953918457, + "step": 1813 + }, + { + "epoch": 2.37, + "learning_rate": 4.389125866736049e-05, + "logits/chosen": -1.921229362487793, + "logits/rejected": -2.0489561557769775, + "logps/chosen": -269.0270080566406, + "logps/rejected": -355.5064392089844, + "loss": 0.0057, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.6717588901519775, + "rewards/margins": 8.599470138549805, + "rewards/rejected": -12.27122974395752, + "step": 1814 + }, + { + "epoch": 2.38, + "learning_rate": 4.3884434529842076e-05, + "logits/chosen": -1.8691997528076172, + "logits/rejected": -1.7107853889465332, + "logps/chosen": -236.80758666992188, + "logps/rejected": -315.18115234375, + "loss": 0.0149, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.2298078536987305, + "rewards/margins": 8.78706169128418, + "rewards/rejected": -13.016868591308594, + "step": 1815 + }, + { + "epoch": 2.38, + "learning_rate": 4.387760711393052e-05, + "logits/chosen": -2.011821746826172, + "logits/rejected": -2.2487993240356445, + "logps/chosen": -233.17649841308594, + "logps/rejected": -328.5198059082031, + "loss": 0.0873, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.249701976776123, + "rewards/margins": 7.541766166687012, + "rewards/rejected": -11.791467666625977, + "step": 1816 + }, + { + "epoch": 2.38, + "learning_rate": 4.3870776420811085e-05, + "logits/chosen": -2.020697832107544, + "logits/rejected": -2.0499825477600098, + "logps/chosen": -172.5773468017578, + "logps/rejected": -285.0962219238281, + "loss": 0.0137, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.833531379699707, + "rewards/margins": 9.668582916259766, + "rewards/rejected": -12.502114295959473, + "step": 1817 + }, + { + "epoch": 2.38, + "learning_rate": 4.386394245166959e-05, + "logits/chosen": -1.884272813796997, + "logits/rejected": -1.8307147026062012, + "logps/chosen": -201.13003540039062, + "logps/rejected": -290.59246826171875, + "loss": 0.0467, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.7439048290252686, + "rewards/margins": 8.601780891418457, + "rewards/rejected": -12.345686912536621, + "step": 1818 + }, + { + "epoch": 2.38, + "learning_rate": 4.385710520769245e-05, + "logits/chosen": -1.6563705205917358, + "logits/rejected": -1.6766928434371948, + "logps/chosen": -200.04208374023438, + "logps/rejected": -301.2552490234375, + "loss": 0.088, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.669889450073242, + "rewards/margins": 8.020367622375488, + "rewards/rejected": -12.690258026123047, + "step": 1819 + }, + { + "epoch": 2.38, + "learning_rate": 4.385026469006662e-05, + "logits/chosen": -1.7984764575958252, + "logits/rejected": -1.9804240465164185, + "logps/chosen": -227.8343048095703, + "logps/rejected": -377.7736511230469, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.096676826477051, + "rewards/margins": 10.553126335144043, + "rewards/rejected": -14.64980411529541, + "step": 1820 + }, + { + "epoch": 2.38, + "learning_rate": 4.3843420899979624e-05, + "logits/chosen": -1.9728909730911255, + "logits/rejected": -2.002688407897949, + "logps/chosen": -224.78646850585938, + "logps/rejected": -340.319091796875, + "loss": 0.045, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.982545852661133, + "rewards/margins": 8.316381454467773, + "rewards/rejected": -13.298928260803223, + "step": 1821 + }, + { + "epoch": 2.38, + "learning_rate": 4.3836573838619585e-05, + "logits/chosen": -2.0015525817871094, + "logits/rejected": -1.9768602848052979, + "logps/chosen": -243.19590759277344, + "logps/rejected": -351.15631103515625, + "loss": 0.0027, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.434335708618164, + "rewards/margins": 8.954057693481445, + "rewards/rejected": -14.38839340209961, + "step": 1822 + }, + { + "epoch": 2.39, + "learning_rate": 4.3829723507175154e-05, + "logits/chosen": -2.051961898803711, + "logits/rejected": -2.0231282711029053, + "logps/chosen": -239.29270935058594, + "logps/rejected": -332.600341796875, + "loss": 0.0012, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.319908618927002, + "rewards/margins": 9.003852844238281, + "rewards/rejected": -14.323761940002441, + "step": 1823 + }, + { + "epoch": 2.39, + "learning_rate": 4.382286990683558e-05, + "logits/chosen": -1.9241172075271606, + "logits/rejected": -1.9671472311019897, + "logps/chosen": -254.70701599121094, + "logps/rejected": -381.7937316894531, + "loss": 0.0452, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.170557498931885, + "rewards/margins": 8.524139404296875, + "rewards/rejected": -13.694698333740234, + "step": 1824 + }, + { + "epoch": 2.39, + "learning_rate": 4.381601303879066e-05, + "logits/chosen": -1.8312177658081055, + "logits/rejected": -1.9521081447601318, + "logps/chosen": -256.5509338378906, + "logps/rejected": -398.9842224121094, + "loss": 0.0874, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.040108680725098, + "rewards/margins": 8.639599800109863, + "rewards/rejected": -13.679709434509277, + "step": 1825 + }, + { + "epoch": 2.39, + "learning_rate": 4.3809152904230774e-05, + "logits/chosen": -1.9518373012542725, + "logits/rejected": -1.9363269805908203, + "logps/chosen": -231.14834594726562, + "logps/rejected": -355.7394104003906, + "loss": 0.0049, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.255949974060059, + "rewards/margins": 9.791579246520996, + "rewards/rejected": -16.047529220581055, + "step": 1826 + }, + { + "epoch": 2.39, + "learning_rate": 4.380228950434686e-05, + "logits/chosen": -2.1273605823516846, + "logits/rejected": -2.1018548011779785, + "logps/chosen": -242.73568725585938, + "logps/rejected": -335.8653564453125, + "loss": 0.12, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.828900337219238, + "rewards/margins": 6.7826690673828125, + "rewards/rejected": -13.61156940460205, + "step": 1827 + }, + { + "epoch": 2.39, + "learning_rate": 4.379542284033043e-05, + "logits/chosen": -2.1322808265686035, + "logits/rejected": -2.161734104156494, + "logps/chosen": -246.03045654296875, + "logps/rejected": -309.78912353515625, + "loss": 0.0295, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.126348495483398, + "rewards/margins": 9.210639953613281, + "rewards/rejected": -13.336989402770996, + "step": 1828 + }, + { + "epoch": 2.39, + "learning_rate": 4.3788552913373535e-05, + "logits/chosen": -1.9826735258102417, + "logits/rejected": -2.0525856018066406, + "logps/chosen": -229.6805419921875, + "logps/rejected": -300.07208251953125, + "loss": 0.0483, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.896882057189941, + "rewards/margins": 7.262111663818359, + "rewards/rejected": -12.1589937210083, + "step": 1829 + }, + { + "epoch": 2.39, + "learning_rate": 4.378167972466884e-05, + "logits/chosen": -2.012612819671631, + "logits/rejected": -2.0180180072784424, + "logps/chosen": -281.0361328125, + "logps/rejected": -357.9614562988281, + "loss": 0.0468, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.319505214691162, + "rewards/margins": 8.661141395568848, + "rewards/rejected": -13.980646133422852, + "step": 1830 + }, + { + "epoch": 2.4, + "learning_rate": 4.377480327540954e-05, + "logits/chosen": -2.094277858734131, + "logits/rejected": -2.123206615447998, + "logps/chosen": -244.86651611328125, + "logps/rejected": -343.1483154296875, + "loss": 0.0076, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.594821453094482, + "rewards/margins": 8.944721221923828, + "rewards/rejected": -14.539543151855469, + "step": 1831 + }, + { + "epoch": 2.4, + "learning_rate": 4.3767923566789394e-05, + "logits/chosen": -2.1138906478881836, + "logits/rejected": -2.116511583328247, + "logps/chosen": -260.0635681152344, + "logps/rejected": -358.54510498046875, + "loss": 0.0938, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.075669288635254, + "rewards/margins": 7.116991996765137, + "rewards/rejected": -12.19266128540039, + "step": 1832 + }, + { + "epoch": 2.4, + "learning_rate": 4.376104060000276e-05, + "logits/chosen": -1.975137710571289, + "logits/rejected": -1.9259370565414429, + "logps/chosen": -246.47845458984375, + "logps/rejected": -337.7164001464844, + "loss": 0.1525, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.622036457061768, + "rewards/margins": 7.510520935058594, + "rewards/rejected": -13.132558822631836, + "step": 1833 + }, + { + "epoch": 2.4, + "learning_rate": 4.375415437624454e-05, + "logits/chosen": -1.8363233804702759, + "logits/rejected": -1.923293113708496, + "logps/chosen": -194.30352783203125, + "logps/rejected": -285.00360107421875, + "loss": 0.1239, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.983138084411621, + "rewards/margins": 8.842789649963379, + "rewards/rejected": -12.825925827026367, + "step": 1834 + }, + { + "epoch": 2.4, + "learning_rate": 4.3747264896710185e-05, + "logits/chosen": -2.0146749019622803, + "logits/rejected": -2.1133487224578857, + "logps/chosen": -230.74244689941406, + "logps/rejected": -340.82720947265625, + "loss": 0.0173, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.959469318389893, + "rewards/margins": 9.281587600708008, + "rewards/rejected": -14.241056442260742, + "step": 1835 + }, + { + "epoch": 2.4, + "learning_rate": 4.374037216259574e-05, + "logits/chosen": -1.9939115047454834, + "logits/rejected": -2.0178937911987305, + "logps/chosen": -303.00439453125, + "logps/rejected": -386.6276550292969, + "loss": 0.0016, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.874594688415527, + "rewards/margins": 8.983979225158691, + "rewards/rejected": -14.858574867248535, + "step": 1836 + }, + { + "epoch": 2.4, + "learning_rate": 4.373347617509781e-05, + "logits/chosen": -1.9509072303771973, + "logits/rejected": -2.012117385864258, + "logps/chosen": -193.2029266357422, + "logps/rejected": -295.21923828125, + "loss": 0.0443, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.532169342041016, + "rewards/margins": 7.784310340881348, + "rewards/rejected": -12.316478729248047, + "step": 1837 + }, + { + "epoch": 2.41, + "learning_rate": 4.3726576935413554e-05, + "logits/chosen": -2.150205373764038, + "logits/rejected": -2.1467673778533936, + "logps/chosen": -237.6910400390625, + "logps/rejected": -307.8211975097656, + "loss": 0.0046, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.909461975097656, + "rewards/margins": 8.52404499053955, + "rewards/rejected": -13.433507919311523, + "step": 1838 + }, + { + "epoch": 2.41, + "learning_rate": 4.3719674444740697e-05, + "logits/chosen": -1.9739196300506592, + "logits/rejected": -1.9100971221923828, + "logps/chosen": -246.0147705078125, + "logps/rejected": -313.3359069824219, + "loss": 0.097, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.311476707458496, + "rewards/margins": 6.511356353759766, + "rewards/rejected": -12.822834014892578, + "step": 1839 + }, + { + "epoch": 2.41, + "learning_rate": 4.371276870427753e-05, + "logits/chosen": -1.8190784454345703, + "logits/rejected": -1.910037875175476, + "logps/chosen": -184.87718200683594, + "logps/rejected": -297.67181396484375, + "loss": 0.0461, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.034555912017822, + "rewards/margins": 7.538914680480957, + "rewards/rejected": -11.573470115661621, + "step": 1840 + }, + { + "epoch": 2.41, + "learning_rate": 4.370585971522291e-05, + "logits/chosen": -1.8734077215194702, + "logits/rejected": -1.891687035560608, + "logps/chosen": -230.254150390625, + "logps/rejected": -329.91033935546875, + "loss": 0.0512, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.072037220001221, + "rewards/margins": 8.25704574584961, + "rewards/rejected": -13.329083442687988, + "step": 1841 + }, + { + "epoch": 2.41, + "learning_rate": 4.3698947478776264e-05, + "logits/chosen": -1.8912385702133179, + "logits/rejected": -1.9524736404418945, + "logps/chosen": -197.22183227539062, + "logps/rejected": -304.8646240234375, + "loss": 0.0283, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.874556541442871, + "rewards/margins": 7.648215293884277, + "rewards/rejected": -11.522772789001465, + "step": 1842 + }, + { + "epoch": 2.41, + "learning_rate": 4.369203199613758e-05, + "logits/chosen": -2.2239837646484375, + "logits/rejected": -2.220667600631714, + "logps/chosen": -290.1179504394531, + "logps/rejected": -369.70819091796875, + "loss": 0.0013, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.312349319458008, + "rewards/margins": 8.92453670501709, + "rewards/rejected": -14.236885070800781, + "step": 1843 + }, + { + "epoch": 2.41, + "learning_rate": 4.368511326850739e-05, + "logits/chosen": -1.8601927757263184, + "logits/rejected": -1.8277055025100708, + "logps/chosen": -231.20823669433594, + "logps/rejected": -274.91693115234375, + "loss": 0.229, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.912055492401123, + "rewards/margins": 6.686162948608398, + "rewards/rejected": -11.598217964172363, + "step": 1844 + }, + { + "epoch": 2.41, + "learning_rate": 4.367819129708682e-05, + "logits/chosen": -2.0417680740356445, + "logits/rejected": -2.13813853263855, + "logps/chosen": -223.94223022460938, + "logps/rejected": -338.69415283203125, + "loss": 0.0757, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.077688217163086, + "rewards/margins": 7.460683345794678, + "rewards/rejected": -12.538371086120605, + "step": 1845 + }, + { + "epoch": 2.42, + "learning_rate": 4.367126608307753e-05, + "logits/chosen": -1.9008760452270508, + "logits/rejected": -2.027259111404419, + "logps/chosen": -222.43666076660156, + "logps/rejected": -346.3556823730469, + "loss": 0.0308, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.042816162109375, + "rewards/margins": 9.784735679626465, + "rewards/rejected": -13.82755184173584, + "step": 1846 + }, + { + "epoch": 2.42, + "learning_rate": 4.3664337627681774e-05, + "logits/chosen": -2.1032116413116455, + "logits/rejected": -2.1320929527282715, + "logps/chosen": -204.25173950195312, + "logps/rejected": -300.614501953125, + "loss": 0.1124, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.063549041748047, + "rewards/margins": 6.760867118835449, + "rewards/rejected": -11.824416160583496, + "step": 1847 + }, + { + "epoch": 2.42, + "learning_rate": 4.365740593210234e-05, + "logits/chosen": -2.150740146636963, + "logits/rejected": -2.2131547927856445, + "logps/chosen": -228.9392547607422, + "logps/rejected": -333.38836669921875, + "loss": 0.1739, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.91583251953125, + "rewards/margins": 8.279642105102539, + "rewards/rejected": -12.195475578308105, + "step": 1848 + }, + { + "epoch": 2.42, + "learning_rate": 4.36504709975426e-05, + "logits/chosen": -1.8772621154785156, + "logits/rejected": -1.9525573253631592, + "logps/chosen": -310.34375, + "logps/rejected": -355.4771728515625, + "loss": 0.0907, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.125977993011475, + "rewards/margins": 6.897149085998535, + "rewards/rejected": -13.023127555847168, + "step": 1849 + }, + { + "epoch": 2.42, + "learning_rate": 4.364353282520648e-05, + "logits/chosen": -2.17010235786438, + "logits/rejected": -2.115602493286133, + "logps/chosen": -233.1490936279297, + "logps/rejected": -316.7723083496094, + "loss": 0.0603, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.2152438163757324, + "rewards/margins": 10.278971672058105, + "rewards/rejected": -13.49421501159668, + "step": 1850 + }, + { + "epoch": 2.42, + "learning_rate": 4.363659141629845e-05, + "logits/chosen": -2.1375784873962402, + "logits/rejected": -2.1189472675323486, + "logps/chosen": -221.9202117919922, + "logps/rejected": -287.76275634765625, + "loss": 0.0914, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.150267601013184, + "rewards/margins": 6.575910568237305, + "rewards/rejected": -11.726178169250488, + "step": 1851 + }, + { + "epoch": 2.42, + "learning_rate": 4.362964677202357e-05, + "logits/chosen": -2.344611167907715, + "logits/rejected": -2.2970376014709473, + "logps/chosen": -251.08851623535156, + "logps/rejected": -334.6526794433594, + "loss": 0.0102, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.216737270355225, + "rewards/margins": 7.752537250518799, + "rewards/rejected": -12.969274520874023, + "step": 1852 + }, + { + "epoch": 2.42, + "learning_rate": 4.3622698893587456e-05, + "logits/chosen": -2.0821337699890137, + "logits/rejected": -2.083778142929077, + "logps/chosen": -243.42315673828125, + "logps/rejected": -315.04339599609375, + "loss": 0.0886, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.053452491760254, + "rewards/margins": 8.2511625289917, + "rewards/rejected": -12.304615020751953, + "step": 1853 + }, + { + "epoch": 2.43, + "learning_rate": 4.361574778219628e-05, + "logits/chosen": -2.0339465141296387, + "logits/rejected": -2.088513135910034, + "logps/chosen": -206.04376220703125, + "logps/rejected": -341.8985290527344, + "loss": 0.0449, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.140441417694092, + "rewards/margins": 9.311591148376465, + "rewards/rejected": -14.452032089233398, + "step": 1854 + }, + { + "epoch": 2.43, + "learning_rate": 4.360879343905676e-05, + "logits/chosen": -2.0117568969726562, + "logits/rejected": -1.986091136932373, + "logps/chosen": -258.1641540527344, + "logps/rejected": -312.88116455078125, + "loss": 0.101, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.49587345123291, + "rewards/margins": 8.615702629089355, + "rewards/rejected": -13.111577033996582, + "step": 1855 + }, + { + "epoch": 2.43, + "learning_rate": 4.360183586537621e-05, + "logits/chosen": -1.5971702337265015, + "logits/rejected": -1.8032184839248657, + "logps/chosen": -194.74679565429688, + "logps/rejected": -318.2879333496094, + "loss": 0.0887, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.8355278968811035, + "rewards/margins": 7.676074504852295, + "rewards/rejected": -13.511604309082031, + "step": 1856 + }, + { + "epoch": 2.43, + "learning_rate": 4.359487506236246e-05, + "logits/chosen": -2.150571346282959, + "logits/rejected": -2.0732815265655518, + "logps/chosen": -219.47314453125, + "logps/rejected": -299.68218994140625, + "loss": 0.0283, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.4884231090545654, + "rewards/margins": 9.733719825744629, + "rewards/rejected": -13.22214126586914, + "step": 1857 + }, + { + "epoch": 2.43, + "learning_rate": 4.358791103122395e-05, + "logits/chosen": -1.8017619848251343, + "logits/rejected": -1.721257209777832, + "logps/chosen": -204.084716796875, + "logps/rejected": -274.8855895996094, + "loss": 0.132, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.801064968109131, + "rewards/margins": 6.824024677276611, + "rewards/rejected": -11.625089645385742, + "step": 1858 + }, + { + "epoch": 2.43, + "learning_rate": 4.358094377316965e-05, + "logits/chosen": -1.9682602882385254, + "logits/rejected": -1.961120843887329, + "logps/chosen": -206.16127014160156, + "logps/rejected": -274.9136657714844, + "loss": 0.0888, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.209860801696777, + "rewards/margins": 5.908940315246582, + "rewards/rejected": -11.118800163269043, + "step": 1859 + }, + { + "epoch": 2.43, + "learning_rate": 4.3573973289409094e-05, + "logits/chosen": -1.7236840724945068, + "logits/rejected": -1.6636816263198853, + "logps/chosen": -251.0654754638672, + "logps/rejected": -302.43121337890625, + "loss": 0.2175, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.596639633178711, + "rewards/margins": 7.8836469650268555, + "rewards/rejected": -12.480286598205566, + "step": 1860 + }, + { + "epoch": 2.44, + "learning_rate": 4.356699958115238e-05, + "logits/chosen": -1.7644553184509277, + "logits/rejected": -1.7401866912841797, + "logps/chosen": -251.42462158203125, + "logps/rejected": -335.5991516113281, + "loss": 0.1323, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.755103588104248, + "rewards/margins": 6.958304405212402, + "rewards/rejected": -11.713406562805176, + "step": 1861 + }, + { + "epoch": 2.44, + "learning_rate": 4.356002264961016e-05, + "logits/chosen": -2.147675037384033, + "logits/rejected": -2.179133176803589, + "logps/chosen": -230.9063262939453, + "logps/rejected": -317.8185119628906, + "loss": 0.0708, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.664783477783203, + "rewards/margins": 9.451215744018555, + "rewards/rejected": -13.115998268127441, + "step": 1862 + }, + { + "epoch": 2.44, + "learning_rate": 4.3553042495993643e-05, + "logits/chosen": -2.1215152740478516, + "logits/rejected": -2.17391300201416, + "logps/chosen": -287.8443908691406, + "logps/rejected": -335.31256103515625, + "loss": 0.1452, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.5345940589904785, + "rewards/margins": 5.930063247680664, + "rewards/rejected": -10.4646577835083, + "step": 1863 + }, + { + "epoch": 2.44, + "learning_rate": 4.354605912151463e-05, + "logits/chosen": -2.1102986335754395, + "logits/rejected": -2.286177635192871, + "logps/chosen": -181.72535705566406, + "logps/rejected": -275.7710876464844, + "loss": 0.1328, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.561611175537109, + "rewards/margins": 7.474075794219971, + "rewards/rejected": -12.035687446594238, + "step": 1864 + }, + { + "epoch": 2.44, + "learning_rate": 4.353907252738543e-05, + "logits/chosen": -1.6983438730239868, + "logits/rejected": -1.6498935222625732, + "logps/chosen": -227.6995849609375, + "logps/rejected": -276.0758972167969, + "loss": 0.0712, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.649226188659668, + "rewards/margins": 6.456134796142578, + "rewards/rejected": -11.105360984802246, + "step": 1865 + }, + { + "epoch": 2.44, + "learning_rate": 4.353208271481894e-05, + "logits/chosen": -2.0330216884613037, + "logits/rejected": -2.0915470123291016, + "logps/chosen": -237.63294982910156, + "logps/rejected": -305.3509826660156, + "loss": 0.0881, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.406057357788086, + "rewards/margins": 6.762484550476074, + "rewards/rejected": -11.168540954589844, + "step": 1866 + }, + { + "epoch": 2.44, + "learning_rate": 4.352508968502862e-05, + "logits/chosen": -1.7718350887298584, + "logits/rejected": -1.7792692184448242, + "logps/chosen": -220.42466735839844, + "logps/rejected": -275.79290771484375, + "loss": 0.1527, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.707272529602051, + "rewards/margins": 5.57955265045166, + "rewards/rejected": -10.286825180053711, + "step": 1867 + }, + { + "epoch": 2.44, + "learning_rate": 4.351809343922848e-05, + "logits/chosen": -2.1098427772521973, + "logits/rejected": -2.210489273071289, + "logps/chosen": -278.0294189453125, + "logps/rejected": -352.3443908691406, + "loss": 0.0948, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.806293487548828, + "rewards/margins": 7.794907093048096, + "rewards/rejected": -12.60119915008545, + "step": 1868 + }, + { + "epoch": 2.45, + "learning_rate": 4.351109397863309e-05, + "logits/chosen": -2.126844644546509, + "logits/rejected": -2.1765525341033936, + "logps/chosen": -266.9029541015625, + "logps/rejected": -331.0353088378906, + "loss": 0.0945, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.96994686126709, + "rewards/margins": 7.34199857711792, + "rewards/rejected": -12.311944961547852, + "step": 1869 + }, + { + "epoch": 2.45, + "learning_rate": 4.350409130445756e-05, + "logits/chosen": -2.012559413909912, + "logits/rejected": -2.187136650085449, + "logps/chosen": -217.30624389648438, + "logps/rejected": -329.05841064453125, + "loss": 0.0053, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.945742130279541, + "rewards/margins": 8.809176445007324, + "rewards/rejected": -13.754919052124023, + "step": 1870 + }, + { + "epoch": 2.45, + "learning_rate": 4.34970854179176e-05, + "logits/chosen": -2.165018081665039, + "logits/rejected": -2.2561163902282715, + "logps/chosen": -266.87152099609375, + "logps/rejected": -364.1900329589844, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.06184196472168, + "rewards/margins": 9.541501998901367, + "rewards/rejected": -14.603343963623047, + "step": 1871 + }, + { + "epoch": 2.45, + "learning_rate": 4.349007632022943e-05, + "logits/chosen": -1.7294069528579712, + "logits/rejected": -1.7365280389785767, + "logps/chosen": -224.39157104492188, + "logps/rejected": -304.15411376953125, + "loss": 0.0886, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.065281391143799, + "rewards/margins": 6.858569622039795, + "rewards/rejected": -12.923851013183594, + "step": 1872 + }, + { + "epoch": 2.45, + "learning_rate": 4.348306401260987e-05, + "logits/chosen": -2.3519177436828613, + "logits/rejected": -2.365565538406372, + "logps/chosen": -260.7571716308594, + "logps/rejected": -350.22882080078125, + "loss": 0.0909, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.139139652252197, + "rewards/margins": 8.925704002380371, + "rewards/rejected": -14.064842224121094, + "step": 1873 + }, + { + "epoch": 2.45, + "learning_rate": 4.3476048496276265e-05, + "logits/chosen": -2.2548577785491943, + "logits/rejected": -2.293238878250122, + "logps/chosen": -257.31414794921875, + "logps/rejected": -358.8238525390625, + "loss": 0.0174, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.687314033508301, + "rewards/margins": 8.911933898925781, + "rewards/rejected": -14.599246978759766, + "step": 1874 + }, + { + "epoch": 2.45, + "learning_rate": 4.3469029772446525e-05, + "logits/chosen": -1.6724498271942139, + "logits/rejected": -1.61297607421875, + "logps/chosen": -295.10333251953125, + "logps/rejected": -342.6184997558594, + "loss": 0.0941, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.655141830444336, + "rewards/margins": 7.57160758972168, + "rewards/rejected": -12.226749420166016, + "step": 1875 + }, + { + "epoch": 2.46, + "learning_rate": 4.3462007842339135e-05, + "logits/chosen": -1.7761496305465698, + "logits/rejected": -2.0170705318450928, + "logps/chosen": -142.1776123046875, + "logps/rejected": -264.1295166015625, + "loss": 0.0008, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.0851376056671143, + "rewards/margins": 8.722681999206543, + "rewards/rejected": -11.807819366455078, + "step": 1876 + }, + { + "epoch": 2.46, + "learning_rate": 4.345498270717311e-05, + "logits/chosen": -1.8887841701507568, + "logits/rejected": -1.9506733417510986, + "logps/chosen": -242.97447204589844, + "logps/rejected": -345.9141540527344, + "loss": 0.1465, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.468897342681885, + "rewards/margins": 9.694093704223633, + "rewards/rejected": -14.162991523742676, + "step": 1877 + }, + { + "epoch": 2.46, + "learning_rate": 4.3447954368168036e-05, + "logits/chosen": -1.8739227056503296, + "logits/rejected": -1.8192765712738037, + "logps/chosen": -264.2427062988281, + "logps/rejected": -359.7630310058594, + "loss": 0.0272, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.784821033477783, + "rewards/margins": 7.629404544830322, + "rewards/rejected": -13.414225578308105, + "step": 1878 + }, + { + "epoch": 2.46, + "learning_rate": 4.344092282654406e-05, + "logits/chosen": -1.9280517101287842, + "logits/rejected": -1.841347575187683, + "logps/chosen": -246.22378540039062, + "logps/rejected": -298.2712097167969, + "loss": 0.0817, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.732930660247803, + "rewards/margins": 7.216610908508301, + "rewards/rejected": -11.949542045593262, + "step": 1879 + }, + { + "epoch": 2.46, + "learning_rate": 4.343388808352187e-05, + "logits/chosen": -2.0055572986602783, + "logits/rejected": -2.025848865509033, + "logps/chosen": -204.11595153808594, + "logps/rejected": -263.9717102050781, + "loss": 0.1232, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.866933822631836, + "rewards/margins": 6.790870189666748, + "rewards/rejected": -11.657803535461426, + "step": 1880 + }, + { + "epoch": 2.46, + "learning_rate": 4.342685014032273e-05, + "logits/chosen": -2.0591530799865723, + "logits/rejected": -2.0639140605926514, + "logps/chosen": -214.9796905517578, + "logps/rejected": -300.53106689453125, + "loss": 0.0562, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.563952445983887, + "rewards/margins": 7.759393692016602, + "rewards/rejected": -12.323345184326172, + "step": 1881 + }, + { + "epoch": 2.46, + "learning_rate": 4.341980899816843e-05, + "logits/chosen": -1.9732942581176758, + "logits/rejected": -2.136146306991577, + "logps/chosen": -234.76837158203125, + "logps/rejected": -342.0783996582031, + "loss": 0.0012, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.5489654541015625, + "rewards/margins": 8.756852149963379, + "rewards/rejected": -13.305819511413574, + "step": 1882 + }, + { + "epoch": 2.46, + "learning_rate": 4.341276465828136e-05, + "logits/chosen": -1.8425331115722656, + "logits/rejected": -2.025358200073242, + "logps/chosen": -227.8318634033203, + "logps/rejected": -352.81524658203125, + "loss": 0.0104, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.6305718421936035, + "rewards/margins": 7.565232753753662, + "rewards/rejected": -13.19580364227295, + "step": 1883 + }, + { + "epoch": 2.47, + "learning_rate": 4.3405717121884406e-05, + "logits/chosen": -2.106154441833496, + "logits/rejected": -2.1269004344940186, + "logps/chosen": -205.65060424804688, + "logps/rejected": -257.221923828125, + "loss": 0.1787, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.801647663116455, + "rewards/margins": 5.70452356338501, + "rewards/rejected": -9.506172180175781, + "step": 1884 + }, + { + "epoch": 2.47, + "learning_rate": 4.339866639020106e-05, + "logits/chosen": -1.9552853107452393, + "logits/rejected": -2.0579018592834473, + "logps/chosen": -244.7666015625, + "logps/rejected": -307.3655700683594, + "loss": 0.014, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.008973121643066, + "rewards/margins": 7.487423896789551, + "rewards/rejected": -11.496397972106934, + "step": 1885 + }, + { + "epoch": 2.47, + "learning_rate": 4.3391612464455344e-05, + "logits/chosen": -1.4287627935409546, + "logits/rejected": -1.5520751476287842, + "logps/chosen": -212.376708984375, + "logps/rejected": -295.0533752441406, + "loss": 0.0884, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.4821248054504395, + "rewards/margins": 7.69401741027832, + "rewards/rejected": -11.176141738891602, + "step": 1886 + }, + { + "epoch": 2.47, + "learning_rate": 4.3384555345871845e-05, + "logits/chosen": -2.0539603233337402, + "logits/rejected": -2.0998637676239014, + "logps/chosen": -225.72454833984375, + "logps/rejected": -307.70245361328125, + "loss": 0.0874, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.347853183746338, + "rewards/margins": 7.575061798095703, + "rewards/rejected": -11.922914505004883, + "step": 1887 + }, + { + "epoch": 2.47, + "learning_rate": 4.337749503567569e-05, + "logits/chosen": -2.240464925765991, + "logits/rejected": -2.2448160648345947, + "logps/chosen": -222.15724182128906, + "logps/rejected": -313.15093994140625, + "loss": 0.0471, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.5812578201293945, + "rewards/margins": 8.621621131896973, + "rewards/rejected": -12.202878952026367, + "step": 1888 + }, + { + "epoch": 2.47, + "learning_rate": 4.337043153509258e-05, + "logits/chosen": -1.7158111333847046, + "logits/rejected": -1.7638497352600098, + "logps/chosen": -295.1845397949219, + "logps/rejected": -395.8709411621094, + "loss": 0.0444, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.137034893035889, + "rewards/margins": 8.628808975219727, + "rewards/rejected": -12.765844345092773, + "step": 1889 + }, + { + "epoch": 2.47, + "learning_rate": 4.3363364845348755e-05, + "logits/chosen": -1.9096884727478027, + "logits/rejected": -1.9671432971954346, + "logps/chosen": -214.6175537109375, + "logps/rejected": -330.2419738769531, + "loss": 0.0323, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.5655999183654785, + "rewards/margins": 8.131664276123047, + "rewards/rejected": -12.697264671325684, + "step": 1890 + }, + { + "epoch": 2.47, + "learning_rate": 4.335629496767101e-05, + "logits/chosen": -2.1084601879119873, + "logits/rejected": -2.018869400024414, + "logps/chosen": -228.5397186279297, + "logps/rejected": -305.6234130859375, + "loss": 0.0035, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.8502936363220215, + "rewards/margins": 8.437298774719238, + "rewards/rejected": -12.287591934204102, + "step": 1891 + }, + { + "epoch": 2.48, + "learning_rate": 4.334922190328671e-05, + "logits/chosen": -2.083664894104004, + "logits/rejected": -2.084148406982422, + "logps/chosen": -226.5367431640625, + "logps/rejected": -316.456298828125, + "loss": 0.0627, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.226537227630615, + "rewards/margins": 6.392550468444824, + "rewards/rejected": -11.619088172912598, + "step": 1892 + }, + { + "epoch": 2.48, + "learning_rate": 4.3342145653423736e-05, + "logits/chosen": -2.1525473594665527, + "logits/rejected": -2.1263246536254883, + "logps/chosen": -282.1151123046875, + "logps/rejected": -358.0589599609375, + "loss": 0.0451, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.356586456298828, + "rewards/margins": 8.507760047912598, + "rewards/rejected": -11.86434555053711, + "step": 1893 + }, + { + "epoch": 2.48, + "learning_rate": 4.3335066219310564e-05, + "logits/chosen": -2.1275317668914795, + "logits/rejected": -2.2363266944885254, + "logps/chosen": -217.88424682617188, + "logps/rejected": -314.2412109375, + "loss": 0.012, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.613062858581543, + "rewards/margins": 7.576802730560303, + "rewards/rejected": -12.18986701965332, + "step": 1894 + }, + { + "epoch": 2.48, + "learning_rate": 4.3327983602176196e-05, + "logits/chosen": -2.2973012924194336, + "logits/rejected": -2.352452516555786, + "logps/chosen": -258.4927978515625, + "logps/rejected": -352.5173034667969, + "loss": 0.0449, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.170706748962402, + "rewards/margins": 8.284988403320312, + "rewards/rejected": -12.455696105957031, + "step": 1895 + }, + { + "epoch": 2.48, + "learning_rate": 4.3320897803250206e-05, + "logits/chosen": -2.1969974040985107, + "logits/rejected": -2.2443318367004395, + "logps/chosen": -283.5622863769531, + "logps/rejected": -373.3434143066406, + "loss": 0.0531, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.566214561462402, + "rewards/margins": 8.141165733337402, + "rewards/rejected": -12.707380294799805, + "step": 1896 + }, + { + "epoch": 2.48, + "learning_rate": 4.331380882376269e-05, + "logits/chosen": -1.893761157989502, + "logits/rejected": -2.058621644973755, + "logps/chosen": -239.73043823242188, + "logps/rejected": -331.2525939941406, + "loss": 0.0058, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.567351341247559, + "rewards/margins": 7.453845024108887, + "rewards/rejected": -12.021196365356445, + "step": 1897 + }, + { + "epoch": 2.48, + "learning_rate": 4.3306716664944344e-05, + "logits/chosen": -2.0633487701416016, + "logits/rejected": -2.200991153717041, + "logps/chosen": -208.24517822265625, + "logps/rejected": -329.59649658203125, + "loss": 0.1383, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.751850128173828, + "rewards/margins": 6.950207233428955, + "rewards/rejected": -11.702056884765625, + "step": 1898 + }, + { + "epoch": 2.49, + "learning_rate": 4.329962132802636e-05, + "logits/chosen": -2.092829704284668, + "logits/rejected": -2.0219295024871826, + "logps/chosen": -235.95083618164062, + "logps/rejected": -304.3815002441406, + "loss": 0.0024, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.290584564208984, + "rewards/margins": 8.294286727905273, + "rewards/rejected": -12.584871292114258, + "step": 1899 + }, + { + "epoch": 2.49, + "learning_rate": 4.329252281424052e-05, + "logits/chosen": -2.03996205329895, + "logits/rejected": -2.141855239868164, + "logps/chosen": -238.59591674804688, + "logps/rejected": -335.9581298828125, + "loss": 0.0167, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.734767913818359, + "rewards/margins": 8.400701522827148, + "rewards/rejected": -13.135469436645508, + "step": 1900 + }, + { + "epoch": 2.49, + "learning_rate": 4.328542112481915e-05, + "logits/chosen": -2.228651523590088, + "logits/rejected": -2.1372406482696533, + "logps/chosen": -215.5223388671875, + "logps/rejected": -306.9183044433594, + "loss": 0.0005, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.169659614562988, + "rewards/margins": 10.764291763305664, + "rewards/rejected": -14.933952331542969, + "step": 1901 + }, + { + "epoch": 2.49, + "learning_rate": 4.327831626099512e-05, + "logits/chosen": -2.0305263996124268, + "logits/rejected": -2.1352574825286865, + "logps/chosen": -200.35475158691406, + "logps/rejected": -329.01995849609375, + "loss": 0.0539, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.779258728027344, + "rewards/margins": 7.56350040435791, + "rewards/rejected": -12.342758178710938, + "step": 1902 + }, + { + "epoch": 2.49, + "learning_rate": 4.3271208224001856e-05, + "logits/chosen": -2.0742321014404297, + "logits/rejected": -2.106760263442993, + "logps/chosen": -312.0980529785156, + "logps/rejected": -396.3582458496094, + "loss": 0.0448, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.070708274841309, + "rewards/margins": 7.27605676651001, + "rewards/rejected": -13.346765518188477, + "step": 1903 + }, + { + "epoch": 2.49, + "learning_rate": 4.3264097015073334e-05, + "logits/chosen": -1.9420523643493652, + "logits/rejected": -2.0621085166931152, + "logps/chosen": -238.076904296875, + "logps/rejected": -330.4910888671875, + "loss": 0.0938, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.611868381500244, + "rewards/margins": 6.453281879425049, + "rewards/rejected": -12.06515121459961, + "step": 1904 + }, + { + "epoch": 2.49, + "learning_rate": 4.325698263544409e-05, + "logits/chosen": -1.7279473543167114, + "logits/rejected": -1.8486162424087524, + "logps/chosen": -156.3711700439453, + "logps/rejected": -285.4295349121094, + "loss": 0.09, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.322530746459961, + "rewards/margins": 9.202067375183105, + "rewards/rejected": -12.52459716796875, + "step": 1905 + }, + { + "epoch": 2.49, + "learning_rate": 4.3249865086349186e-05, + "logits/chosen": -1.7305097579956055, + "logits/rejected": -1.7878034114837646, + "logps/chosen": -225.58343505859375, + "logps/rejected": -343.70068359375, + "loss": 0.0461, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.001452922821045, + "rewards/margins": 8.839422225952148, + "rewards/rejected": -12.840875625610352, + "step": 1906 + }, + { + "epoch": 2.5, + "learning_rate": 4.324274436902425e-05, + "logits/chosen": -2.0604379177093506, + "logits/rejected": -2.040114641189575, + "logps/chosen": -216.70742797851562, + "logps/rejected": -292.2000427246094, + "loss": 0.0883, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.9439444541931152, + "rewards/margins": 7.755221843719482, + "rewards/rejected": -11.699165344238281, + "step": 1907 + }, + { + "epoch": 2.5, + "learning_rate": 4.323562048470547e-05, + "logits/chosen": -1.9717481136322021, + "logits/rejected": -1.9101167917251587, + "logps/chosen": -255.74168395996094, + "logps/rejected": -342.4549560546875, + "loss": 0.0031, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.992917537689209, + "rewards/margins": 9.150410652160645, + "rewards/rejected": -13.143327713012695, + "step": 1908 + }, + { + "epoch": 2.5, + "learning_rate": 4.322849343462957e-05, + "logits/chosen": -2.0949337482452393, + "logits/rejected": -2.132563591003418, + "logps/chosen": -266.8226013183594, + "logps/rejected": -333.36676025390625, + "loss": 0.0466, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.620141983032227, + "rewards/margins": 7.952317237854004, + "rewards/rejected": -13.572460174560547, + "step": 1909 + }, + { + "epoch": 2.5, + "learning_rate": 4.3221363220033814e-05, + "logits/chosen": -1.9663276672363281, + "logits/rejected": -1.9690686464309692, + "logps/chosen": -256.2469482421875, + "logps/rejected": -334.94122314453125, + "loss": 0.1329, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.505515098571777, + "rewards/margins": 8.074156761169434, + "rewards/rejected": -12.579671859741211, + "step": 1910 + }, + { + "epoch": 2.5, + "learning_rate": 4.321422984215605e-05, + "logits/chosen": -1.745408058166504, + "logits/rejected": -1.7534682750701904, + "logps/chosen": -214.59886169433594, + "logps/rejected": -293.2696533203125, + "loss": 0.0908, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.432751178741455, + "rewards/margins": 6.949195384979248, + "rewards/rejected": -11.381946563720703, + "step": 1911 + }, + { + "epoch": 2.5, + "learning_rate": 4.320709330223464e-05, + "logits/chosen": -1.8495633602142334, + "logits/rejected": -1.7307114601135254, + "logps/chosen": -282.5513610839844, + "logps/rejected": -316.41162109375, + "loss": 0.1304, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.877445697784424, + "rewards/margins": 10.04818344116211, + "rewards/rejected": -13.925628662109375, + "step": 1912 + }, + { + "epoch": 2.5, + "learning_rate": 4.319995360150851e-05, + "logits/chosen": -2.143476963043213, + "logits/rejected": -2.24068021774292, + "logps/chosen": -257.17938232421875, + "logps/rejected": -412.9932556152344, + "loss": 0.0023, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.908987522125244, + "rewards/margins": 9.167608261108398, + "rewards/rejected": -15.076594352722168, + "step": 1913 + }, + { + "epoch": 2.5, + "learning_rate": 4.319281074121714e-05, + "logits/chosen": -2.158090829849243, + "logits/rejected": -2.0695207118988037, + "logps/chosen": -242.647705078125, + "logps/rejected": -316.803955078125, + "loss": 0.0452, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.062627792358398, + "rewards/margins": 8.320158004760742, + "rewards/rejected": -13.38278579711914, + "step": 1914 + }, + { + "epoch": 2.51, + "learning_rate": 4.318566472260054e-05, + "logits/chosen": -1.846653699874878, + "logits/rejected": -1.94084632396698, + "logps/chosen": -207.33084106445312, + "logps/rejected": -337.37139892578125, + "loss": 0.0906, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.298641681671143, + "rewards/margins": 7.939152240753174, + "rewards/rejected": -12.237793922424316, + "step": 1915 + }, + { + "epoch": 2.51, + "learning_rate": 4.317851554689928e-05, + "logits/chosen": -1.935530185699463, + "logits/rejected": -1.9438458681106567, + "logps/chosen": -217.8266143798828, + "logps/rejected": -332.8846130371094, + "loss": 0.0205, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.889610767364502, + "rewards/margins": 10.1079683303833, + "rewards/rejected": -14.997577667236328, + "step": 1916 + }, + { + "epoch": 2.51, + "learning_rate": 4.3171363215354485e-05, + "logits/chosen": -2.0048739910125732, + "logits/rejected": -2.141301155090332, + "logps/chosen": -208.8019256591797, + "logps/rejected": -275.9781799316406, + "loss": 0.0511, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.808550834655762, + "rewards/margins": 6.669939041137695, + "rewards/rejected": -12.478490829467773, + "step": 1917 + }, + { + "epoch": 2.51, + "learning_rate": 4.316420772920782e-05, + "logits/chosen": -1.9847973585128784, + "logits/rejected": -2.108638286590576, + "logps/chosen": -181.55844116210938, + "logps/rejected": -295.7936706542969, + "loss": 0.0444, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.9625251293182373, + "rewards/margins": 8.532015800476074, + "rewards/rejected": -12.494542121887207, + "step": 1918 + }, + { + "epoch": 2.51, + "learning_rate": 4.3157049089701495e-05, + "logits/chosen": -2.231801748275757, + "logits/rejected": -2.2912943363189697, + "logps/chosen": -291.9457702636719, + "logps/rejected": -368.4671325683594, + "loss": 0.0041, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.943848133087158, + "rewards/margins": 7.637473106384277, + "rewards/rejected": -13.581321716308594, + "step": 1919 + }, + { + "epoch": 2.51, + "learning_rate": 4.3149887298078276e-05, + "logits/chosen": -1.9980155229568481, + "logits/rejected": -1.9702084064483643, + "logps/chosen": -221.4971160888672, + "logps/rejected": -364.04388427734375, + "loss": 0.047, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.686988830566406, + "rewards/margins": 6.865217685699463, + "rewards/rejected": -12.552206993103027, + "step": 1920 + }, + { + "epoch": 2.51, + "learning_rate": 4.3142722355581466e-05, + "logits/chosen": -1.8039441108703613, + "logits/rejected": -1.716475009918213, + "logps/chosen": -250.4287872314453, + "logps/rejected": -317.84368896484375, + "loss": 0.1178, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.042150497436523, + "rewards/margins": 7.001775741577148, + "rewards/rejected": -12.043927192687988, + "step": 1921 + }, + { + "epoch": 2.52, + "learning_rate": 4.313555426345492e-05, + "logits/chosen": -1.9541935920715332, + "logits/rejected": -2.0513226985931396, + "logps/chosen": -260.1856689453125, + "logps/rejected": -363.9661560058594, + "loss": 0.0913, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.970685958862305, + "rewards/margins": 8.336792945861816, + "rewards/rejected": -13.307477951049805, + "step": 1922 + }, + { + "epoch": 2.52, + "learning_rate": 4.3128383022943045e-05, + "logits/chosen": -2.263624906539917, + "logits/rejected": -2.3434362411499023, + "logps/chosen": -217.94793701171875, + "logps/rejected": -312.8447265625, + "loss": 0.0872, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.745199680328369, + "rewards/margins": 7.728671550750732, + "rewards/rejected": -11.473870277404785, + "step": 1923 + }, + { + "epoch": 2.52, + "learning_rate": 4.3121208635290775e-05, + "logits/chosen": -2.2434561252593994, + "logits/rejected": -2.2021243572235107, + "logps/chosen": -271.01287841796875, + "logps/rejected": -342.8251037597656, + "loss": 0.0479, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.077291488647461, + "rewards/margins": 7.209915637969971, + "rewards/rejected": -12.28720760345459, + "step": 1924 + }, + { + "epoch": 2.52, + "learning_rate": 4.3114031101743625e-05, + "logits/chosen": -2.0379481315612793, + "logits/rejected": -2.0702476501464844, + "logps/chosen": -255.35540771484375, + "logps/rejected": -362.4584045410156, + "loss": 0.0079, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.8948774337768555, + "rewards/margins": 7.793493747711182, + "rewards/rejected": -13.688371658325195, + "step": 1925 + }, + { + "epoch": 2.52, + "learning_rate": 4.3106850423547626e-05, + "logits/chosen": -2.2296431064605713, + "logits/rejected": -2.2409510612487793, + "logps/chosen": -271.7919921875, + "logps/rejected": -329.9810791015625, + "loss": 0.0459, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.642647743225098, + "rewards/margins": 8.106131553649902, + "rewards/rejected": -13.748778343200684, + "step": 1926 + }, + { + "epoch": 2.52, + "learning_rate": 4.309966660194936e-05, + "logits/chosen": -2.1193394660949707, + "logits/rejected": -2.2471766471862793, + "logps/chosen": -233.57217407226562, + "logps/rejected": -302.92852783203125, + "loss": 0.0997, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.8353095054626465, + "rewards/margins": 7.02433967590332, + "rewards/rejected": -12.859648704528809, + "step": 1927 + }, + { + "epoch": 2.52, + "learning_rate": 4.309247963819597e-05, + "logits/chosen": -2.1643905639648438, + "logits/rejected": -2.1485488414764404, + "logps/chosen": -219.8082733154297, + "logps/rejected": -281.7580261230469, + "loss": 0.1519, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.9344429969787598, + "rewards/margins": 6.320281982421875, + "rewards/rejected": -10.254724502563477, + "step": 1928 + }, + { + "epoch": 2.52, + "learning_rate": 4.3085289533535126e-05, + "logits/chosen": -2.2482452392578125, + "logits/rejected": -2.1758103370666504, + "logps/chosen": -306.8867492675781, + "logps/rejected": -343.5700988769531, + "loss": 0.0011, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.638154983520508, + "rewards/margins": 8.893589973449707, + "rewards/rejected": -13.531745910644531, + "step": 1929 + }, + { + "epoch": 2.53, + "learning_rate": 4.307809628921505e-05, + "logits/chosen": -2.1660923957824707, + "logits/rejected": -2.193176507949829, + "logps/chosen": -303.06817626953125, + "logps/rejected": -382.7147216796875, + "loss": 0.0022, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.131650447845459, + "rewards/margins": 8.652751922607422, + "rewards/rejected": -14.784402847290039, + "step": 1930 + }, + { + "epoch": 2.53, + "learning_rate": 4.3070899906484515e-05, + "logits/chosen": -1.7756067514419556, + "logits/rejected": -1.7884242534637451, + "logps/chosen": -237.867919921875, + "logps/rejected": -322.68572998046875, + "loss": 0.0874, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.668308258056641, + "rewards/margins": 9.159331321716309, + "rewards/rejected": -13.827640533447266, + "step": 1931 + }, + { + "epoch": 2.53, + "learning_rate": 4.306370038659283e-05, + "logits/chosen": -1.9823037385940552, + "logits/rejected": -1.8524787425994873, + "logps/chosen": -227.22921752929688, + "logps/rejected": -342.2689208984375, + "loss": 0.0005, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.744987964630127, + "rewards/margins": 10.096012115478516, + "rewards/rejected": -14.840999603271484, + "step": 1932 + }, + { + "epoch": 2.53, + "learning_rate": 4.305649773078987e-05, + "logits/chosen": -1.9377398490905762, + "logits/rejected": -1.9592432975769043, + "logps/chosen": -294.3237609863281, + "logps/rejected": -344.45953369140625, + "loss": 0.0617, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.796104431152344, + "rewards/margins": 8.316561698913574, + "rewards/rejected": -13.112667083740234, + "step": 1933 + }, + { + "epoch": 2.53, + "learning_rate": 4.3049291940326007e-05, + "logits/chosen": -2.130969762802124, + "logits/rejected": -2.2177491188049316, + "logps/chosen": -216.68133544921875, + "logps/rejected": -305.097900390625, + "loss": 0.0447, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.305083274841309, + "rewards/margins": 7.978094100952148, + "rewards/rejected": -13.283178329467773, + "step": 1934 + }, + { + "epoch": 2.53, + "learning_rate": 4.3042083016452205e-05, + "logits/chosen": -2.074617624282837, + "logits/rejected": -2.2133822441101074, + "logps/chosen": -206.17091369628906, + "logps/rejected": -344.6942138671875, + "loss": 0.0455, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.658586502075195, + "rewards/margins": 10.279059410095215, + "rewards/rejected": -14.93764591217041, + "step": 1935 + }, + { + "epoch": 2.53, + "learning_rate": 4.303487096041996e-05, + "logits/chosen": -2.004681348800659, + "logits/rejected": -2.045778751373291, + "logps/chosen": -205.9289093017578, + "logps/rejected": -282.0126953125, + "loss": 0.054, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.651937484741211, + "rewards/margins": 9.119819641113281, + "rewards/rejected": -12.771757125854492, + "step": 1936 + }, + { + "epoch": 2.53, + "learning_rate": 4.3027655773481294e-05, + "logits/chosen": -2.2388157844543457, + "logits/rejected": -2.2951581478118896, + "logps/chosen": -220.75875854492188, + "logps/rejected": -293.530029296875, + "loss": 0.048, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.052151679992676, + "rewards/margins": 7.8823113441467285, + "rewards/rejected": -12.934463500976562, + "step": 1937 + }, + { + "epoch": 2.54, + "learning_rate": 4.302043745688879e-05, + "logits/chosen": -1.7866822481155396, + "logits/rejected": -1.8886862993240356, + "logps/chosen": -230.29017639160156, + "logps/rejected": -349.378173828125, + "loss": 0.1366, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.138594150543213, + "rewards/margins": 8.649319648742676, + "rewards/rejected": -13.78791332244873, + "step": 1938 + }, + { + "epoch": 2.54, + "learning_rate": 4.3013216011895575e-05, + "logits/chosen": -1.9889922142028809, + "logits/rejected": -1.9763219356536865, + "logps/chosen": -223.85325622558594, + "logps/rejected": -302.8853759765625, + "loss": 0.0452, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.246161937713623, + "rewards/margins": 8.81850814819336, + "rewards/rejected": -14.064668655395508, + "step": 1939 + }, + { + "epoch": 2.54, + "learning_rate": 4.300599143975529e-05, + "logits/chosen": -1.5978270769119263, + "logits/rejected": -1.6645034551620483, + "logps/chosen": -237.7843780517578, + "logps/rejected": -333.2315368652344, + "loss": 0.0893, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.7656989097595215, + "rewards/margins": 7.666550636291504, + "rewards/rejected": -13.432250022888184, + "step": 1940 + }, + { + "epoch": 2.54, + "learning_rate": 4.299876374172217e-05, + "logits/chosen": -2.169827699661255, + "logits/rejected": -2.2364706993103027, + "logps/chosen": -246.08718872070312, + "logps/rejected": -326.4261169433594, + "loss": 0.0677, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.478606224060059, + "rewards/margins": 8.510499000549316, + "rewards/rejected": -12.989105224609375, + "step": 1941 + }, + { + "epoch": 2.54, + "learning_rate": 4.2991532919050945e-05, + "logits/chosen": -1.9586464166641235, + "logits/rejected": -1.9860866069793701, + "logps/chosen": -219.77731323242188, + "logps/rejected": -288.02972412109375, + "loss": 0.056, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.910635948181152, + "rewards/margins": 7.954824447631836, + "rewards/rejected": -12.865461349487305, + "step": 1942 + }, + { + "epoch": 2.54, + "learning_rate": 4.298429897299692e-05, + "logits/chosen": -2.0101494789123535, + "logits/rejected": -2.1010165214538574, + "logps/chosen": -252.91891479492188, + "logps/rejected": -373.2341613769531, + "loss": 0.0096, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.272066593170166, + "rewards/margins": 8.000301361083984, + "rewards/rejected": -14.272368431091309, + "step": 1943 + }, + { + "epoch": 2.54, + "learning_rate": 4.297706190481592e-05, + "logits/chosen": -2.028812885284424, + "logits/rejected": -2.1959941387176514, + "logps/chosen": -293.4753112792969, + "logps/rejected": -407.5643615722656, + "loss": 0.0575, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.024116516113281, + "rewards/margins": 7.825641632080078, + "rewards/rejected": -13.84975814819336, + "step": 1944 + }, + { + "epoch": 2.55, + "learning_rate": 4.2969821715764335e-05, + "logits/chosen": -1.7866820096969604, + "logits/rejected": -1.804237723350525, + "logps/chosen": -227.78514099121094, + "logps/rejected": -305.8616943359375, + "loss": 0.0882, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.905126094818115, + "rewards/margins": 8.556636810302734, + "rewards/rejected": -13.461762428283691, + "step": 1945 + }, + { + "epoch": 2.55, + "learning_rate": 4.2962578407099065e-05, + "logits/chosen": -1.9624371528625488, + "logits/rejected": -2.035775661468506, + "logps/chosen": -218.71646118164062, + "logps/rejected": -313.052734375, + "loss": 0.0134, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.000354766845703, + "rewards/margins": 9.645698547363281, + "rewards/rejected": -14.6460542678833, + "step": 1946 + }, + { + "epoch": 2.55, + "learning_rate": 4.2955331980077574e-05, + "logits/chosen": -1.8719345331192017, + "logits/rejected": -2.02571177482605, + "logps/chosen": -217.122314453125, + "logps/rejected": -345.24407958984375, + "loss": 0.044, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.4896368980407715, + "rewards/margins": 9.622235298156738, + "rewards/rejected": -15.111871719360352, + "step": 1947 + }, + { + "epoch": 2.55, + "learning_rate": 4.294808243595787e-05, + "logits/chosen": -2.0347461700439453, + "logits/rejected": -2.038332939147949, + "logps/chosen": -234.9548797607422, + "logps/rejected": -350.6724853515625, + "loss": 0.0013, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.863475799560547, + "rewards/margins": 11.791455268859863, + "rewards/rejected": -16.654932022094727, + "step": 1948 + }, + { + "epoch": 2.55, + "learning_rate": 4.2940829775998494e-05, + "logits/chosen": -1.955130934715271, + "logits/rejected": -1.9799152612686157, + "logps/chosen": -293.515869140625, + "logps/rejected": -395.7729187011719, + "loss": 0.1102, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.166569709777832, + "rewards/margins": 8.504650115966797, + "rewards/rejected": -14.671219825744629, + "step": 1949 + }, + { + "epoch": 2.55, + "learning_rate": 4.2933574001458524e-05, + "logits/chosen": -1.9489941596984863, + "logits/rejected": -1.9857006072998047, + "logps/chosen": -256.5664367675781, + "logps/rejected": -341.5423583984375, + "loss": 0.0007, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.8608479499816895, + "rewards/margins": 10.079227447509766, + "rewards/rejected": -14.94007682800293, + "step": 1950 + }, + { + "epoch": 2.55, + "learning_rate": 4.2926315113597585e-05, + "logits/chosen": -1.9592641592025757, + "logits/rejected": -1.9216690063476562, + "logps/chosen": -266.249755859375, + "logps/rejected": -317.3481750488281, + "loss": 0.039, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.881652355194092, + "rewards/margins": 8.534133911132812, + "rewards/rejected": -13.41578483581543, + "step": 1951 + }, + { + "epoch": 2.55, + "learning_rate": 4.2919053113675845e-05, + "logits/chosen": -1.9309496879577637, + "logits/rejected": -1.9673490524291992, + "logps/chosen": -243.59275817871094, + "logps/rejected": -375.9563903808594, + "loss": 0.0044, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.986482620239258, + "rewards/margins": 9.081639289855957, + "rewards/rejected": -14.068122863769531, + "step": 1952 + }, + { + "epoch": 2.56, + "learning_rate": 4.2911788002954e-05, + "logits/chosen": -2.217352867126465, + "logits/rejected": -2.2447142601013184, + "logps/chosen": -230.89761352539062, + "logps/rejected": -357.30499267578125, + "loss": 0.0028, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.1550750732421875, + "rewards/margins": 8.492586135864258, + "rewards/rejected": -13.647661209106445, + "step": 1953 + }, + { + "epoch": 2.56, + "learning_rate": 4.290451978269331e-05, + "logits/chosen": -2.000579833984375, + "logits/rejected": -2.112433910369873, + "logps/chosen": -211.8125457763672, + "logps/rejected": -282.7967224121094, + "loss": 0.1571, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.382208824157715, + "rewards/margins": 7.275820732116699, + "rewards/rejected": -11.658029556274414, + "step": 1954 + }, + { + "epoch": 2.56, + "learning_rate": 4.2897248454155544e-05, + "logits/chosen": -1.8008928298950195, + "logits/rejected": -1.8354158401489258, + "logps/chosen": -209.97618103027344, + "logps/rejected": -256.56048583984375, + "loss": 0.1602, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.440002679824829, + "rewards/margins": 7.413758754730225, + "rewards/rejected": -10.853761672973633, + "step": 1955 + }, + { + "epoch": 2.56, + "learning_rate": 4.288997401860303e-05, + "logits/chosen": -2.3489272594451904, + "logits/rejected": -2.386540174484253, + "logps/chosen": -252.65460205078125, + "logps/rejected": -392.34906005859375, + "loss": 0.0468, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.8349080085754395, + "rewards/margins": 8.861000061035156, + "rewards/rejected": -14.695908546447754, + "step": 1956 + }, + { + "epoch": 2.56, + "learning_rate": 4.288269647729863e-05, + "logits/chosen": -2.171231985092163, + "logits/rejected": -2.142423391342163, + "logps/chosen": -292.1837158203125, + "logps/rejected": -372.4999694824219, + "loss": 0.0442, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.801056861877441, + "rewards/margins": 9.458931922912598, + "rewards/rejected": -14.259988784790039, + "step": 1957 + }, + { + "epoch": 2.56, + "learning_rate": 4.287541583150576e-05, + "logits/chosen": -1.8236223459243774, + "logits/rejected": -1.9413220882415771, + "logps/chosen": -229.91751098632812, + "logps/rejected": -327.6510314941406, + "loss": 0.0887, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.326351165771484, + "rewards/margins": 8.002523422241211, + "rewards/rejected": -13.328875541687012, + "step": 1958 + }, + { + "epoch": 2.56, + "learning_rate": 4.286813208248834e-05, + "logits/chosen": -1.8356335163116455, + "logits/rejected": -1.9949692487716675, + "logps/chosen": -212.74758911132812, + "logps/rejected": -322.6806640625, + "loss": 0.0663, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.013042449951172, + "rewards/margins": 8.232125282287598, + "rewards/rejected": -13.245166778564453, + "step": 1959 + }, + { + "epoch": 2.57, + "learning_rate": 4.2860845231510873e-05, + "logits/chosen": -2.0356662273406982, + "logits/rejected": -1.986034870147705, + "logps/chosen": -189.760498046875, + "logps/rejected": -233.2946014404297, + "loss": 0.0669, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.721360683441162, + "rewards/margins": 5.633640766143799, + "rewards/rejected": -10.355001449584961, + "step": 1960 + }, + { + "epoch": 2.57, + "learning_rate": 4.285355527983836e-05, + "logits/chosen": -2.2257447242736816, + "logits/rejected": -2.207811117172241, + "logps/chosen": -252.7733154296875, + "logps/rejected": -355.315673828125, + "loss": 0.0071, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.237861156463623, + "rewards/margins": 7.754915237426758, + "rewards/rejected": -12.992775917053223, + "step": 1961 + }, + { + "epoch": 2.57, + "learning_rate": 4.2846262228736354e-05, + "logits/chosen": -1.7899184226989746, + "logits/rejected": -1.8761553764343262, + "logps/chosen": -227.38531494140625, + "logps/rejected": -309.0900573730469, + "loss": 0.0556, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.205245018005371, + "rewards/margins": 7.232666015625, + "rewards/rejected": -12.437909126281738, + "step": 1962 + }, + { + "epoch": 2.57, + "learning_rate": 4.283896607947098e-05, + "logits/chosen": -1.791359782218933, + "logits/rejected": -1.85542631149292, + "logps/chosen": -215.9515838623047, + "logps/rejected": -274.5942687988281, + "loss": 0.0975, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.920288562774658, + "rewards/margins": 6.694026947021484, + "rewards/rejected": -11.614316940307617, + "step": 1963 + }, + { + "epoch": 2.57, + "learning_rate": 4.283166683330883e-05, + "logits/chosen": -1.8729884624481201, + "logits/rejected": -2.0725789070129395, + "logps/chosen": -237.73728942871094, + "logps/rejected": -363.7380065917969, + "loss": 0.0008, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.638874053955078, + "rewards/margins": 9.738240242004395, + "rewards/rejected": -15.377115249633789, + "step": 1964 + }, + { + "epoch": 2.57, + "learning_rate": 4.282436449151711e-05, + "logits/chosen": -1.958586573600769, + "logits/rejected": -1.9482567310333252, + "logps/chosen": -311.7152404785156, + "logps/rejected": -435.96923828125, + "loss": 0.1322, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.224272727966309, + "rewards/margins": 6.595800399780273, + "rewards/rejected": -12.820073127746582, + "step": 1965 + }, + { + "epoch": 2.57, + "learning_rate": 4.2817059055363505e-05, + "logits/chosen": -1.7612496614456177, + "logits/rejected": -1.7791756391525269, + "logps/chosen": -194.8032684326172, + "logps/rejected": -292.1495666503906, + "loss": 0.2856, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.353596210479736, + "rewards/margins": 5.999757766723633, + "rewards/rejected": -11.353353500366211, + "step": 1966 + }, + { + "epoch": 2.57, + "learning_rate": 4.280975052611627e-05, + "logits/chosen": -2.081310749053955, + "logits/rejected": -2.175403118133545, + "logps/chosen": -210.8900909423828, + "logps/rejected": -279.8389892578125, + "loss": 0.0466, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.361638069152832, + "rewards/margins": 7.6152801513671875, + "rewards/rejected": -12.976919174194336, + "step": 1967 + }, + { + "epoch": 2.58, + "learning_rate": 4.280243890504419e-05, + "logits/chosen": -2.237908363342285, + "logits/rejected": -2.206061601638794, + "logps/chosen": -254.23226928710938, + "logps/rejected": -341.8685302734375, + "loss": 0.0449, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.933377742767334, + "rewards/margins": 8.773107528686523, + "rewards/rejected": -13.706486701965332, + "step": 1968 + }, + { + "epoch": 2.58, + "learning_rate": 4.279512419341657e-05, + "logits/chosen": -2.1924283504486084, + "logits/rejected": -2.058635950088501, + "logps/chosen": -260.7196350097656, + "logps/rejected": -359.5116271972656, + "loss": 0.0917, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.284665107727051, + "rewards/margins": 7.44911527633667, + "rewards/rejected": -12.733780860900879, + "step": 1969 + }, + { + "epoch": 2.58, + "learning_rate": 4.278780639250328e-05, + "logits/chosen": -2.099184274673462, + "logits/rejected": -2.174762010574341, + "logps/chosen": -282.62225341796875, + "logps/rejected": -377.4095153808594, + "loss": 0.1133, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.72321081161499, + "rewards/margins": 6.761861801147461, + "rewards/rejected": -12.48507308959961, + "step": 1970 + }, + { + "epoch": 2.58, + "learning_rate": 4.27804855035747e-05, + "logits/chosen": -1.8911799192428589, + "logits/rejected": -2.0045738220214844, + "logps/chosen": -269.79632568359375, + "logps/rejected": -364.15740966796875, + "loss": 0.0892, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.91278076171875, + "rewards/margins": 7.7511305809021, + "rewards/rejected": -12.663909912109375, + "step": 1971 + }, + { + "epoch": 2.58, + "learning_rate": 4.277316152790177e-05, + "logits/chosen": -1.9748018980026245, + "logits/rejected": -2.0676732063293457, + "logps/chosen": -235.08203125, + "logps/rejected": -371.6739807128906, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.827308177947998, + "rewards/margins": 9.296005249023438, + "rewards/rejected": -15.123311996459961, + "step": 1972 + }, + { + "epoch": 2.58, + "learning_rate": 4.276583446675595e-05, + "logits/chosen": -2.147160291671753, + "logits/rejected": -2.117455005645752, + "logps/chosen": -238.53048706054688, + "logps/rejected": -305.900390625, + "loss": 0.0106, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.343995571136475, + "rewards/margins": 7.646453857421875, + "rewards/rejected": -12.990449905395508, + "step": 1973 + }, + { + "epoch": 2.58, + "learning_rate": 4.275850432140923e-05, + "logits/chosen": -2.092256546020508, + "logits/rejected": -2.1781668663024902, + "logps/chosen": -225.18380737304688, + "logps/rejected": -339.3202819824219, + "loss": 0.0457, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.446767807006836, + "rewards/margins": 8.153047561645508, + "rewards/rejected": -13.599814414978027, + "step": 1974 + }, + { + "epoch": 2.58, + "learning_rate": 4.275117109313415e-05, + "logits/chosen": -1.8921672105789185, + "logits/rejected": -1.9029520750045776, + "logps/chosen": -227.0023651123047, + "logps/rejected": -343.0833740234375, + "loss": 0.1068, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.755160331726074, + "rewards/margins": 7.769000053405762, + "rewards/rejected": -12.52415943145752, + "step": 1975 + }, + { + "epoch": 2.59, + "learning_rate": 4.2743834783203785e-05, + "logits/chosen": -2.2617921829223633, + "logits/rejected": -2.29345440864563, + "logps/chosen": -250.0797119140625, + "logps/rejected": -349.07073974609375, + "loss": 0.0253, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.5307135581970215, + "rewards/margins": 8.098857879638672, + "rewards/rejected": -12.629570007324219, + "step": 1976 + }, + { + "epoch": 2.59, + "learning_rate": 4.273649539289174e-05, + "logits/chosen": -2.126615524291992, + "logits/rejected": -2.1196091175079346, + "logps/chosen": -215.05816650390625, + "logps/rejected": -294.6878662109375, + "loss": 0.0083, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.706331253051758, + "rewards/margins": 9.184209823608398, + "rewards/rejected": -13.890541076660156, + "step": 1977 + }, + { + "epoch": 2.59, + "learning_rate": 4.272915292347215e-05, + "logits/chosen": -2.0412535667419434, + "logits/rejected": -2.0456371307373047, + "logps/chosen": -299.03790283203125, + "logps/rejected": -320.651123046875, + "loss": 0.0272, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.878506660461426, + "rewards/margins": 7.792044162750244, + "rewards/rejected": -12.670549392700195, + "step": 1978 + }, + { + "epoch": 2.59, + "learning_rate": 4.272180737621968e-05, + "logits/chosen": -2.234879493713379, + "logits/rejected": -2.155164957046509, + "logps/chosen": -241.6581268310547, + "logps/rejected": -274.40850830078125, + "loss": 0.128, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.20209264755249, + "rewards/margins": 7.107535362243652, + "rewards/rejected": -12.3096284866333, + "step": 1979 + }, + { + "epoch": 2.59, + "learning_rate": 4.271445875240956e-05, + "logits/chosen": -2.014817714691162, + "logits/rejected": -2.011441469192505, + "logps/chosen": -248.09115600585938, + "logps/rejected": -301.4862365722656, + "loss": 0.054, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.681390762329102, + "rewards/margins": 7.81306791305542, + "rewards/rejected": -12.494458198547363, + "step": 1980 + }, + { + "epoch": 2.59, + "learning_rate": 4.270710705331752e-05, + "logits/chosen": -2.0360469818115234, + "logits/rejected": -2.0951154232025146, + "logps/chosen": -235.5208740234375, + "logps/rejected": -318.1795959472656, + "loss": 0.0456, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.8345842361450195, + "rewards/margins": 8.38646411895752, + "rewards/rejected": -13.221047401428223, + "step": 1981 + }, + { + "epoch": 2.59, + "learning_rate": 4.269975228021983e-05, + "logits/chosen": -2.0260274410247803, + "logits/rejected": -2.068878412246704, + "logps/chosen": -277.013671875, + "logps/rejected": -365.9017639160156, + "loss": 0.0443, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.482072353363037, + "rewards/margins": 9.523672103881836, + "rewards/rejected": -14.005743980407715, + "step": 1982 + }, + { + "epoch": 2.6, + "learning_rate": 4.2692394434393313e-05, + "logits/chosen": -2.2093918323516846, + "logits/rejected": -2.239203453063965, + "logps/chosen": -237.828125, + "logps/rejected": -312.05755615234375, + "loss": 0.1387, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.4257378578186035, + "rewards/margins": 7.293093681335449, + "rewards/rejected": -11.718832969665527, + "step": 1983 + }, + { + "epoch": 2.6, + "learning_rate": 4.268503351711531e-05, + "logits/chosen": -1.9702582359313965, + "logits/rejected": -2.1271867752075195, + "logps/chosen": -304.22900390625, + "logps/rejected": -398.12481689453125, + "loss": 0.0886, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.615696907043457, + "rewards/margins": 7.297685623168945, + "rewards/rejected": -13.913382530212402, + "step": 1984 + }, + { + "epoch": 2.6, + "learning_rate": 4.267766952966369e-05, + "logits/chosen": -2.177107810974121, + "logits/rejected": -2.2635247707366943, + "logps/chosen": -201.6862030029297, + "logps/rejected": -304.0697021484375, + "loss": 0.0172, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.194709777832031, + "rewards/margins": 8.228707313537598, + "rewards/rejected": -12.423417091369629, + "step": 1985 + }, + { + "epoch": 2.6, + "learning_rate": 4.267030247331687e-05, + "logits/chosen": -1.8618240356445312, + "logits/rejected": -1.7992628812789917, + "logps/chosen": -226.5166015625, + "logps/rejected": -268.01641845703125, + "loss": 0.0919, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.920095920562744, + "rewards/margins": 7.572173118591309, + "rewards/rejected": -11.492269515991211, + "step": 1986 + }, + { + "epoch": 2.6, + "learning_rate": 4.266293234935379e-05, + "logits/chosen": -2.151167392730713, + "logits/rejected": -2.200986385345459, + "logps/chosen": -209.70037841796875, + "logps/rejected": -284.14422607421875, + "loss": 0.0521, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.9986391067504883, + "rewards/margins": 6.333474159240723, + "rewards/rejected": -10.332113265991211, + "step": 1987 + }, + { + "epoch": 2.6, + "learning_rate": 4.265555915905393e-05, + "logits/chosen": -1.9930312633514404, + "logits/rejected": -1.8740456104278564, + "logps/chosen": -240.62808227539062, + "logps/rejected": -283.4354553222656, + "loss": 0.1025, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.969852447509766, + "rewards/margins": 6.814610004425049, + "rewards/rejected": -11.784461975097656, + "step": 1988 + }, + { + "epoch": 2.6, + "learning_rate": 4.264818290369729e-05, + "logits/chosen": -2.1442923545837402, + "logits/rejected": -2.1885173320770264, + "logps/chosen": -227.04296875, + "logps/rejected": -342.5315856933594, + "loss": 0.0049, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.07236909866333, + "rewards/margins": 9.052876472473145, + "rewards/rejected": -14.125244140625, + "step": 1989 + }, + { + "epoch": 2.6, + "learning_rate": 4.264080358456441e-05, + "logits/chosen": -1.810939908027649, + "logits/rejected": -1.8494899272918701, + "logps/chosen": -250.21864318847656, + "logps/rejected": -322.9114990234375, + "loss": 0.0893, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.772063255310059, + "rewards/margins": 7.34108829498291, + "rewards/rejected": -12.113152503967285, + "step": 1990 + }, + { + "epoch": 2.61, + "learning_rate": 4.263342120293636e-05, + "logits/chosen": -1.9354889392852783, + "logits/rejected": -2.0048763751983643, + "logps/chosen": -231.1033477783203, + "logps/rejected": -296.833984375, + "loss": 0.0519, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.519188404083252, + "rewards/margins": 8.479740142822266, + "rewards/rejected": -11.998929023742676, + "step": 1991 + }, + { + "epoch": 2.61, + "learning_rate": 4.2626035760094763e-05, + "logits/chosen": -2.1611340045928955, + "logits/rejected": -2.1724023818969727, + "logps/chosen": -274.818359375, + "logps/rejected": -365.2533874511719, + "loss": 0.044, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.13316535949707, + "rewards/margins": 7.889025688171387, + "rewards/rejected": -12.02219009399414, + "step": 1992 + }, + { + "epoch": 2.61, + "learning_rate": 4.261864725732172e-05, + "logits/chosen": -2.252372980117798, + "logits/rejected": -2.216064453125, + "logps/chosen": -274.22711181640625, + "logps/rejected": -386.2915344238281, + "loss": 0.0006, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.8807830810546875, + "rewards/margins": 9.966548919677734, + "rewards/rejected": -13.847332954406738, + "step": 1993 + }, + { + "epoch": 2.61, + "learning_rate": 4.261125569589993e-05, + "logits/chosen": -2.0184173583984375, + "logits/rejected": -2.0977327823638916, + "logps/chosen": -200.14089965820312, + "logps/rejected": -281.9545593261719, + "loss": 0.028, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.331934928894043, + "rewards/margins": 8.05714225769043, + "rewards/rejected": -11.389076232910156, + "step": 1994 + }, + { + "epoch": 2.61, + "learning_rate": 4.260386107711256e-05, + "logits/chosen": -1.8014366626739502, + "logits/rejected": -1.7428510189056396, + "logps/chosen": -237.07882690429688, + "logps/rejected": -313.3893737792969, + "loss": 0.0968, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.013710021972656, + "rewards/margins": 6.934698581695557, + "rewards/rejected": -10.948408126831055, + "step": 1995 + }, + { + "epoch": 2.61, + "learning_rate": 4.259646340224336e-05, + "logits/chosen": -2.0515670776367188, + "logits/rejected": -2.1084394454956055, + "logps/chosen": -253.11888122558594, + "logps/rejected": -400.4759216308594, + "loss": 0.0211, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.113040924072266, + "rewards/margins": 8.693325996398926, + "rewards/rejected": -13.806365966796875, + "step": 1996 + }, + { + "epoch": 2.61, + "learning_rate": 4.2589062672576576e-05, + "logits/chosen": -1.991852879524231, + "logits/rejected": -2.1616296768188477, + "logps/chosen": -201.46896362304688, + "logps/rejected": -320.1032409667969, + "loss": 0.0021, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.442726135253906, + "rewards/margins": 8.485641479492188, + "rewards/rejected": -12.928366661071777, + "step": 1997 + }, + { + "epoch": 2.61, + "learning_rate": 4.2581658889397e-05, + "logits/chosen": -1.7517123222351074, + "logits/rejected": -1.7525990009307861, + "logps/chosen": -241.78564453125, + "logps/rejected": -282.4005126953125, + "loss": 0.1314, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.3433334827423096, + "rewards/margins": 7.258170127868652, + "rewards/rejected": -10.6015043258667, + "step": 1998 + }, + { + "epoch": 2.62, + "learning_rate": 4.257425205398995e-05, + "logits/chosen": -2.0399134159088135, + "logits/rejected": -2.096323013305664, + "logps/chosen": -213.169677734375, + "logps/rejected": -356.84088134765625, + "loss": 0.1302, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.694550037384033, + "rewards/margins": 9.911086082458496, + "rewards/rejected": -13.605636596679688, + "step": 1999 + }, + { + "epoch": 2.62, + "learning_rate": 4.256684216764127e-05, + "logits/chosen": -2.1457009315490723, + "logits/rejected": -2.224804401397705, + "logps/chosen": -271.61932373046875, + "logps/rejected": -375.3747863769531, + "loss": 0.0008, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.7626380920410156, + "rewards/margins": 9.301042556762695, + "rewards/rejected": -13.063680648803711, + "step": 2000 + }, + { + "epoch": 2.62, + "learning_rate": 4.255942923163736e-05, + "logits/chosen": -2.1172068119049072, + "logits/rejected": -2.1737592220306396, + "logps/chosen": -203.1175537109375, + "logps/rejected": -280.4482727050781, + "loss": 0.1161, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.822445869445801, + "rewards/margins": 6.7819695472717285, + "rewards/rejected": -11.604415893554688, + "step": 2001 + }, + { + "epoch": 2.62, + "learning_rate": 4.2552013247265096e-05, + "logits/chosen": -1.992425560951233, + "logits/rejected": -1.9609733819961548, + "logps/chosen": -237.23434448242188, + "logps/rejected": -298.7255554199219, + "loss": 0.0156, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.9892876148223877, + "rewards/margins": 7.11054801940918, + "rewards/rejected": -11.099835395812988, + "step": 2002 + }, + { + "epoch": 2.62, + "learning_rate": 4.2544594215811936e-05, + "logits/chosen": -2.161130428314209, + "logits/rejected": -2.1829681396484375, + "logps/chosen": -256.2269287109375, + "logps/rejected": -337.739013671875, + "loss": 0.09, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.339508533477783, + "rewards/margins": 6.66135311126709, + "rewards/rejected": -11.000860214233398, + "step": 2003 + }, + { + "epoch": 2.62, + "learning_rate": 4.253717213856583e-05, + "logits/chosen": -1.951271891593933, + "logits/rejected": -2.142191171646118, + "logps/chosen": -194.9743194580078, + "logps/rejected": -322.3970031738281, + "loss": 0.0223, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.501600742340088, + "rewards/margins": 9.656272888183594, + "rewards/rejected": -14.157873153686523, + "step": 2004 + }, + { + "epoch": 2.62, + "learning_rate": 4.252974701681528e-05, + "logits/chosen": -2.2527217864990234, + "logits/rejected": -2.328521728515625, + "logps/chosen": -218.23036193847656, + "logps/rejected": -293.6946105957031, + "loss": 0.046, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.480393886566162, + "rewards/margins": 6.972497940063477, + "rewards/rejected": -11.452892303466797, + "step": 2005 + }, + { + "epoch": 2.63, + "learning_rate": 4.252231885184932e-05, + "logits/chosen": -2.1791348457336426, + "logits/rejected": -2.2902534008026123, + "logps/chosen": -266.1272888183594, + "logps/rejected": -347.8664855957031, + "loss": 0.0595, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.511211395263672, + "rewards/margins": 6.929591655731201, + "rewards/rejected": -12.440802574157715, + "step": 2006 + }, + { + "epoch": 2.63, + "learning_rate": 4.251488764495748e-05, + "logits/chosen": -1.8994340896606445, + "logits/rejected": -1.9029484987258911, + "logps/chosen": -256.0934143066406, + "logps/rejected": -326.8026428222656, + "loss": 0.0878, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.944005012512207, + "rewards/margins": 7.92893648147583, + "rewards/rejected": -12.872941017150879, + "step": 2007 + }, + { + "epoch": 2.63, + "learning_rate": 4.2507453397429855e-05, + "logits/chosen": -1.8833651542663574, + "logits/rejected": -2.036100149154663, + "logps/chosen": -243.87295532226562, + "logps/rejected": -350.85479736328125, + "loss": 0.0291, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.893523693084717, + "rewards/margins": 7.373872756958008, + "rewards/rejected": -12.267395973205566, + "step": 2008 + }, + { + "epoch": 2.63, + "learning_rate": 4.2500016110557044e-05, + "logits/chosen": -2.0685205459594727, + "logits/rejected": -2.0962882041931152, + "logps/chosen": -209.53982543945312, + "logps/rejected": -284.69525146484375, + "loss": 0.0324, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.660156726837158, + "rewards/margins": 7.2062087059021, + "rewards/rejected": -11.866365432739258, + "step": 2009 + }, + { + "epoch": 2.63, + "learning_rate": 4.249257578563019e-05, + "logits/chosen": -2.0176610946655273, + "logits/rejected": -2.0090808868408203, + "logps/chosen": -218.67474365234375, + "logps/rejected": -309.4306945800781, + "loss": 0.0883, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.934572219848633, + "rewards/margins": 10.24348258972168, + "rewards/rejected": -13.178053855895996, + "step": 2010 + }, + { + "epoch": 2.63, + "learning_rate": 4.2485132423940944e-05, + "logits/chosen": -2.4200572967529297, + "logits/rejected": -2.2854561805725098, + "logps/chosen": -251.91578674316406, + "logps/rejected": -281.0067138671875, + "loss": 0.1013, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.3066606521606445, + "rewards/margins": 5.664590358734131, + "rewards/rejected": -10.971250534057617, + "step": 2011 + }, + { + "epoch": 2.63, + "learning_rate": 4.2477686026781505e-05, + "logits/chosen": -1.8213557004928589, + "logits/rejected": -1.8547894954681396, + "logps/chosen": -276.2442932128906, + "logps/rejected": -335.45361328125, + "loss": 0.1162, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.994606971740723, + "rewards/margins": 7.177623271942139, + "rewards/rejected": -13.17223072052002, + "step": 2012 + }, + { + "epoch": 2.63, + "learning_rate": 4.247023659544458e-05, + "logits/chosen": -1.8445768356323242, + "logits/rejected": -1.8069664239883423, + "logps/chosen": -237.87130737304688, + "logps/rejected": -332.58685302734375, + "loss": 0.1008, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.717880725860596, + "rewards/margins": 7.067141056060791, + "rewards/rejected": -12.785021781921387, + "step": 2013 + }, + { + "epoch": 2.64, + "learning_rate": 4.2462784131223434e-05, + "logits/chosen": -1.8459657430648804, + "logits/rejected": -1.9359010457992554, + "logps/chosen": -231.81597900390625, + "logps/rejected": -315.25128173828125, + "loss": 0.0558, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.855999231338501, + "rewards/margins": 8.611515045166016, + "rewards/rejected": -12.467513084411621, + "step": 2014 + }, + { + "epoch": 2.64, + "learning_rate": 4.24553286354118e-05, + "logits/chosen": -1.9260315895080566, + "logits/rejected": -2.0133044719696045, + "logps/chosen": -242.34796142578125, + "logps/rejected": -330.73486328125, + "loss": 0.0883, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.197949409484863, + "rewards/margins": 7.258923530578613, + "rewards/rejected": -12.456872940063477, + "step": 2015 + }, + { + "epoch": 2.64, + "learning_rate": 4.2447870109304007e-05, + "logits/chosen": -2.1096689701080322, + "logits/rejected": -2.100594997406006, + "logps/chosen": -176.09457397460938, + "logps/rejected": -262.0912780761719, + "loss": 0.0945, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.2592055797576904, + "rewards/margins": 6.781350135803223, + "rewards/rejected": -10.040555000305176, + "step": 2016 + }, + { + "epoch": 2.64, + "learning_rate": 4.244040855419486e-05, + "logits/chosen": -2.171529769897461, + "logits/rejected": -2.1618311405181885, + "logps/chosen": -303.1656494140625, + "logps/rejected": -379.5803527832031, + "loss": 0.0464, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.887218952178955, + "rewards/margins": 8.025300025939941, + "rewards/rejected": -12.912520408630371, + "step": 2017 + }, + { + "epoch": 2.64, + "learning_rate": 4.243294397137971e-05, + "logits/chosen": -1.879332423210144, + "logits/rejected": -1.950081467628479, + "logps/chosen": -233.27285766601562, + "logps/rejected": -300.0467529296875, + "loss": 0.0454, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.6014180183410645, + "rewards/margins": 7.412932395935059, + "rewards/rejected": -12.014349937438965, + "step": 2018 + }, + { + "epoch": 2.64, + "learning_rate": 4.2425476362154436e-05, + "logits/chosen": -2.1640138626098633, + "logits/rejected": -2.1587491035461426, + "logps/chosen": -239.59934997558594, + "logps/rejected": -304.4157409667969, + "loss": 0.0461, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.2458882331848145, + "rewards/margins": 7.059645652770996, + "rewards/rejected": -12.305534362792969, + "step": 2019 + }, + { + "epoch": 2.64, + "learning_rate": 4.241800572781543e-05, + "logits/chosen": -2.160501718521118, + "logits/rejected": -2.304579496383667, + "logps/chosen": -261.8168029785156, + "logps/rejected": -395.4276428222656, + "loss": 0.0165, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.653003215789795, + "rewards/margins": 9.462180137634277, + "rewards/rejected": -15.115182876586914, + "step": 2020 + }, + { + "epoch": 2.64, + "learning_rate": 4.241053206965962e-05, + "logits/chosen": -1.970011830329895, + "logits/rejected": -2.1001381874084473, + "logps/chosen": -247.37672424316406, + "logps/rejected": -360.9378662109375, + "loss": 0.0853, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.5289154052734375, + "rewards/margins": 7.392343521118164, + "rewards/rejected": -12.921258926391602, + "step": 2021 + }, + { + "epoch": 2.65, + "learning_rate": 4.240305538898445e-05, + "logits/chosen": -2.243610143661499, + "logits/rejected": -2.2371904850006104, + "logps/chosen": -259.6739807128906, + "logps/rejected": -319.3279113769531, + "loss": 0.1314, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.689486026763916, + "rewards/margins": 7.2585906982421875, + "rewards/rejected": -11.948076248168945, + "step": 2022 + }, + { + "epoch": 2.65, + "learning_rate": 4.23955756870879e-05, + "logits/chosen": -2.2176706790924072, + "logits/rejected": -2.1681039333343506, + "logps/chosen": -203.82867431640625, + "logps/rejected": -276.8878479003906, + "loss": 0.0925, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.021929740905762, + "rewards/margins": 6.763903617858887, + "rewards/rejected": -11.785833358764648, + "step": 2023 + }, + { + "epoch": 2.65, + "learning_rate": 4.238809296526847e-05, + "logits/chosen": -1.669848918914795, + "logits/rejected": -1.7608784437179565, + "logps/chosen": -234.96856689453125, + "logps/rejected": -347.46417236328125, + "loss": 0.0494, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.062384128570557, + "rewards/margins": 8.291193962097168, + "rewards/rejected": -13.353578567504883, + "step": 2024 + }, + { + "epoch": 2.65, + "learning_rate": 4.238060722482517e-05, + "logits/chosen": -1.968036413192749, + "logits/rejected": -2.05326509475708, + "logps/chosen": -270.92071533203125, + "logps/rejected": -378.5557861328125, + "loss": 0.0983, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.9035468101501465, + "rewards/margins": 7.38877534866333, + "rewards/rejected": -13.292322158813477, + "step": 2025 + }, + { + "epoch": 2.65, + "learning_rate": 4.237311846705755e-05, + "logits/chosen": -2.2504067420959473, + "logits/rejected": -2.1848998069763184, + "logps/chosen": -256.95086669921875, + "logps/rejected": -323.0563049316406, + "loss": 0.0443, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.150045394897461, + "rewards/margins": 9.079219818115234, + "rewards/rejected": -14.229267120361328, + "step": 2026 + }, + { + "epoch": 2.65, + "learning_rate": 4.2365626693265694e-05, + "logits/chosen": -1.9853413105010986, + "logits/rejected": -1.9951058626174927, + "logps/chosen": -275.75726318359375, + "logps/rejected": -333.6006164550781, + "loss": 0.0894, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.940235614776611, + "rewards/margins": 8.546769142150879, + "rewards/rejected": -13.487004280090332, + "step": 2027 + }, + { + "epoch": 2.65, + "learning_rate": 4.235813190475018e-05, + "logits/chosen": -2.1142847537994385, + "logits/rejected": -2.0551466941833496, + "logps/chosen": -197.76902770996094, + "logps/rejected": -251.27593994140625, + "loss": 0.0499, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.9538817405700684, + "rewards/margins": 8.75600528717041, + "rewards/rejected": -12.70988655090332, + "step": 2028 + }, + { + "epoch": 2.66, + "learning_rate": 4.2350634102812134e-05, + "logits/chosen": -1.9945062398910522, + "logits/rejected": -2.0655226707458496, + "logps/chosen": -203.58526611328125, + "logps/rejected": -327.26776123046875, + "loss": 0.0887, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.519442558288574, + "rewards/margins": 6.988471984863281, + "rewards/rejected": -11.507914543151855, + "step": 2029 + }, + { + "epoch": 2.66, + "learning_rate": 4.2343133288753194e-05, + "logits/chosen": -1.9564071893692017, + "logits/rejected": -2.0216352939605713, + "logps/chosen": -222.12017822265625, + "logps/rejected": -285.66717529296875, + "loss": 0.0892, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.716222763061523, + "rewards/margins": 7.841244220733643, + "rewards/rejected": -12.557466506958008, + "step": 2030 + }, + { + "epoch": 2.66, + "learning_rate": 4.233562946387553e-05, + "logits/chosen": -1.8869245052337646, + "logits/rejected": -1.865543246269226, + "logps/chosen": -232.640380859375, + "logps/rejected": -314.2353820800781, + "loss": 0.0494, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.838282108306885, + "rewards/margins": 7.584604740142822, + "rewards/rejected": -12.422886848449707, + "step": 2031 + }, + { + "epoch": 2.66, + "learning_rate": 4.232812262948182e-05, + "logits/chosen": -1.9131499528884888, + "logits/rejected": -2.000537395477295, + "logps/chosen": -198.13389587402344, + "logps/rejected": -307.91473388671875, + "loss": 0.0014, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.581729888916016, + "rewards/margins": 8.919013023376465, + "rewards/rejected": -13.50074291229248, + "step": 2032 + }, + { + "epoch": 2.66, + "learning_rate": 4.232061278687526e-05, + "logits/chosen": -1.8416846990585327, + "logits/rejected": -1.8420649766921997, + "logps/chosen": -271.35919189453125, + "logps/rejected": -370.6395568847656, + "loss": 0.0871, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.359163284301758, + "rewards/margins": 8.96020793914795, + "rewards/rejected": -14.319372177124023, + "step": 2033 + }, + { + "epoch": 2.66, + "learning_rate": 4.23130999373596e-05, + "logits/chosen": -2.199634075164795, + "logits/rejected": -2.315343141555786, + "logps/chosen": -244.52755737304688, + "logps/rejected": -343.6764831542969, + "loss": 0.0091, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.335355281829834, + "rewards/margins": 9.197492599487305, + "rewards/rejected": -13.532848358154297, + "step": 2034 + }, + { + "epoch": 2.66, + "learning_rate": 4.230558408223909e-05, + "logits/chosen": -2.3857579231262207, + "logits/rejected": -2.346330404281616, + "logps/chosen": -286.7845153808594, + "logps/rejected": -358.5130310058594, + "loss": 0.0875, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.38930606842041, + "rewards/margins": 7.837110996246338, + "rewards/rejected": -13.22641658782959, + "step": 2035 + }, + { + "epoch": 2.66, + "learning_rate": 4.22980652228185e-05, + "logits/chosen": -1.974481463432312, + "logits/rejected": -1.9705196619033813, + "logps/chosen": -259.6114807128906, + "logps/rejected": -350.9654235839844, + "loss": 0.1726, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.30286169052124, + "rewards/margins": 8.534625053405762, + "rewards/rejected": -13.837486267089844, + "step": 2036 + }, + { + "epoch": 2.67, + "learning_rate": 4.229054336040312e-05, + "logits/chosen": -1.9328861236572266, + "logits/rejected": -1.965613842010498, + "logps/chosen": -223.0769500732422, + "logps/rejected": -322.9892578125, + "loss": 0.0453, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.8855221271514893, + "rewards/margins": 10.301925659179688, + "rewards/rejected": -14.187447547912598, + "step": 2037 + }, + { + "epoch": 2.67, + "learning_rate": 4.228301849629879e-05, + "logits/chosen": -2.1937811374664307, + "logits/rejected": -2.216660737991333, + "logps/chosen": -253.26025390625, + "logps/rejected": -344.1639099121094, + "loss": 0.1048, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.741037368774414, + "rewards/margins": 8.45821762084961, + "rewards/rejected": -13.19925594329834, + "step": 2038 + }, + { + "epoch": 2.67, + "learning_rate": 4.227549063181183e-05, + "logits/chosen": -2.151825189590454, + "logits/rejected": -2.158660650253296, + "logps/chosen": -276.98419189453125, + "logps/rejected": -352.35699462890625, + "loss": 0.0375, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.893413543701172, + "rewards/margins": 9.326420783996582, + "rewards/rejected": -14.219833374023438, + "step": 2039 + }, + { + "epoch": 2.67, + "learning_rate": 4.22679597682491e-05, + "logits/chosen": -2.0169548988342285, + "logits/rejected": -2.21177077293396, + "logps/chosen": -262.75469970703125, + "logps/rejected": -369.38201904296875, + "loss": 0.0013, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.374359607696533, + "rewards/margins": 8.58126449584961, + "rewards/rejected": -13.955623626708984, + "step": 2040 + }, + { + "epoch": 2.67, + "learning_rate": 4.2260425906917975e-05, + "logits/chosen": -2.1499552726745605, + "logits/rejected": -2.195781946182251, + "logps/chosen": -215.60107421875, + "logps/rejected": -310.01287841796875, + "loss": 0.133, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.024834156036377, + "rewards/margins": 7.125798225402832, + "rewards/rejected": -11.15063190460205, + "step": 2041 + }, + { + "epoch": 2.67, + "learning_rate": 4.2252889049126366e-05, + "logits/chosen": -2.129696846008301, + "logits/rejected": -2.079493284225464, + "logps/chosen": -232.66456604003906, + "logps/rejected": -298.35205078125, + "loss": 0.094, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.621220588684082, + "rewards/margins": 7.01010799407959, + "rewards/rejected": -11.631328582763672, + "step": 2042 + }, + { + "epoch": 2.67, + "learning_rate": 4.224534919618269e-05, + "logits/chosen": -2.1040329933166504, + "logits/rejected": -2.150583267211914, + "logps/chosen": -217.6129608154297, + "logps/rejected": -315.8685302734375, + "loss": 0.0523, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.8543291091918945, + "rewards/margins": 8.18506908416748, + "rewards/rejected": -13.039399147033691, + "step": 2043 + }, + { + "epoch": 2.67, + "learning_rate": 4.22378063493959e-05, + "logits/chosen": -2.154564380645752, + "logits/rejected": -2.1789019107818604, + "logps/chosen": -208.6300506591797, + "logps/rejected": -282.248046875, + "loss": 0.04, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.85252046585083, + "rewards/margins": 7.109794616699219, + "rewards/rejected": -10.96231460571289, + "step": 2044 + }, + { + "epoch": 2.68, + "learning_rate": 4.223026051007544e-05, + "logits/chosen": -1.9202934503555298, + "logits/rejected": -1.9896492958068848, + "logps/chosen": -287.2677001953125, + "logps/rejected": -405.3075256347656, + "loss": 0.0458, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.884298324584961, + "rewards/margins": 8.85289478302002, + "rewards/rejected": -12.73719310760498, + "step": 2045 + }, + { + "epoch": 2.68, + "learning_rate": 4.222271167953128e-05, + "logits/chosen": -1.9422993659973145, + "logits/rejected": -1.962449073791504, + "logps/chosen": -291.1929626464844, + "logps/rejected": -366.3737487792969, + "loss": 0.0457, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.793270111083984, + "rewards/margins": 7.609700679779053, + "rewards/rejected": -12.402970314025879, + "step": 2046 + }, + { + "epoch": 2.68, + "learning_rate": 4.2215159859073944e-05, + "logits/chosen": -2.332904577255249, + "logits/rejected": -2.3232131004333496, + "logps/chosen": -207.947998046875, + "logps/rejected": -289.6778564453125, + "loss": 0.0938, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.780367374420166, + "rewards/margins": 8.799994468688965, + "rewards/rejected": -12.580362319946289, + "step": 2047 + }, + { + "epoch": 2.68, + "learning_rate": 4.2207605050014435e-05, + "logits/chosen": -2.3431074619293213, + "logits/rejected": -2.3058929443359375, + "logps/chosen": -224.7828369140625, + "logps/rejected": -314.53668212890625, + "loss": 0.1395, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.5304112434387207, + "rewards/margins": 6.5715413093566895, + "rewards/rejected": -10.10195255279541, + "step": 2048 + }, + { + "epoch": 2.68, + "learning_rate": 4.22000472536643e-05, + "logits/chosen": -1.9623531103134155, + "logits/rejected": -2.0505189895629883, + "logps/chosen": -237.1623077392578, + "logps/rejected": -324.4232177734375, + "loss": 0.0067, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.392932415008545, + "rewards/margins": 7.678095817565918, + "rewards/rejected": -12.071028709411621, + "step": 2049 + }, + { + "epoch": 2.68, + "learning_rate": 4.2192486471335585e-05, + "logits/chosen": -2.3527064323425293, + "logits/rejected": -2.3956918716430664, + "logps/chosen": -237.3239288330078, + "logps/rejected": -343.18817138671875, + "loss": 0.0481, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.818669557571411, + "rewards/margins": 9.966001510620117, + "rewards/rejected": -13.784668922424316, + "step": 2050 + }, + { + "epoch": 2.68, + "learning_rate": 4.2184922704340866e-05, + "logits/chosen": -2.1499099731445312, + "logits/rejected": -2.116560935974121, + "logps/chosen": -213.3365936279297, + "logps/rejected": -318.2062072753906, + "loss": 0.0703, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.0955376625061035, + "rewards/margins": 8.705997467041016, + "rewards/rejected": -12.801535606384277, + "step": 2051 + }, + { + "epoch": 2.69, + "learning_rate": 4.217735595399324e-05, + "logits/chosen": -2.3184332847595215, + "logits/rejected": -2.389354944229126, + "logps/chosen": -230.14830017089844, + "logps/rejected": -325.69573974609375, + "loss": 0.0013, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.083515167236328, + "rewards/margins": 9.438323020935059, + "rewards/rejected": -13.52183723449707, + "step": 2052 + }, + { + "epoch": 2.69, + "learning_rate": 4.2169786221606315e-05, + "logits/chosen": -2.1536691188812256, + "logits/rejected": -2.2255914211273193, + "logps/chosen": -208.54898071289062, + "logps/rejected": -316.613037109375, + "loss": 0.0439, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.340245723724365, + "rewards/margins": 9.141230583190918, + "rewards/rejected": -13.481475830078125, + "step": 2053 + }, + { + "epoch": 2.69, + "learning_rate": 4.216221350849422e-05, + "logits/chosen": -2.342555046081543, + "logits/rejected": -2.3749189376831055, + "logps/chosen": -271.072509765625, + "logps/rejected": -318.45648193359375, + "loss": 0.0783, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.5725584030151367, + "rewards/margins": 8.381831169128418, + "rewards/rejected": -11.954388618469238, + "step": 2054 + }, + { + "epoch": 2.69, + "learning_rate": 4.215463781597159e-05, + "logits/chosen": -2.0869643688201904, + "logits/rejected": -2.116182327270508, + "logps/chosen": -261.493896484375, + "logps/rejected": -365.8189697265625, + "loss": 0.0987, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.4405364990234375, + "rewards/margins": 7.905498504638672, + "rewards/rejected": -12.346035957336426, + "step": 2055 + }, + { + "epoch": 2.69, + "learning_rate": 4.214705914535361e-05, + "logits/chosen": -2.2096025943756104, + "logits/rejected": -2.250379800796509, + "logps/chosen": -270.6283264160156, + "logps/rejected": -394.0135803222656, + "loss": 0.0497, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.837217807769775, + "rewards/margins": 8.825190544128418, + "rewards/rejected": -13.662408828735352, + "step": 2056 + }, + { + "epoch": 2.69, + "learning_rate": 4.213947749795593e-05, + "logits/chosen": -2.2673234939575195, + "logits/rejected": -2.196312665939331, + "logps/chosen": -227.28076171875, + "logps/rejected": -277.54034423828125, + "loss": 0.0913, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.908783435821533, + "rewards/margins": 6.475468158721924, + "rewards/rejected": -11.38425064086914, + "step": 2057 + }, + { + "epoch": 2.69, + "learning_rate": 4.2131892875094756e-05, + "logits/chosen": -2.2475051879882812, + "logits/rejected": -2.249516010284424, + "logps/chosen": -254.70135498046875, + "logps/rejected": -320.4151306152344, + "loss": 0.0451, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.25231409072876, + "rewards/margins": 6.865804195404053, + "rewards/rejected": -12.118119239807129, + "step": 2058 + }, + { + "epoch": 2.69, + "learning_rate": 4.212430527808682e-05, + "logits/chosen": -2.326888084411621, + "logits/rejected": -2.471912384033203, + "logps/chosen": -258.8665466308594, + "logps/rejected": -347.352294921875, + "loss": 0.0453, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.683292388916016, + "rewards/margins": 7.736345291137695, + "rewards/rejected": -12.419637680053711, + "step": 2059 + }, + { + "epoch": 2.7, + "learning_rate": 4.211671470824933e-05, + "logits/chosen": -2.1365933418273926, + "logits/rejected": -2.1905436515808105, + "logps/chosen": -235.756103515625, + "logps/rejected": -328.03961181640625, + "loss": 0.1719, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.880552291870117, + "rewards/margins": 7.123145580291748, + "rewards/rejected": -12.003698348999023, + "step": 2060 + }, + { + "epoch": 2.7, + "learning_rate": 4.210912116690004e-05, + "logits/chosen": -2.2770068645477295, + "logits/rejected": -2.2150559425354004, + "logps/chosen": -282.68084716796875, + "logps/rejected": -355.66058349609375, + "loss": 0.0005, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.084074020385742, + "rewards/margins": 10.210107803344727, + "rewards/rejected": -14.294180870056152, + "step": 2061 + }, + { + "epoch": 2.7, + "learning_rate": 4.210152465535721e-05, + "logits/chosen": -1.9398760795593262, + "logits/rejected": -1.9859808683395386, + "logps/chosen": -252.51089477539062, + "logps/rejected": -308.23846435546875, + "loss": 0.0735, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.215598106384277, + "rewards/margins": 7.410407543182373, + "rewards/rejected": -11.626004219055176, + "step": 2062 + }, + { + "epoch": 2.7, + "learning_rate": 4.209392517493961e-05, + "logits/chosen": -2.078042984008789, + "logits/rejected": -2.0762107372283936, + "logps/chosen": -200.397216796875, + "logps/rejected": -326.5500183105469, + "loss": 0.0443, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.259744644165039, + "rewards/margins": 8.913934707641602, + "rewards/rejected": -13.173680305480957, + "step": 2063 + }, + { + "epoch": 2.7, + "learning_rate": 4.2086322726966546e-05, + "logits/chosen": -2.240522861480713, + "logits/rejected": -2.254432201385498, + "logps/chosen": -209.32215881347656, + "logps/rejected": -310.88641357421875, + "loss": 0.0067, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.953611373901367, + "rewards/margins": 9.082300186157227, + "rewards/rejected": -13.035911560058594, + "step": 2064 + }, + { + "epoch": 2.7, + "learning_rate": 4.207871731275781e-05, + "logits/chosen": -2.1679046154022217, + "logits/rejected": -2.14817476272583, + "logps/chosen": -257.8780517578125, + "logps/rejected": -345.7042541503906, + "loss": 0.0065, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.281774520874023, + "rewards/margins": 8.699878692626953, + "rewards/rejected": -12.981653213500977, + "step": 2065 + }, + { + "epoch": 2.7, + "learning_rate": 4.207110893363373e-05, + "logits/chosen": -2.295118808746338, + "logits/rejected": -2.2843916416168213, + "logps/chosen": -238.00244140625, + "logps/rejected": -338.3034362792969, + "loss": 0.0455, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.327823162078857, + "rewards/margins": 8.419173240661621, + "rewards/rejected": -12.74699592590332, + "step": 2066 + }, + { + "epoch": 2.71, + "learning_rate": 4.206349759091515e-05, + "logits/chosen": -1.808960199356079, + "logits/rejected": -1.8792856931686401, + "logps/chosen": -262.61285400390625, + "logps/rejected": -356.2742614746094, + "loss": 0.0874, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.9227986335754395, + "rewards/margins": 7.593912124633789, + "rewards/rejected": -13.516711235046387, + "step": 2067 + }, + { + "epoch": 2.71, + "learning_rate": 4.205588328592341e-05, + "logits/chosen": -2.258066177368164, + "logits/rejected": -2.3306429386138916, + "logps/chosen": -232.1697998046875, + "logps/rejected": -367.2823181152344, + "loss": 0.0439, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.021013259887695, + "rewards/margins": 10.31048583984375, + "rewards/rejected": -15.331500053405762, + "step": 2068 + }, + { + "epoch": 2.71, + "learning_rate": 4.2048266019980385e-05, + "logits/chosen": -2.03212833404541, + "logits/rejected": -1.9772014617919922, + "logps/chosen": -242.79864501953125, + "logps/rejected": -329.5115051269531, + "loss": 0.0578, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.13314151763916, + "rewards/margins": 8.251870155334473, + "rewards/rejected": -13.385011672973633, + "step": 2069 + }, + { + "epoch": 2.71, + "learning_rate": 4.2040645794408445e-05, + "logits/chosen": -2.1458396911621094, + "logits/rejected": -2.4645326137542725, + "logps/chosen": -261.87310791015625, + "logps/rejected": -394.9166259765625, + "loss": 0.0015, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.958534240722656, + "rewards/margins": 8.967354774475098, + "rewards/rejected": -13.92588996887207, + "step": 2070 + }, + { + "epoch": 2.71, + "learning_rate": 4.2033022610530495e-05, + "logits/chosen": -2.5229756832122803, + "logits/rejected": -2.5427346229553223, + "logps/chosen": -283.5986022949219, + "logps/rejected": -337.8445129394531, + "loss": 0.0473, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.727269172668457, + "rewards/margins": 7.151669025421143, + "rewards/rejected": -12.878936767578125, + "step": 2071 + }, + { + "epoch": 2.71, + "learning_rate": 4.202539646966993e-05, + "logits/chosen": -2.301621675491333, + "logits/rejected": -2.393721103668213, + "logps/chosen": -258.2679748535156, + "logps/rejected": -371.85040283203125, + "loss": 0.062, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.82385778427124, + "rewards/margins": 8.38097095489502, + "rewards/rejected": -13.204830169677734, + "step": 2072 + }, + { + "epoch": 2.71, + "learning_rate": 4.201776737315069e-05, + "logits/chosen": -1.9535908699035645, + "logits/rejected": -1.902817726135254, + "logps/chosen": -195.61341857910156, + "logps/rejected": -286.3548889160156, + "loss": 0.0976, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.179656505584717, + "rewards/margins": 7.419997215270996, + "rewards/rejected": -11.599653244018555, + "step": 2073 + }, + { + "epoch": 2.71, + "learning_rate": 4.201013532229717e-05, + "logits/chosen": -2.2434518337249756, + "logits/rejected": -2.2938058376312256, + "logps/chosen": -273.43939208984375, + "logps/rejected": -355.3121643066406, + "loss": 0.0015, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.7452874183654785, + "rewards/margins": 8.803860664367676, + "rewards/rejected": -13.549148559570312, + "step": 2074 + }, + { + "epoch": 2.72, + "learning_rate": 4.200250031843436e-05, + "logits/chosen": -2.147690773010254, + "logits/rejected": -2.103121280670166, + "logps/chosen": -252.52203369140625, + "logps/rejected": -345.4867248535156, + "loss": 0.0054, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.055788516998291, + "rewards/margins": 8.033418655395508, + "rewards/rejected": -13.08920669555664, + "step": 2075 + }, + { + "epoch": 2.72, + "learning_rate": 4.19948623628877e-05, + "logits/chosen": -2.1535098552703857, + "logits/rejected": -2.178994655609131, + "logps/chosen": -201.6510009765625, + "logps/rejected": -293.91864013671875, + "loss": 0.0877, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.277956962585449, + "rewards/margins": 8.379180908203125, + "rewards/rejected": -12.657137870788574, + "step": 2076 + }, + { + "epoch": 2.72, + "learning_rate": 4.198722145698316e-05, + "logits/chosen": -2.069924831390381, + "logits/rejected": -2.08437180519104, + "logps/chosen": -180.403564453125, + "logps/rejected": -288.7211608886719, + "loss": 0.0462, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.470130920410156, + "rewards/margins": 9.821898460388184, + "rewards/rejected": -14.29202938079834, + "step": 2077 + }, + { + "epoch": 2.72, + "learning_rate": 4.1979577602047225e-05, + "logits/chosen": -1.9232547283172607, + "logits/rejected": -1.9797210693359375, + "logps/chosen": -221.45703125, + "logps/rejected": -298.70947265625, + "loss": 0.1112, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.912723064422607, + "rewards/margins": 6.433436870574951, + "rewards/rejected": -11.346160888671875, + "step": 2078 + }, + { + "epoch": 2.72, + "learning_rate": 4.1971930799406886e-05, + "logits/chosen": -1.878727674484253, + "logits/rejected": -1.8984040021896362, + "logps/chosen": -220.32061767578125, + "logps/rejected": -301.4939880371094, + "loss": 0.3477, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.683475494384766, + "rewards/margins": 8.25886344909668, + "rewards/rejected": -12.942338943481445, + "step": 2079 + }, + { + "epoch": 2.72, + "learning_rate": 4.196428105038966e-05, + "logits/chosen": -2.0797531604766846, + "logits/rejected": -2.1228551864624023, + "logps/chosen": -267.1173400878906, + "logps/rejected": -339.40972900390625, + "loss": 0.0006, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.964832782745361, + "rewards/margins": 8.00648307800293, + "rewards/rejected": -12.971315383911133, + "step": 2080 + }, + { + "epoch": 2.72, + "learning_rate": 4.195662835632356e-05, + "logits/chosen": -2.11684513092041, + "logits/rejected": -2.075150728225708, + "logps/chosen": -258.8525390625, + "logps/rejected": -323.33953857421875, + "loss": 0.1371, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.296041488647461, + "rewards/margins": 6.962072849273682, + "rewards/rejected": -12.2581148147583, + "step": 2081 + }, + { + "epoch": 2.72, + "learning_rate": 4.194897271853712e-05, + "logits/chosen": -2.0427944660186768, + "logits/rejected": -2.0676662921905518, + "logps/chosen": -270.5080871582031, + "logps/rejected": -346.3038635253906, + "loss": 0.0945, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.30461311340332, + "rewards/margins": 7.249161720275879, + "rewards/rejected": -12.553773880004883, + "step": 2082 + }, + { + "epoch": 2.73, + "learning_rate": 4.194131413835937e-05, + "logits/chosen": -2.061204433441162, + "logits/rejected": -2.257317543029785, + "logps/chosen": -209.44093322753906, + "logps/rejected": -327.5050048828125, + "loss": 0.0033, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.970865249633789, + "rewards/margins": 8.313035011291504, + "rewards/rejected": -14.283899307250977, + "step": 2083 + }, + { + "epoch": 2.73, + "learning_rate": 4.1933652617119884e-05, + "logits/chosen": -2.4756479263305664, + "logits/rejected": -2.410625457763672, + "logps/chosen": -236.7713165283203, + "logps/rejected": -275.470947265625, + "loss": 0.1362, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.773379325866699, + "rewards/margins": 5.622978210449219, + "rewards/rejected": -10.396357536315918, + "step": 2084 + }, + { + "epoch": 2.73, + "learning_rate": 4.19259881561487e-05, + "logits/chosen": -2.2148780822753906, + "logits/rejected": -2.1019504070281982, + "logps/chosen": -210.04232788085938, + "logps/rejected": -298.0214538574219, + "loss": 0.0892, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.023033142089844, + "rewards/margins": 8.117776870727539, + "rewards/rejected": -13.140810012817383, + "step": 2085 + }, + { + "epoch": 2.73, + "learning_rate": 4.1918320756776406e-05, + "logits/chosen": -2.0880420207977295, + "logits/rejected": -2.0963172912597656, + "logps/chosen": -234.06309509277344, + "logps/rejected": -323.31591796875, + "loss": 0.0448, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.505844593048096, + "rewards/margins": 8.520071029663086, + "rewards/rejected": -13.025914192199707, + "step": 2086 + }, + { + "epoch": 2.73, + "learning_rate": 4.1910650420334084e-05, + "logits/chosen": -2.298971176147461, + "logits/rejected": -2.3134219646453857, + "logps/chosen": -253.9832763671875, + "logps/rejected": -341.1661376953125, + "loss": 0.0957, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.26953125, + "rewards/margins": 7.074673652648926, + "rewards/rejected": -13.344204902648926, + "step": 2087 + }, + { + "epoch": 2.73, + "learning_rate": 4.190297714815332e-05, + "logits/chosen": -2.1651523113250732, + "logits/rejected": -2.3193411827087402, + "logps/chosen": -243.05581665039062, + "logps/rejected": -396.4129333496094, + "loss": 0.003, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.518342018127441, + "rewards/margins": 10.284740447998047, + "rewards/rejected": -15.803083419799805, + "step": 2088 + }, + { + "epoch": 2.73, + "learning_rate": 4.189530094156622e-05, + "logits/chosen": -2.185380220413208, + "logits/rejected": -2.256591320037842, + "logps/chosen": -275.3895568847656, + "logps/rejected": -398.8544616699219, + "loss": 0.0008, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.499395847320557, + "rewards/margins": 9.482010841369629, + "rewards/rejected": -14.981407165527344, + "step": 2089 + }, + { + "epoch": 2.74, + "learning_rate": 4.1887621801905396e-05, + "logits/chosen": -2.2365474700927734, + "logits/rejected": -2.258640766143799, + "logps/chosen": -266.8921813964844, + "logps/rejected": -392.3157653808594, + "loss": 0.0005, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.301426887512207, + "rewards/margins": 9.61983871459961, + "rewards/rejected": -14.9212646484375, + "step": 2090 + }, + { + "epoch": 2.74, + "learning_rate": 4.187993973050397e-05, + "logits/chosen": -2.1382174491882324, + "logits/rejected": -2.1736865043640137, + "logps/chosen": -223.3988037109375, + "logps/rejected": -318.99798583984375, + "loss": 0.0911, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.835805416107178, + "rewards/margins": 8.29453182220459, + "rewards/rejected": -14.130338668823242, + "step": 2091 + }, + { + "epoch": 2.74, + "learning_rate": 4.187225472869557e-05, + "logits/chosen": -1.7872908115386963, + "logits/rejected": -1.9052330255508423, + "logps/chosen": -248.73348999023438, + "logps/rejected": -378.89990234375, + "loss": 0.0541, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.4027419090271, + "rewards/margins": 8.380805969238281, + "rewards/rejected": -13.783547401428223, + "step": 2092 + }, + { + "epoch": 2.74, + "learning_rate": 4.186456679781434e-05, + "logits/chosen": -2.0523717403411865, + "logits/rejected": -2.1558446884155273, + "logps/chosen": -246.02601623535156, + "logps/rejected": -373.6942138671875, + "loss": 0.0447, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.165873050689697, + "rewards/margins": 8.869769096374512, + "rewards/rejected": -14.035642623901367, + "step": 2093 + }, + { + "epoch": 2.74, + "learning_rate": 4.1856875939194917e-05, + "logits/chosen": -2.036025047302246, + "logits/rejected": -2.0010640621185303, + "logps/chosen": -219.66464233398438, + "logps/rejected": -335.1212463378906, + "loss": 0.0489, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.9590160846710205, + "rewards/margins": 8.931244850158691, + "rewards/rejected": -12.890260696411133, + "step": 2094 + }, + { + "epoch": 2.74, + "learning_rate": 4.1849182154172475e-05, + "logits/chosen": -2.077622890472412, + "logits/rejected": -2.213207483291626, + "logps/chosen": -234.46327209472656, + "logps/rejected": -375.966552734375, + "loss": 0.0858, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.090786457061768, + "rewards/margins": 9.148885726928711, + "rewards/rejected": -15.239669799804688, + "step": 2095 + }, + { + "epoch": 2.74, + "learning_rate": 4.184148544408265e-05, + "logits/chosen": -1.9233466386795044, + "logits/rejected": -1.9716625213623047, + "logps/chosen": -260.3994445800781, + "logps/rejected": -336.7559509277344, + "loss": 0.0941, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.399205684661865, + "rewards/margins": 6.785099029541016, + "rewards/rejected": -12.184305191040039, + "step": 2096 + }, + { + "epoch": 2.74, + "learning_rate": 4.183378581026164e-05, + "logits/chosen": -1.9934498071670532, + "logits/rejected": -2.0427863597869873, + "logps/chosen": -233.28619384765625, + "logps/rejected": -308.950439453125, + "loss": 0.0463, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.482067108154297, + "rewards/margins": 7.9834465980529785, + "rewards/rejected": -13.4655122756958, + "step": 2097 + }, + { + "epoch": 2.75, + "learning_rate": 4.182608325404611e-05, + "logits/chosen": -2.2864651679992676, + "logits/rejected": -2.3209176063537598, + "logps/chosen": -228.93246459960938, + "logps/rejected": -322.6388244628906, + "loss": 0.0869, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.583909034729004, + "rewards/margins": 9.239480972290039, + "rewards/rejected": -12.823390007019043, + "step": 2098 + }, + { + "epoch": 2.75, + "learning_rate": 4.1818377776773255e-05, + "logits/chosen": -2.0176432132720947, + "logits/rejected": -2.086442232131958, + "logps/chosen": -198.83099365234375, + "logps/rejected": -302.553466796875, + "loss": 0.0441, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.565010070800781, + "rewards/margins": 8.124502182006836, + "rewards/rejected": -13.689510345458984, + "step": 2099 + }, + { + "epoch": 2.75, + "learning_rate": 4.181066937978075e-05, + "logits/chosen": -2.3962371349334717, + "logits/rejected": -2.3214006423950195, + "logps/chosen": -244.8721923828125, + "logps/rejected": -286.8533935546875, + "loss": 0.2173, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.8245086669921875, + "rewards/margins": 6.910382270812988, + "rewards/rejected": -10.734890937805176, + "step": 2100 + }, + { + "epoch": 2.75, + "learning_rate": 4.180295806440682e-05, + "logits/chosen": -1.9995003938674927, + "logits/rejected": -2.089867115020752, + "logps/chosen": -198.92347717285156, + "logps/rejected": -302.6363830566406, + "loss": 0.0461, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.106322765350342, + "rewards/margins": 7.756969928741455, + "rewards/rejected": -12.863292694091797, + "step": 2101 + }, + { + "epoch": 2.75, + "learning_rate": 4.179524383199016e-05, + "logits/chosen": -1.8776270151138306, + "logits/rejected": -2.059648036956787, + "logps/chosen": -245.83056640625, + "logps/rejected": -301.4813537597656, + "loss": 0.0485, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.165894031524658, + "rewards/margins": 7.827360153198242, + "rewards/rejected": -12.993253707885742, + "step": 2102 + }, + { + "epoch": 2.75, + "learning_rate": 4.178752668386997e-05, + "logits/chosen": -2.220716953277588, + "logits/rejected": -2.236607313156128, + "logps/chosen": -242.704833984375, + "logps/rejected": -326.8563232421875, + "loss": 0.0522, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.967065811157227, + "rewards/margins": 7.383824348449707, + "rewards/rejected": -13.350890159606934, + "step": 2103 + }, + { + "epoch": 2.75, + "learning_rate": 4.1779806621386e-05, + "logits/chosen": -1.9287221431732178, + "logits/rejected": -1.9850231409072876, + "logps/chosen": -232.6581268310547, + "logps/rejected": -301.498046875, + "loss": 0.0957, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.314054489135742, + "rewards/margins": 6.8057403564453125, + "rewards/rejected": -12.119794845581055, + "step": 2104 + }, + { + "epoch": 2.75, + "learning_rate": 4.1772083645878444e-05, + "logits/chosen": -2.194317579269409, + "logits/rejected": -2.2162156105041504, + "logps/chosen": -189.45413208007812, + "logps/rejected": -277.4154052734375, + "loss": 0.0177, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.497744560241699, + "rewards/margins": 7.991286277770996, + "rewards/rejected": -12.489030838012695, + "step": 2105 + }, + { + "epoch": 2.76, + "learning_rate": 4.176435775868805e-05, + "logits/chosen": -2.1279244422912598, + "logits/rejected": -2.165595769882202, + "logps/chosen": -273.2611999511719, + "logps/rejected": -371.8103332519531, + "loss": 0.1316, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.4172234535217285, + "rewards/margins": 8.07325553894043, + "rewards/rejected": -12.490479469299316, + "step": 2106 + }, + { + "epoch": 2.76, + "learning_rate": 4.1756628961156064e-05, + "logits/chosen": -1.984653115272522, + "logits/rejected": -2.1006929874420166, + "logps/chosen": -293.9515686035156, + "logps/rejected": -393.9189453125, + "loss": 0.0439, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.42185115814209, + "rewards/margins": 8.856366157531738, + "rewards/rejected": -15.278217315673828, + "step": 2107 + }, + { + "epoch": 2.76, + "learning_rate": 4.174889725462421e-05, + "logits/chosen": -2.2485594749450684, + "logits/rejected": -2.168731451034546, + "logps/chosen": -238.37648010253906, + "logps/rejected": -324.60223388671875, + "loss": 0.002, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.376681804656982, + "rewards/margins": 9.401522636413574, + "rewards/rejected": -14.778203964233398, + "step": 2108 + }, + { + "epoch": 2.76, + "learning_rate": 4.1741162640434736e-05, + "logits/chosen": -1.9732999801635742, + "logits/rejected": -2.150045871734619, + "logps/chosen": -243.09902954101562, + "logps/rejected": -379.98779296875, + "loss": 0.0034, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.749682903289795, + "rewards/margins": 9.43840217590332, + "rewards/rejected": -15.188084602355957, + "step": 2109 + }, + { + "epoch": 2.76, + "learning_rate": 4.17334251199304e-05, + "logits/chosen": -2.0670886039733887, + "logits/rejected": -1.9970879554748535, + "logps/chosen": -212.93011474609375, + "logps/rejected": -290.45220947265625, + "loss": 0.0594, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.849340438842773, + "rewards/margins": 8.22266960144043, + "rewards/rejected": -13.072010040283203, + "step": 2110 + }, + { + "epoch": 2.76, + "learning_rate": 4.172568469445446e-05, + "logits/chosen": -2.146941661834717, + "logits/rejected": -2.128082036972046, + "logps/chosen": -288.23114013671875, + "logps/rejected": -340.2186584472656, + "loss": 0.0008, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.491695404052734, + "rewards/margins": 8.292865753173828, + "rewards/rejected": -14.784561157226562, + "step": 2111 + }, + { + "epoch": 2.76, + "learning_rate": 4.171794136535067e-05, + "logits/chosen": -2.09254789352417, + "logits/rejected": -2.106165885925293, + "logps/chosen": -246.88629150390625, + "logps/rejected": -331.7405700683594, + "loss": 0.049, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.748941898345947, + "rewards/margins": 8.975343704223633, + "rewards/rejected": -14.724285125732422, + "step": 2112 + }, + { + "epoch": 2.77, + "learning_rate": 4.17101951339633e-05, + "logits/chosen": -1.9903786182403564, + "logits/rejected": -1.9985647201538086, + "logps/chosen": -216.80055236816406, + "logps/rejected": -314.84552001953125, + "loss": 0.0941, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.6483473777771, + "rewards/margins": 7.770323276519775, + "rewards/rejected": -12.418669700622559, + "step": 2113 + }, + { + "epoch": 2.77, + "learning_rate": 4.170244600163712e-05, + "logits/chosen": -2.100982904434204, + "logits/rejected": -2.082028865814209, + "logps/chosen": -215.8398895263672, + "logps/rejected": -252.7769317626953, + "loss": 0.0464, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.179358959197998, + "rewards/margins": 7.598566055297852, + "rewards/rejected": -11.777924537658691, + "step": 2114 + }, + { + "epoch": 2.77, + "learning_rate": 4.169469396971739e-05, + "logits/chosen": -2.2202951908111572, + "logits/rejected": -2.112492084503174, + "logps/chosen": -271.6481018066406, + "logps/rejected": -403.6411437988281, + "loss": 0.0063, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.043911933898926, + "rewards/margins": 8.47602367401123, + "rewards/rejected": -14.519936561584473, + "step": 2115 + }, + { + "epoch": 2.77, + "learning_rate": 4.16869390395499e-05, + "logits/chosen": -2.1215877532958984, + "logits/rejected": -2.1512339115142822, + "logps/chosen": -226.36605834960938, + "logps/rejected": -314.9847412109375, + "loss": 0.0776, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.515926837921143, + "rewards/margins": 7.796207904815674, + "rewards/rejected": -13.3121337890625, + "step": 2116 + }, + { + "epoch": 2.77, + "learning_rate": 4.167918121248092e-05, + "logits/chosen": -2.2617945671081543, + "logits/rejected": -2.2887606620788574, + "logps/chosen": -289.6715087890625, + "logps/rejected": -347.873291015625, + "loss": 0.0192, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.545827865600586, + "rewards/margins": 9.474298477172852, + "rewards/rejected": -16.020126342773438, + "step": 2117 + }, + { + "epoch": 2.77, + "learning_rate": 4.1671420489857236e-05, + "logits/chosen": -2.0493292808532715, + "logits/rejected": -2.0911078453063965, + "logps/chosen": -249.25364685058594, + "logps/rejected": -380.7524719238281, + "loss": 0.0518, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.965695858001709, + "rewards/margins": 8.083683013916016, + "rewards/rejected": -15.049379348754883, + "step": 2118 + }, + { + "epoch": 2.77, + "learning_rate": 4.166365687302614e-05, + "logits/chosen": -1.9570484161376953, + "logits/rejected": -1.8582700490951538, + "logps/chosen": -211.3433074951172, + "logps/rejected": -304.293701171875, + "loss": 0.0492, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.687077045440674, + "rewards/margins": 8.289334297180176, + "rewards/rejected": -13.976410865783691, + "step": 2119 + }, + { + "epoch": 2.77, + "learning_rate": 4.1655890363335394e-05, + "logits/chosen": -2.077530860900879, + "logits/rejected": -2.1311354637145996, + "logps/chosen": -252.41888427734375, + "logps/rejected": -354.2386474609375, + "loss": 0.0439, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.369992256164551, + "rewards/margins": 10.087926864624023, + "rewards/rejected": -16.457918167114258, + "step": 2120 + }, + { + "epoch": 2.78, + "learning_rate": 4.164812096213331e-05, + "logits/chosen": -1.9602264165878296, + "logits/rejected": -2.0098724365234375, + "logps/chosen": -237.26214599609375, + "logps/rejected": -300.9123840332031, + "loss": 0.0886, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.2542524337768555, + "rewards/margins": 6.468628406524658, + "rewards/rejected": -12.722880363464355, + "step": 2121 + }, + { + "epoch": 2.78, + "learning_rate": 4.164034867076866e-05, + "logits/chosen": -2.0425612926483154, + "logits/rejected": -2.056826114654541, + "logps/chosen": -228.2157745361328, + "logps/rejected": -342.6788024902344, + "loss": 0.0438, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.24842643737793, + "rewards/margins": 10.305375099182129, + "rewards/rejected": -17.553800582885742, + "step": 2122 + }, + { + "epoch": 2.78, + "learning_rate": 4.1632573490590754e-05, + "logits/chosen": -1.9484583139419556, + "logits/rejected": -1.9971625804901123, + "logps/chosen": -205.79505920410156, + "logps/rejected": -284.4195556640625, + "loss": 0.1798, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.692427158355713, + "rewards/margins": 8.066761016845703, + "rewards/rejected": -13.759187698364258, + "step": 2123 + }, + { + "epoch": 2.78, + "learning_rate": 4.162479542294938e-05, + "logits/chosen": -2.029684543609619, + "logits/rejected": -2.0470805168151855, + "logps/chosen": -287.4469909667969, + "logps/rejected": -360.28680419921875, + "loss": 0.0261, + "rewards/accuracies": 1.0, + "rewards/chosen": -8.992703437805176, + "rewards/margins": 8.311714172363281, + "rewards/rejected": -17.30441665649414, + "step": 2124 + }, + { + "epoch": 2.78, + "learning_rate": 4.1617014469194836e-05, + "logits/chosen": -1.9437496662139893, + "logits/rejected": -1.940740942955017, + "logps/chosen": -280.3132019042969, + "logps/rejected": -343.2749328613281, + "loss": 0.2108, + "rewards/accuracies": 0.9375, + "rewards/chosen": -7.682542324066162, + "rewards/margins": 8.812956809997559, + "rewards/rejected": -16.495498657226562, + "step": 2125 + }, + { + "epoch": 2.78, + "learning_rate": 4.160923063067791e-05, + "logits/chosen": -1.6997432708740234, + "logits/rejected": -1.8715956211090088, + "logps/chosen": -235.1721649169922, + "logps/rejected": -387.16510009765625, + "loss": 0.1111, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.875338554382324, + "rewards/margins": 9.192512512207031, + "rewards/rejected": -16.067852020263672, + "step": 2126 + }, + { + "epoch": 2.78, + "learning_rate": 4.1601443908749904e-05, + "logits/chosen": -1.8663256168365479, + "logits/rejected": -1.9868435859680176, + "logps/chosen": -274.02374267578125, + "logps/rejected": -363.5721435546875, + "loss": 0.0043, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.386130332946777, + "rewards/margins": 8.717323303222656, + "rewards/rejected": -15.103453636169434, + "step": 2127 + }, + { + "epoch": 2.78, + "learning_rate": 4.159365430476261e-05, + "logits/chosen": -1.8189918994903564, + "logits/rejected": -1.8256369829177856, + "logps/chosen": -236.00656127929688, + "logps/rejected": -336.5587158203125, + "loss": 0.0442, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.672895431518555, + "rewards/margins": 9.486919403076172, + "rewards/rejected": -15.15981388092041, + "step": 2128 + }, + { + "epoch": 2.79, + "learning_rate": 4.158586182006834e-05, + "logits/chosen": -1.9643597602844238, + "logits/rejected": -1.9393858909606934, + "logps/chosen": -276.75238037109375, + "logps/rejected": -361.45660400390625, + "loss": 0.0755, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.428586959838867, + "rewards/margins": 9.840063095092773, + "rewards/rejected": -16.26865005493164, + "step": 2129 + }, + { + "epoch": 2.79, + "learning_rate": 4.157806645601988e-05, + "logits/chosen": -2.0720882415771484, + "logits/rejected": -2.129720449447632, + "logps/chosen": -305.78948974609375, + "logps/rejected": -402.7105712890625, + "loss": 0.1377, + "rewards/accuracies": 0.9375, + "rewards/chosen": -7.76812744140625, + "rewards/margins": 6.746726989746094, + "rewards/rejected": -14.514853477478027, + "step": 2130 + }, + { + "epoch": 2.79, + "learning_rate": 4.1570268213970534e-05, + "logits/chosen": -1.9224815368652344, + "logits/rejected": -1.9703435897827148, + "logps/chosen": -272.35504150390625, + "logps/rejected": -383.93963623046875, + "loss": 0.0487, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.817448616027832, + "rewards/margins": 9.154064178466797, + "rewards/rejected": -15.971511840820312, + "step": 2131 + }, + { + "epoch": 2.79, + "learning_rate": 4.15624670952741e-05, + "logits/chosen": -2.1934380531311035, + "logits/rejected": -2.250469923019409, + "logps/chosen": -285.520263671875, + "logps/rejected": -397.2008361816406, + "loss": 0.1347, + "rewards/accuracies": 0.8125, + "rewards/chosen": -7.1292219161987305, + "rewards/margins": 8.519986152648926, + "rewards/rejected": -15.649208068847656, + "step": 2132 + }, + { + "epoch": 2.79, + "learning_rate": 4.155466310128487e-05, + "logits/chosen": -2.1712989807128906, + "logits/rejected": -2.1715948581695557, + "logps/chosen": -237.79794311523438, + "logps/rejected": -321.9048767089844, + "loss": 0.0453, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.765093803405762, + "rewards/margins": 8.670004844665527, + "rewards/rejected": -14.435096740722656, + "step": 2133 + }, + { + "epoch": 2.79, + "learning_rate": 4.1546856233357645e-05, + "logits/chosen": -2.097083806991577, + "logits/rejected": -2.052821397781372, + "logps/chosen": -240.02984619140625, + "logps/rejected": -311.79925537109375, + "loss": 0.0614, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.495797634124756, + "rewards/margins": 7.645787239074707, + "rewards/rejected": -14.141584396362305, + "step": 2134 + }, + { + "epoch": 2.79, + "learning_rate": 4.153904649284771e-05, + "logits/chosen": -2.127629518508911, + "logits/rejected": -2.191781520843506, + "logps/chosen": -333.68499755859375, + "logps/rejected": -373.86328125, + "loss": 0.048, + "rewards/accuracies": 0.9375, + "rewards/chosen": -7.099422931671143, + "rewards/margins": 8.105523109436035, + "rewards/rejected": -15.204946517944336, + "step": 2135 + }, + { + "epoch": 2.8, + "learning_rate": 4.1531233881110875e-05, + "logits/chosen": -1.9232991933822632, + "logits/rejected": -2.016394853591919, + "logps/chosen": -231.78211975097656, + "logps/rejected": -310.239501953125, + "loss": 0.0507, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.600955963134766, + "rewards/margins": 8.300896644592285, + "rewards/rejected": -13.901851654052734, + "step": 2136 + }, + { + "epoch": 2.8, + "learning_rate": 4.152341839950342e-05, + "logits/chosen": -1.766890048980713, + "logits/rejected": -1.7369945049285889, + "logps/chosen": -266.9042053222656, + "logps/rejected": -396.30291748046875, + "loss": 0.2204, + "rewards/accuracies": 0.875, + "rewards/chosen": -7.4335408210754395, + "rewards/margins": 9.401388168334961, + "rewards/rejected": -16.834928512573242, + "step": 2137 + }, + { + "epoch": 2.8, + "learning_rate": 4.151560004938214e-05, + "logits/chosen": -1.7209428548812866, + "logits/rejected": -1.8653645515441895, + "logps/chosen": -246.06756591796875, + "logps/rejected": -348.609130859375, + "loss": 0.133, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.485172271728516, + "rewards/margins": 8.861818313598633, + "rewards/rejected": -14.346990585327148, + "step": 2138 + }, + { + "epoch": 2.8, + "learning_rate": 4.1507778832104326e-05, + "logits/chosen": -2.0044240951538086, + "logits/rejected": -2.0473899841308594, + "logps/chosen": -289.8185729980469, + "logps/rejected": -395.8297424316406, + "loss": 0.0457, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.868648052215576, + "rewards/margins": 9.16476058959961, + "rewards/rejected": -16.033409118652344, + "step": 2139 + }, + { + "epoch": 2.8, + "learning_rate": 4.149995474902776e-05, + "logits/chosen": -2.0526278018951416, + "logits/rejected": -2.1411993503570557, + "logps/chosen": -212.82223510742188, + "logps/rejected": -308.5693359375, + "loss": 0.0115, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.235255718231201, + "rewards/margins": 8.20628547668457, + "rewards/rejected": -13.441540718078613, + "step": 2140 + }, + { + "epoch": 2.8, + "learning_rate": 4.149212780151073e-05, + "logits/chosen": -2.014158248901367, + "logits/rejected": -2.1073756217956543, + "logps/chosen": -211.17178344726562, + "logps/rejected": -347.7945251464844, + "loss": 0.0013, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.059771537780762, + "rewards/margins": 10.15962028503418, + "rewards/rejected": -16.219392776489258, + "step": 2141 + }, + { + "epoch": 2.8, + "learning_rate": 4.1484297990912e-05, + "logits/chosen": -2.045736312866211, + "logits/rejected": -2.090200424194336, + "logps/chosen": -269.7496337890625, + "logps/rejected": -362.4156799316406, + "loss": 0.0069, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.597815990447998, + "rewards/margins": 7.994004726409912, + "rewards/rejected": -15.591819763183594, + "step": 2142 + }, + { + "epoch": 2.8, + "learning_rate": 4.1476465318590876e-05, + "logits/chosen": -2.0298562049865723, + "logits/rejected": -2.122493267059326, + "logps/chosen": -218.05921936035156, + "logps/rejected": -343.3038635253906, + "loss": 0.0487, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.693921089172363, + "rewards/margins": 8.963953018188477, + "rewards/rejected": -14.657873153686523, + "step": 2143 + }, + { + "epoch": 2.81, + "learning_rate": 4.146862978590712e-05, + "logits/chosen": -1.8342090845108032, + "logits/rejected": -1.959254503250122, + "logps/chosen": -264.7921142578125, + "logps/rejected": -354.5204772949219, + "loss": 0.0889, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.0057196617126465, + "rewards/margins": 8.22575855255127, + "rewards/rejected": -14.231477737426758, + "step": 2144 + }, + { + "epoch": 2.81, + "learning_rate": 4.1460791394221e-05, + "logits/chosen": -1.7905089855194092, + "logits/rejected": -1.8561276197433472, + "logps/chosen": -291.75823974609375, + "logps/rejected": -356.01495361328125, + "loss": 0.2051, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.946643352508545, + "rewards/margins": 6.129225730895996, + "rewards/rejected": -12.0758695602417, + "step": 2145 + }, + { + "epoch": 2.81, + "learning_rate": 4.1452950144893274e-05, + "logits/chosen": -2.0241174697875977, + "logits/rejected": -2.05717396736145, + "logps/chosen": -253.92105102539062, + "logps/rejected": -348.9212341308594, + "loss": 0.0081, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.968001365661621, + "rewards/margins": 9.411520957946777, + "rewards/rejected": -16.3795223236084, + "step": 2146 + }, + { + "epoch": 2.81, + "learning_rate": 4.144510603928523e-05, + "logits/chosen": -1.9531909227371216, + "logits/rejected": -2.044700860977173, + "logps/chosen": -290.6370544433594, + "logps/rejected": -353.8155517578125, + "loss": 0.0503, + "rewards/accuracies": 0.9375, + "rewards/chosen": -7.445758819580078, + "rewards/margins": 7.122509479522705, + "rewards/rejected": -14.568268775939941, + "step": 2147 + }, + { + "epoch": 2.81, + "learning_rate": 4.143725907875862e-05, + "logits/chosen": -1.7878367900848389, + "logits/rejected": -1.8614583015441895, + "logps/chosen": -290.497314453125, + "logps/rejected": -393.19500732421875, + "loss": 0.0872, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.828826904296875, + "rewards/margins": 8.184590339660645, + "rewards/rejected": -14.013418197631836, + "step": 2148 + }, + { + "epoch": 2.81, + "learning_rate": 4.142940926467568e-05, + "logits/chosen": -1.9906575679779053, + "logits/rejected": -1.9775463342666626, + "logps/chosen": -212.44091796875, + "logps/rejected": -325.2464294433594, + "loss": 0.0876, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.200534820556641, + "rewards/margins": 7.944638252258301, + "rewards/rejected": -14.145173072814941, + "step": 2149 + }, + { + "epoch": 2.81, + "learning_rate": 4.142155659839919e-05, + "logits/chosen": -2.054295063018799, + "logits/rejected": -2.074632406234741, + "logps/chosen": -193.02542114257812, + "logps/rejected": -294.2581787109375, + "loss": 0.0285, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.436005115509033, + "rewards/margins": 7.639278411865234, + "rewards/rejected": -13.07528305053711, + "step": 2150 + }, + { + "epoch": 2.81, + "learning_rate": 4.141370108129238e-05, + "logits/chosen": -2.0766634941101074, + "logits/rejected": -2.038219451904297, + "logps/chosen": -236.00892639160156, + "logps/rejected": -347.83319091796875, + "loss": 0.087, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.99525260925293, + "rewards/margins": 8.909188270568848, + "rewards/rejected": -13.904440879821777, + "step": 2151 + }, + { + "epoch": 2.82, + "learning_rate": 4.1405842714718985e-05, + "logits/chosen": -2.1349785327911377, + "logits/rejected": -2.10587739944458, + "logps/chosen": -295.6947326660156, + "logps/rejected": -348.4321594238281, + "loss": 0.0535, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.5709710121154785, + "rewards/margins": 6.291179656982422, + "rewards/rejected": -11.862152099609375, + "step": 2152 + }, + { + "epoch": 2.82, + "learning_rate": 4.139798150004325e-05, + "logits/chosen": -1.93121337890625, + "logits/rejected": -2.0886569023132324, + "logps/chosen": -217.17417907714844, + "logps/rejected": -329.839111328125, + "loss": 0.0019, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.0091552734375, + "rewards/margins": 9.298357963562012, + "rewards/rejected": -15.307512283325195, + "step": 2153 + }, + { + "epoch": 2.82, + "learning_rate": 4.139011743862991e-05, + "logits/chosen": -2.001394510269165, + "logits/rejected": -2.1222944259643555, + "logps/chosen": -207.75503540039062, + "logps/rejected": -310.826171875, + "loss": 0.1308, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.948476314544678, + "rewards/margins": 8.621466636657715, + "rewards/rejected": -13.56994342803955, + "step": 2154 + }, + { + "epoch": 2.82, + "learning_rate": 4.138225053184417e-05, + "logits/chosen": -2.2703278064727783, + "logits/rejected": -2.2544658184051514, + "logps/chosen": -218.79788208007812, + "logps/rejected": -288.935791015625, + "loss": 0.0203, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.186158180236816, + "rewards/margins": 6.715668678283691, + "rewards/rejected": -11.901827812194824, + "step": 2155 + }, + { + "epoch": 2.82, + "learning_rate": 4.137438078105176e-05, + "logits/chosen": -2.1223785877227783, + "logits/rejected": -2.1393320560455322, + "logps/chosen": -240.15248107910156, + "logps/rejected": -336.35992431640625, + "loss": 0.0464, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.761418342590332, + "rewards/margins": 8.959957122802734, + "rewards/rejected": -13.72137451171875, + "step": 2156 + }, + { + "epoch": 2.82, + "learning_rate": 4.1366508187618895e-05, + "logits/chosen": -1.9917843341827393, + "logits/rejected": -1.9294066429138184, + "logps/chosen": -202.04116821289062, + "logps/rejected": -256.02581787109375, + "loss": 0.0574, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.986374855041504, + "rewards/margins": 6.414172172546387, + "rewards/rejected": -11.40054702758789, + "step": 2157 + }, + { + "epoch": 2.82, + "learning_rate": 4.135863275291229e-05, + "logits/chosen": -1.9927433729171753, + "logits/rejected": -1.9758617877960205, + "logps/chosen": -264.622314453125, + "logps/rejected": -306.6430358886719, + "loss": 0.0157, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.7758002281188965, + "rewards/margins": 6.953046798706055, + "rewards/rejected": -12.72884750366211, + "step": 2158 + }, + { + "epoch": 2.83, + "learning_rate": 4.1350754478299114e-05, + "logits/chosen": -2.1066510677337646, + "logits/rejected": -2.15097713470459, + "logps/chosen": -297.3056335449219, + "logps/rejected": -377.4056396484375, + "loss": 0.0444, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.846955299377441, + "rewards/margins": 8.380105018615723, + "rewards/rejected": -13.227059364318848, + "step": 2159 + }, + { + "epoch": 2.83, + "learning_rate": 4.1342873365147074e-05, + "logits/chosen": -1.971630573272705, + "logits/rejected": -1.931064486503601, + "logps/chosen": -307.5804748535156, + "logps/rejected": -372.26531982421875, + "loss": 0.0626, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.562823295593262, + "rewards/margins": 8.449759483337402, + "rewards/rejected": -14.01258373260498, + "step": 2160 + }, + { + "epoch": 2.83, + "learning_rate": 4.1334989414824364e-05, + "logits/chosen": -1.9147617816925049, + "logits/rejected": -1.9771555662155151, + "logps/chosen": -235.437744140625, + "logps/rejected": -322.82086181640625, + "loss": 0.0019, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.258989334106445, + "rewards/margins": 8.508306503295898, + "rewards/rejected": -12.76729679107666, + "step": 2161 + }, + { + "epoch": 2.83, + "learning_rate": 4.1327102628699646e-05, + "logits/chosen": -1.9146307706832886, + "logits/rejected": -1.8994238376617432, + "logps/chosen": -223.66262817382812, + "logps/rejected": -278.8446350097656, + "loss": 0.087, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.353767156600952, + "rewards/margins": 8.824553489685059, + "rewards/rejected": -12.178321838378906, + "step": 2162 + }, + { + "epoch": 2.83, + "learning_rate": 4.1319213008142106e-05, + "logits/chosen": -1.9332091808319092, + "logits/rejected": -2.0296545028686523, + "logps/chosen": -244.3536376953125, + "logps/rejected": -288.45928955078125, + "loss": 0.047, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.429356575012207, + "rewards/margins": 7.586400985717773, + "rewards/rejected": -11.015758514404297, + "step": 2163 + }, + { + "epoch": 2.83, + "learning_rate": 4.131132055452138e-05, + "logits/chosen": -2.0869193077087402, + "logits/rejected": -2.067127227783203, + "logps/chosen": -226.43116760253906, + "logps/rejected": -282.193359375, + "loss": 0.0718, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.105947971343994, + "rewards/margins": 7.346132278442383, + "rewards/rejected": -11.452079772949219, + "step": 2164 + }, + { + "epoch": 2.83, + "learning_rate": 4.130342526920765e-05, + "logits/chosen": -2.236833333969116, + "logits/rejected": -2.1940701007843018, + "logps/chosen": -336.6755676269531, + "logps/rejected": -400.3515319824219, + "loss": 0.0466, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.350348472595215, + "rewards/margins": 8.958721160888672, + "rewards/rejected": -13.309070587158203, + "step": 2165 + }, + { + "epoch": 2.83, + "learning_rate": 4.129552715357153e-05, + "logits/chosen": -1.699863314628601, + "logits/rejected": -1.746483325958252, + "logps/chosen": -229.27069091796875, + "logps/rejected": -304.8611755371094, + "loss": 0.1094, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.467954635620117, + "rewards/margins": 7.739739418029785, + "rewards/rejected": -11.207694053649902, + "step": 2166 + }, + { + "epoch": 2.84, + "learning_rate": 4.128762620898419e-05, + "logits/chosen": -2.0914857387542725, + "logits/rejected": -2.136545181274414, + "logps/chosen": -275.3821716308594, + "logps/rejected": -331.4245910644531, + "loss": 0.0057, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.5834550857543945, + "rewards/margins": 6.8916544914245605, + "rewards/rejected": -11.47511100769043, + "step": 2167 + }, + { + "epoch": 2.84, + "learning_rate": 4.1279722436817226e-05, + "logits/chosen": -1.9021196365356445, + "logits/rejected": -2.0389928817749023, + "logps/chosen": -189.16964721679688, + "logps/rejected": -258.6311340332031, + "loss": 0.0919, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.3057665824890137, + "rewards/margins": 7.311253547668457, + "rewards/rejected": -10.617019653320312, + "step": 2168 + }, + { + "epoch": 2.84, + "learning_rate": 4.1271815838442774e-05, + "logits/chosen": -1.6419003009796143, + "logits/rejected": -1.6636180877685547, + "logps/chosen": -275.1500244140625, + "logps/rejected": -343.6715087890625, + "loss": 0.0442, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.7763137817382812, + "rewards/margins": 9.571589469909668, + "rewards/rejected": -13.347904205322266, + "step": 2169 + }, + { + "epoch": 2.84, + "learning_rate": 4.126390641523344e-05, + "logits/chosen": -2.1074090003967285, + "logits/rejected": -2.1533889770507812, + "logps/chosen": -220.68756103515625, + "logps/rejected": -292.22259521484375, + "loss": 0.0455, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.995213031768799, + "rewards/margins": 7.116742134094238, + "rewards/rejected": -11.111954689025879, + "step": 2170 + }, + { + "epoch": 2.84, + "learning_rate": 4.125599416856232e-05, + "logits/chosen": -1.9926451444625854, + "logits/rejected": -1.931626319885254, + "logps/chosen": -232.9684295654297, + "logps/rejected": -290.0226745605469, + "loss": 0.0133, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.7480080127716064, + "rewards/margins": 7.809638500213623, + "rewards/rejected": -11.557646751403809, + "step": 2171 + }, + { + "epoch": 2.84, + "learning_rate": 4.1248079099803004e-05, + "logits/chosen": -2.080148458480835, + "logits/rejected": -2.0510504245758057, + "logps/chosen": -271.93731689453125, + "logps/rejected": -353.4999694824219, + "loss": 0.0009, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.3109636306762695, + "rewards/margins": 10.576560974121094, + "rewards/rejected": -14.88752269744873, + "step": 2172 + }, + { + "epoch": 2.84, + "learning_rate": 4.124016121032957e-05, + "logits/chosen": -2.082777500152588, + "logits/rejected": -2.1824376583099365, + "logps/chosen": -190.74664306640625, + "logps/rejected": -261.811767578125, + "loss": 0.1759, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.506443500518799, + "rewards/margins": 6.029775619506836, + "rewards/rejected": -9.536218643188477, + "step": 2173 + }, + { + "epoch": 2.85, + "learning_rate": 4.12322405015166e-05, + "logits/chosen": -2.0977468490600586, + "logits/rejected": -1.9874368906021118, + "logps/chosen": -198.05026245117188, + "logps/rejected": -290.55340576171875, + "loss": 0.1391, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.6181581020355225, + "rewards/margins": 6.918950080871582, + "rewards/rejected": -9.537108421325684, + "step": 2174 + }, + { + "epoch": 2.85, + "learning_rate": 4.122431697473912e-05, + "logits/chosen": -1.9559710025787354, + "logits/rejected": -2.025653839111328, + "logps/chosen": -288.446533203125, + "logps/rejected": -345.2745361328125, + "loss": 0.0913, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.877231121063232, + "rewards/margins": 6.645547389984131, + "rewards/rejected": -11.522778511047363, + "step": 2175 + }, + { + "epoch": 2.85, + "learning_rate": 4.12163906313727e-05, + "logits/chosen": -2.0126967430114746, + "logits/rejected": -2.1336889266967773, + "logps/chosen": -253.75491333007812, + "logps/rejected": -331.97210693359375, + "loss": 0.0436, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.9921789169311523, + "rewards/margins": 9.469217300415039, + "rewards/rejected": -13.461396217346191, + "step": 2176 + }, + { + "epoch": 2.85, + "learning_rate": 4.120846147279339e-05, + "logits/chosen": -2.1571877002716064, + "logits/rejected": -2.1231818199157715, + "logps/chosen": -262.9522705078125, + "logps/rejected": -369.7544860839844, + "loss": 0.0023, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.100529670715332, + "rewards/margins": 8.334796905517578, + "rewards/rejected": -13.435324668884277, + "step": 2177 + }, + { + "epoch": 2.85, + "learning_rate": 4.120052950037769e-05, + "logits/chosen": -1.8106213808059692, + "logits/rejected": -2.013967514038086, + "logps/chosen": -216.9224090576172, + "logps/rejected": -329.19482421875, + "loss": 0.0072, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.862124443054199, + "rewards/margins": 9.059403419494629, + "rewards/rejected": -13.921527862548828, + "step": 2178 + }, + { + "epoch": 2.85, + "learning_rate": 4.119259471550262e-05, + "logits/chosen": -1.8358765840530396, + "logits/rejected": -1.9351555109024048, + "logps/chosen": -215.11273193359375, + "logps/rejected": -340.1754150390625, + "loss": 0.0455, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.384608268737793, + "rewards/margins": 8.696837425231934, + "rewards/rejected": -13.081446647644043, + "step": 2179 + }, + { + "epoch": 2.85, + "learning_rate": 4.118465711954569e-05, + "logits/chosen": -2.379478931427002, + "logits/rejected": -2.270138740539551, + "logps/chosen": -311.94403076171875, + "logps/rejected": -377.7488708496094, + "loss": 0.049, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.311163902282715, + "rewards/margins": 7.913090229034424, + "rewards/rejected": -12.224254608154297, + "step": 2180 + }, + { + "epoch": 2.85, + "learning_rate": 4.117671671388489e-05, + "logits/chosen": -2.1019277572631836, + "logits/rejected": -2.0322742462158203, + "logps/chosen": -245.09219360351562, + "logps/rejected": -329.4565734863281, + "loss": 0.0469, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.797667503356934, + "rewards/margins": 7.920835494995117, + "rewards/rejected": -12.718502044677734, + "step": 2181 + }, + { + "epoch": 2.86, + "learning_rate": 4.1168773499898683e-05, + "logits/chosen": -1.9382474422454834, + "logits/rejected": -1.9113414287567139, + "logps/chosen": -236.1922149658203, + "logps/rejected": -353.05841064453125, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.7860846519470215, + "rewards/margins": 10.678539276123047, + "rewards/rejected": -15.464625358581543, + "step": 2182 + }, + { + "epoch": 2.86, + "learning_rate": 4.1160827478966043e-05, + "logits/chosen": -2.0668325424194336, + "logits/rejected": -2.1790409088134766, + "logps/chosen": -292.84307861328125, + "logps/rejected": -359.0924987792969, + "loss": 0.003, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.625582695007324, + "rewards/margins": 7.887023448944092, + "rewards/rejected": -13.512606620788574, + "step": 2183 + }, + { + "epoch": 2.86, + "learning_rate": 4.1152878652466433e-05, + "logits/chosen": -2.0241265296936035, + "logits/rejected": -2.0191540718078613, + "logps/chosen": -287.0768127441406, + "logps/rejected": -323.9988708496094, + "loss": 0.0987, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.580556869506836, + "rewards/margins": 7.032944679260254, + "rewards/rejected": -12.613500595092773, + "step": 2184 + }, + { + "epoch": 2.86, + "learning_rate": 4.114492702177978e-05, + "logits/chosen": -2.0294430255889893, + "logits/rejected": -2.0676746368408203, + "logps/chosen": -222.878662109375, + "logps/rejected": -280.4661865234375, + "loss": 0.0881, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.30618953704834, + "rewards/margins": 7.54060697555542, + "rewards/rejected": -11.846796989440918, + "step": 2185 + }, + { + "epoch": 2.86, + "learning_rate": 4.113697258828651e-05, + "logits/chosen": -2.190463066101074, + "logits/rejected": -2.068010091781616, + "logps/chosen": -263.63275146484375, + "logps/rejected": -285.02313232421875, + "loss": 0.1216, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.808750152587891, + "rewards/margins": 6.679539203643799, + "rewards/rejected": -11.488289833068848, + "step": 2186 + }, + { + "epoch": 2.86, + "learning_rate": 4.112901535336754e-05, + "logits/chosen": -1.9238157272338867, + "logits/rejected": -1.92644202709198, + "logps/chosen": -250.43479919433594, + "logps/rejected": -306.7027893066406, + "loss": 0.0481, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.677032470703125, + "rewards/margins": 7.560239791870117, + "rewards/rejected": -12.237272262573242, + "step": 2187 + }, + { + "epoch": 2.86, + "learning_rate": 4.1121055318404264e-05, + "logits/chosen": -1.8979827165603638, + "logits/rejected": -1.932413935661316, + "logps/chosen": -185.6468505859375, + "logps/rejected": -269.4192810058594, + "loss": 0.131, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.4043474197387695, + "rewards/margins": 6.746544361114502, + "rewards/rejected": -11.150891304016113, + "step": 2188 + }, + { + "epoch": 2.86, + "learning_rate": 4.111309248477858e-05, + "logits/chosen": -2.0596837997436523, + "logits/rejected": -2.1338276863098145, + "logps/chosen": -225.3911895751953, + "logps/rejected": -299.9465026855469, + "loss": 0.0501, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.601292133331299, + "rewards/margins": 7.340612888336182, + "rewards/rejected": -11.941904067993164, + "step": 2189 + }, + { + "epoch": 2.87, + "learning_rate": 4.1105126853872845e-05, + "logits/chosen": -1.857706069946289, + "logits/rejected": -1.870910406112671, + "logps/chosen": -259.96826171875, + "logps/rejected": -323.454833984375, + "loss": 0.0719, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.881323337554932, + "rewards/margins": 7.966041564941406, + "rewards/rejected": -13.84736442565918, + "step": 2190 + }, + { + "epoch": 2.87, + "learning_rate": 4.109715842706993e-05, + "logits/chosen": -1.9305803775787354, + "logits/rejected": -1.949027180671692, + "logps/chosen": -190.01832580566406, + "logps/rejected": -272.2169189453125, + "loss": 0.1018, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.9668006896972656, + "rewards/margins": 7.031905651092529, + "rewards/rejected": -10.998706817626953, + "step": 2191 + }, + { + "epoch": 2.87, + "learning_rate": 4.1089187205753164e-05, + "logits/chosen": -2.0830509662628174, + "logits/rejected": -2.0943126678466797, + "logps/chosen": -226.88882446289062, + "logps/rejected": -289.788330078125, + "loss": 0.1208, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.195775508880615, + "rewards/margins": 6.762811660766602, + "rewards/rejected": -10.958586692810059, + "step": 2192 + }, + { + "epoch": 2.87, + "learning_rate": 4.108121319130638e-05, + "logits/chosen": -1.9216828346252441, + "logits/rejected": -2.0078539848327637, + "logps/chosen": -209.600830078125, + "logps/rejected": -294.1441955566406, + "loss": 0.0111, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.5034751892089844, + "rewards/margins": 8.239934921264648, + "rewards/rejected": -11.743410110473633, + "step": 2193 + }, + { + "epoch": 2.87, + "learning_rate": 4.107323638511389e-05, + "logits/chosen": -1.9660532474517822, + "logits/rejected": -2.1336467266082764, + "logps/chosen": -212.7504425048828, + "logps/rejected": -326.82940673828125, + "loss": 0.0888, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.168840408325195, + "rewards/margins": 7.612202167510986, + "rewards/rejected": -11.781042098999023, + "step": 2194 + }, + { + "epoch": 2.87, + "learning_rate": 4.106525678856049e-05, + "logits/chosen": -2.172245979309082, + "logits/rejected": -2.1358203887939453, + "logps/chosen": -276.8701477050781, + "logps/rejected": -380.2010803222656, + "loss": 0.021, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.663802146911621, + "rewards/margins": 9.204500198364258, + "rewards/rejected": -14.868301391601562, + "step": 2195 + }, + { + "epoch": 2.87, + "learning_rate": 4.1057274403031456e-05, + "logits/chosen": -2.169541120529175, + "logits/rejected": -2.206789493560791, + "logps/chosen": -210.1413116455078, + "logps/rejected": -250.99085998535156, + "loss": 0.0912, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.160667896270752, + "rewards/margins": 6.4327392578125, + "rewards/rejected": -11.593406677246094, + "step": 2196 + }, + { + "epoch": 2.88, + "learning_rate": 4.104928922991256e-05, + "logits/chosen": -1.8071986436843872, + "logits/rejected": -1.9281902313232422, + "logps/chosen": -195.72409057617188, + "logps/rejected": -309.42364501953125, + "loss": 0.0709, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.420380592346191, + "rewards/margins": 7.542280673980713, + "rewards/rejected": -12.96265983581543, + "step": 2197 + }, + { + "epoch": 2.88, + "learning_rate": 4.104130127059006e-05, + "logits/chosen": -1.4423290491104126, + "logits/rejected": -1.486830472946167, + "logps/chosen": -245.29315185546875, + "logps/rejected": -349.1470642089844, + "loss": 0.0448, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.846521854400635, + "rewards/margins": 8.050249099731445, + "rewards/rejected": -12.896770477294922, + "step": 2198 + }, + { + "epoch": 2.88, + "learning_rate": 4.1033310526450675e-05, + "logits/chosen": -2.0059428215026855, + "logits/rejected": -2.086317777633667, + "logps/chosen": -246.69886779785156, + "logps/rejected": -341.550048828125, + "loss": 0.1013, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.417656898498535, + "rewards/margins": 7.65479040145874, + "rewards/rejected": -12.0724458694458, + "step": 2199 + }, + { + "epoch": 2.88, + "learning_rate": 4.102531699888163e-05, + "logits/chosen": -1.9524719715118408, + "logits/rejected": -2.0715389251708984, + "logps/chosen": -200.7295379638672, + "logps/rejected": -330.2983703613281, + "loss": 0.0024, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.0946879386901855, + "rewards/margins": 8.889674186706543, + "rewards/rejected": -12.984362602233887, + "step": 2200 + }, + { + "epoch": 2.88, + "learning_rate": 4.1017320689270614e-05, + "logits/chosen": -1.9974534511566162, + "logits/rejected": -1.9925034046173096, + "logps/chosen": -245.26065063476562, + "logps/rejected": -287.6265563964844, + "loss": 0.094, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.502739906311035, + "rewards/margins": 6.654064655303955, + "rewards/rejected": -10.156805038452148, + "step": 2201 + }, + { + "epoch": 2.88, + "learning_rate": 4.100932159900582e-05, + "logits/chosen": -1.9044215679168701, + "logits/rejected": -1.944065809249878, + "logps/chosen": -229.14227294921875, + "logps/rejected": -277.8479919433594, + "loss": 0.0599, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.378231048583984, + "rewards/margins": 5.870229721069336, + "rewards/rejected": -10.24846076965332, + "step": 2202 + }, + { + "epoch": 2.88, + "learning_rate": 4.1001319729475916e-05, + "logits/chosen": -2.0188095569610596, + "logits/rejected": -2.0112361907958984, + "logps/chosen": -214.5211181640625, + "logps/rejected": -300.25091552734375, + "loss": 0.0069, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.251053333282471, + "rewards/margins": 8.576237678527832, + "rewards/rejected": -13.827289581298828, + "step": 2203 + }, + { + "epoch": 2.88, + "learning_rate": 4.099331508207005e-05, + "logits/chosen": -1.9385230541229248, + "logits/rejected": -1.9427070617675781, + "logps/chosen": -234.5819854736328, + "logps/rejected": -330.6009826660156, + "loss": 0.1316, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.050275802612305, + "rewards/margins": 7.512202739715576, + "rewards/rejected": -12.562477111816406, + "step": 2204 + }, + { + "epoch": 2.89, + "learning_rate": 4.098530765817784e-05, + "logits/chosen": -2.026866912841797, + "logits/rejected": -2.1532793045043945, + "logps/chosen": -199.71560668945312, + "logps/rejected": -320.5267028808594, + "loss": 0.0092, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.68998908996582, + "rewards/margins": 7.659614562988281, + "rewards/rejected": -12.349604606628418, + "step": 2205 + }, + { + "epoch": 2.89, + "learning_rate": 4.09772974591894e-05, + "logits/chosen": -1.7982935905456543, + "logits/rejected": -1.8411973714828491, + "logps/chosen": -237.09228515625, + "logps/rejected": -322.3974609375, + "loss": 0.0102, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.479815483093262, + "rewards/margins": 8.23798656463623, + "rewards/rejected": -13.717802047729492, + "step": 2206 + }, + { + "epoch": 2.89, + "learning_rate": 4.0969284486495335e-05, + "logits/chosen": -1.76961088180542, + "logits/rejected": -1.8508358001708984, + "logps/chosen": -240.76266479492188, + "logps/rejected": -342.0540771484375, + "loss": 0.044, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.3279876708984375, + "rewards/margins": 9.263980865478516, + "rewards/rejected": -14.591968536376953, + "step": 2207 + }, + { + "epoch": 2.89, + "learning_rate": 4.0961268741486716e-05, + "logits/chosen": -1.981732726097107, + "logits/rejected": -2.117330551147461, + "logps/chosen": -237.18115234375, + "logps/rejected": -341.62762451171875, + "loss": 0.0871, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.773239612579346, + "rewards/margins": 8.926799774169922, + "rewards/rejected": -14.700039863586426, + "step": 2208 + }, + { + "epoch": 2.89, + "learning_rate": 4.09532502255551e-05, + "logits/chosen": -2.046422004699707, + "logits/rejected": -2.099958658218384, + "logps/chosen": -232.92481994628906, + "logps/rejected": -325.4584655761719, + "loss": 0.0145, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.9477105140686035, + "rewards/margins": 8.676502227783203, + "rewards/rejected": -13.624212265014648, + "step": 2209 + }, + { + "epoch": 2.89, + "learning_rate": 4.094522894009251e-05, + "logits/chosen": -1.593595027923584, + "logits/rejected": -1.5732452869415283, + "logps/chosen": -209.82452392578125, + "logps/rejected": -287.2315673828125, + "loss": 0.0061, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.161198139190674, + "rewards/margins": 8.423956871032715, + "rewards/rejected": -12.585155487060547, + "step": 2210 + }, + { + "epoch": 2.89, + "learning_rate": 4.0937204886491485e-05, + "logits/chosen": -1.9955334663391113, + "logits/rejected": -2.0291831493377686, + "logps/chosen": -206.82830810546875, + "logps/rejected": -286.96966552734375, + "loss": 0.0463, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.212306022644043, + "rewards/margins": 7.9994988441467285, + "rewards/rejected": -13.211803436279297, + "step": 2211 + }, + { + "epoch": 2.89, + "learning_rate": 4.092917806614503e-05, + "logits/chosen": -2.0906317234039307, + "logits/rejected": -2.156611442565918, + "logps/chosen": -244.69973754882812, + "logps/rejected": -351.0030822753906, + "loss": 0.0988, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.618649005889893, + "rewards/margins": 8.488447189331055, + "rewards/rejected": -14.107095718383789, + "step": 2212 + }, + { + "epoch": 2.9, + "learning_rate": 4.092114848044659e-05, + "logits/chosen": -1.9830385446548462, + "logits/rejected": -2.0258727073669434, + "logps/chosen": -236.79864501953125, + "logps/rejected": -335.24591064453125, + "loss": 0.0021, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.998775959014893, + "rewards/margins": 10.42489242553711, + "rewards/rejected": -15.423666954040527, + "step": 2213 + }, + { + "epoch": 2.9, + "learning_rate": 4.0913116130790164e-05, + "logits/chosen": -2.1110739707946777, + "logits/rejected": -2.1509482860565186, + "logps/chosen": -217.78807067871094, + "logps/rejected": -330.7835388183594, + "loss": 0.0903, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.235716819763184, + "rewards/margins": 7.777309417724609, + "rewards/rejected": -13.01302719116211, + "step": 2214 + }, + { + "epoch": 2.9, + "learning_rate": 4.090508101857017e-05, + "logits/chosen": -2.1012043952941895, + "logits/rejected": -2.162986993789673, + "logps/chosen": -242.8408660888672, + "logps/rejected": -334.86444091796875, + "loss": 0.0873, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.359984397888184, + "rewards/margins": 9.122700691223145, + "rewards/rejected": -13.482685089111328, + "step": 2215 + }, + { + "epoch": 2.9, + "learning_rate": 4.089704314518152e-05, + "logits/chosen": -1.97257661819458, + "logits/rejected": -2.0269057750701904, + "logps/chosen": -178.48583984375, + "logps/rejected": -294.429931640625, + "loss": 0.1325, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.222825050354004, + "rewards/margins": 8.763550758361816, + "rewards/rejected": -12.98637580871582, + "step": 2216 + }, + { + "epoch": 2.9, + "learning_rate": 4.088900251201964e-05, + "logits/chosen": -2.030012607574463, + "logits/rejected": -1.9918521642684937, + "logps/chosen": -221.71617126464844, + "logps/rejected": -312.90399169921875, + "loss": 0.0141, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.6067910194396973, + "rewards/margins": 9.348206520080566, + "rewards/rejected": -12.954997062683105, + "step": 2217 + }, + { + "epoch": 2.9, + "learning_rate": 4.088095912048038e-05, + "logits/chosen": -2.0294079780578613, + "logits/rejected": -2.2200558185577393, + "logps/chosen": -195.28195190429688, + "logps/rejected": -322.5656433105469, + "loss": 0.0878, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.590506553649902, + "rewards/margins": 7.0590667724609375, + "rewards/rejected": -11.649572372436523, + "step": 2218 + }, + { + "epoch": 2.9, + "learning_rate": 4.08729129719601e-05, + "logits/chosen": -2.005953550338745, + "logits/rejected": -2.0370614528656006, + "logps/chosen": -192.7057647705078, + "logps/rejected": -234.94427490234375, + "loss": 0.1856, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.630458354949951, + "rewards/margins": 5.305692195892334, + "rewards/rejected": -9.936150550842285, + "step": 2219 + }, + { + "epoch": 2.91, + "learning_rate": 4.086486406785566e-05, + "logits/chosen": -2.1289520263671875, + "logits/rejected": -2.2230443954467773, + "logps/chosen": -262.62725830078125, + "logps/rejected": -385.7099609375, + "loss": 0.1385, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.901250839233398, + "rewards/margins": 6.619694232940674, + "rewards/rejected": -11.52094554901123, + "step": 2220 + }, + { + "epoch": 2.91, + "learning_rate": 4.085681240956434e-05, + "logits/chosen": -2.0626068115234375, + "logits/rejected": -2.1245715618133545, + "logps/chosen": -244.63568115234375, + "logps/rejected": -400.87127685546875, + "loss": 0.0005, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.728460311889648, + "rewards/margins": 10.4347562789917, + "rewards/rejected": -15.163215637207031, + "step": 2221 + }, + { + "epoch": 2.91, + "learning_rate": 4.084875799848395e-05, + "logits/chosen": -1.9657922983169556, + "logits/rejected": -2.028837203979492, + "logps/chosen": -246.70101928710938, + "logps/rejected": -343.2515563964844, + "loss": 0.0514, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.407756805419922, + "rewards/margins": 7.938384532928467, + "rewards/rejected": -12.346141815185547, + "step": 2222 + }, + { + "epoch": 2.91, + "learning_rate": 4.084070083601276e-05, + "logits/chosen": -2.0747859477996826, + "logits/rejected": -1.979410171508789, + "logps/chosen": -273.43145751953125, + "logps/rejected": -331.90582275390625, + "loss": 0.0018, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.8637495040893555, + "rewards/margins": 8.859865188598633, + "rewards/rejected": -14.723615646362305, + "step": 2223 + }, + { + "epoch": 2.91, + "learning_rate": 4.0832640923549515e-05, + "logits/chosen": -1.9947311878204346, + "logits/rejected": -2.0264732837677, + "logps/chosen": -225.6526336669922, + "logps/rejected": -317.21722412109375, + "loss": 0.0493, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.485606670379639, + "rewards/margins": 8.677902221679688, + "rewards/rejected": -14.1635103225708, + "step": 2224 + }, + { + "epoch": 2.91, + "learning_rate": 4.082457826249343e-05, + "logits/chosen": -2.0638132095336914, + "logits/rejected": -2.0210137367248535, + "logps/chosen": -264.2833251953125, + "logps/rejected": -337.869140625, + "loss": 0.0891, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.095065593719482, + "rewards/margins": 7.04831075668335, + "rewards/rejected": -12.143376350402832, + "step": 2225 + }, + { + "epoch": 2.91, + "learning_rate": 4.0816512854244224e-05, + "logits/chosen": -1.7565228939056396, + "logits/rejected": -1.877976894378662, + "logps/chosen": -218.58175659179688, + "logps/rejected": -353.1058654785156, + "loss": 0.0879, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.139636993408203, + "rewards/margins": 9.21428108215332, + "rewards/rejected": -13.353917121887207, + "step": 2226 + }, + { + "epoch": 2.91, + "learning_rate": 4.080844470020205e-05, + "logits/chosen": -2.0760161876678467, + "logits/rejected": -2.198225259780884, + "logps/chosen": -237.87025451660156, + "logps/rejected": -352.52825927734375, + "loss": 0.0054, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.9761271476745605, + "rewards/margins": 8.944655418395996, + "rewards/rejected": -13.920783042907715, + "step": 2227 + }, + { + "epoch": 2.92, + "learning_rate": 4.08003738017676e-05, + "logits/chosen": -2.160156488418579, + "logits/rejected": -2.2180447578430176, + "logps/chosen": -267.6474304199219, + "logps/rejected": -358.6483459472656, + "loss": 0.009, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.819602012634277, + "rewards/margins": 7.647256851196289, + "rewards/rejected": -13.466859817504883, + "step": 2228 + }, + { + "epoch": 2.92, + "learning_rate": 4.079230016034197e-05, + "logits/chosen": -1.7095756530761719, + "logits/rejected": -1.7194485664367676, + "logps/chosen": -226.6480712890625, + "logps/rejected": -303.1681213378906, + "loss": 0.0772, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.34532356262207, + "rewards/margins": 7.812335968017578, + "rewards/rejected": -12.157660484313965, + "step": 2229 + }, + { + "epoch": 2.92, + "learning_rate": 4.078422377732678e-05, + "logits/chosen": -2.097193956375122, + "logits/rejected": -2.124293327331543, + "logps/chosen": -249.19630432128906, + "logps/rejected": -346.5874938964844, + "loss": 0.0541, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.2485575675964355, + "rewards/margins": 7.850658893585205, + "rewards/rejected": -13.09921646118164, + "step": 2230 + }, + { + "epoch": 2.92, + "learning_rate": 4.0776144654124125e-05, + "logits/chosen": -2.2765426635742188, + "logits/rejected": -2.270753860473633, + "logps/chosen": -241.84669494628906, + "logps/rejected": -336.55096435546875, + "loss": 0.0791, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.211452007293701, + "rewards/margins": 8.208662033081055, + "rewards/rejected": -14.420112609863281, + "step": 2231 + }, + { + "epoch": 2.92, + "learning_rate": 4.0768062792136555e-05, + "logits/chosen": -2.1474051475524902, + "logits/rejected": -2.1365041732788086, + "logps/chosen": -244.55490112304688, + "logps/rejected": -346.3648376464844, + "loss": 0.0013, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.497187614440918, + "rewards/margins": 8.268601417541504, + "rewards/rejected": -12.765789031982422, + "step": 2232 + }, + { + "epoch": 2.92, + "learning_rate": 4.07599781927671e-05, + "logits/chosen": -2.1392455101013184, + "logits/rejected": -2.142500400543213, + "logps/chosen": -250.07745361328125, + "logps/rejected": -311.97808837890625, + "loss": 0.1738, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.954948425292969, + "rewards/margins": 6.751895427703857, + "rewards/rejected": -11.706844329833984, + "step": 2233 + }, + { + "epoch": 2.92, + "learning_rate": 4.075189085741927e-05, + "logits/chosen": -2.1242191791534424, + "logits/rejected": -2.1828739643096924, + "logps/chosen": -265.0927429199219, + "logps/rejected": -344.9898376464844, + "loss": 0.0035, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.8403639793396, + "rewards/margins": 8.913230895996094, + "rewards/rejected": -14.753594398498535, + "step": 2234 + }, + { + "epoch": 2.92, + "learning_rate": 4.074380078749707e-05, + "logits/chosen": -1.9724620580673218, + "logits/rejected": -2.0979995727539062, + "logps/chosen": -230.10606384277344, + "logps/rejected": -340.49420166015625, + "loss": 0.0882, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.136164665222168, + "rewards/margins": 7.166202068328857, + "rewards/rejected": -12.302366256713867, + "step": 2235 + }, + { + "epoch": 2.93, + "learning_rate": 4.073570798440493e-05, + "logits/chosen": -2.173654794692993, + "logits/rejected": -2.158339262008667, + "logps/chosen": -264.9515075683594, + "logps/rejected": -385.02947998046875, + "loss": 0.0441, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.116758823394775, + "rewards/margins": 10.693913459777832, + "rewards/rejected": -14.810672760009766, + "step": 2236 + }, + { + "epoch": 2.93, + "learning_rate": 4.072761244954781e-05, + "logits/chosen": -1.9193484783172607, + "logits/rejected": -1.9324231147766113, + "logps/chosen": -229.14004516601562, + "logps/rejected": -309.6470031738281, + "loss": 0.0882, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.022686004638672, + "rewards/margins": 7.921196460723877, + "rewards/rejected": -13.94388198852539, + "step": 2237 + }, + { + "epoch": 2.93, + "learning_rate": 4.07195141843311e-05, + "logits/chosen": -2.1049351692199707, + "logits/rejected": -2.204519271850586, + "logps/chosen": -237.9530487060547, + "logps/rejected": -337.65203857421875, + "loss": 0.0497, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.841470718383789, + "rewards/margins": 7.777152061462402, + "rewards/rejected": -13.618623733520508, + "step": 2238 + }, + { + "epoch": 2.93, + "learning_rate": 4.07114131901607e-05, + "logits/chosen": -1.9116312265396118, + "logits/rejected": -2.049605131149292, + "logps/chosen": -310.1016845703125, + "logps/rejected": -415.382568359375, + "loss": 0.0878, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.793592929840088, + "rewards/margins": 7.996792793273926, + "rewards/rejected": -13.790386199951172, + "step": 2239 + }, + { + "epoch": 2.93, + "learning_rate": 4.070330946844295e-05, + "logits/chosen": -2.2676711082458496, + "logits/rejected": -2.268871545791626, + "logps/chosen": -257.8155212402344, + "logps/rejected": -305.8218688964844, + "loss": 0.0539, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.3989386558532715, + "rewards/margins": 7.469062805175781, + "rewards/rejected": -12.868001937866211, + "step": 2240 + }, + { + "epoch": 2.93, + "learning_rate": 4.069520302058469e-05, + "logits/chosen": -2.0371265411376953, + "logits/rejected": -2.0505199432373047, + "logps/chosen": -187.00189208984375, + "logps/rejected": -225.2563934326172, + "loss": 0.1371, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.377021789550781, + "rewards/margins": 5.380834579467773, + "rewards/rejected": -9.757856369018555, + "step": 2241 + }, + { + "epoch": 2.93, + "learning_rate": 4.068709384799322e-05, + "logits/chosen": -2.2195491790771484, + "logits/rejected": -2.273913621902466, + "logps/chosen": -231.99951171875, + "logps/rejected": -323.251953125, + "loss": 0.0899, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.185029983520508, + "rewards/margins": 8.207600593566895, + "rewards/rejected": -13.392631530761719, + "step": 2242 + }, + { + "epoch": 2.94, + "learning_rate": 4.067898195207631e-05, + "logits/chosen": -1.9835385084152222, + "logits/rejected": -2.0143542289733887, + "logps/chosen": -260.90447998046875, + "logps/rejected": -374.3984680175781, + "loss": 0.0442, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.085165977478027, + "rewards/margins": 9.697397232055664, + "rewards/rejected": -14.782563209533691, + "step": 2243 + }, + { + "epoch": 2.94, + "learning_rate": 4.0670867334242223e-05, + "logits/chosen": -2.0046844482421875, + "logits/rejected": -2.1477673053741455, + "logps/chosen": -255.0173797607422, + "logps/rejected": -383.2938537597656, + "loss": 0.0578, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.924704074859619, + "rewards/margins": 8.287879943847656, + "rewards/rejected": -14.212583541870117, + "step": 2244 + }, + { + "epoch": 2.94, + "learning_rate": 4.0662749995899666e-05, + "logits/chosen": -1.951084017753601, + "logits/rejected": -2.0860135555267334, + "logps/chosen": -234.13211059570312, + "logps/rejected": -325.2522888183594, + "loss": 0.044, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.4876708984375, + "rewards/margins": 8.078499794006348, + "rewards/rejected": -14.566171646118164, + "step": 2245 + }, + { + "epoch": 2.94, + "learning_rate": 4.065462993845784e-05, + "logits/chosen": -2.1116063594818115, + "logits/rejected": -2.1521196365356445, + "logps/chosen": -194.48851013183594, + "logps/rejected": -307.51019287109375, + "loss": 0.131, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.6868486404418945, + "rewards/margins": 7.20567512512207, + "rewards/rejected": -11.892522811889648, + "step": 2246 + }, + { + "epoch": 2.94, + "learning_rate": 4.064650716332642e-05, + "logits/chosen": -2.218405246734619, + "logits/rejected": -2.277463436126709, + "logps/chosen": -212.8626251220703, + "logps/rejected": -265.5347595214844, + "loss": 0.0454, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.655837535858154, + "rewards/margins": 6.375062942504883, + "rewards/rejected": -12.030900955200195, + "step": 2247 + }, + { + "epoch": 2.94, + "learning_rate": 4.063838167191553e-05, + "logits/chosen": -2.099470615386963, + "logits/rejected": -2.1984596252441406, + "logps/chosen": -228.62374877929688, + "logps/rejected": -336.2490539550781, + "loss": 0.0444, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.770306587219238, + "rewards/margins": 8.472064971923828, + "rewards/rejected": -14.2423734664917, + "step": 2248 + }, + { + "epoch": 2.94, + "learning_rate": 4.063025346563577e-05, + "logits/chosen": -2.292811393737793, + "logits/rejected": -2.297818660736084, + "logps/chosen": -244.33920288085938, + "logps/rejected": -344.06219482421875, + "loss": 0.0031, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.461674690246582, + "rewards/margins": 8.859297752380371, + "rewards/rejected": -14.320972442626953, + "step": 2249 + }, + { + "epoch": 2.94, + "learning_rate": 4.0622122545898246e-05, + "logits/chosen": -1.9312044382095337, + "logits/rejected": -1.9553238153457642, + "logps/chosen": -278.5567932128906, + "logps/rejected": -343.84344482421875, + "loss": 0.0872, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.784635066986084, + "rewards/margins": 7.91162109375, + "rewards/rejected": -13.696255683898926, + "step": 2250 + }, + { + "epoch": 2.95, + "learning_rate": 4.061398891411449e-05, + "logits/chosen": -2.0505380630493164, + "logits/rejected": -2.088383913040161, + "logps/chosen": -240.8919677734375, + "logps/rejected": -322.6705322265625, + "loss": 0.0475, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.79270076751709, + "rewards/margins": 7.686241626739502, + "rewards/rejected": -13.478941917419434, + "step": 2251 + }, + { + "epoch": 2.95, + "learning_rate": 4.060585257169653e-05, + "logits/chosen": -2.134075880050659, + "logits/rejected": -2.1802515983581543, + "logps/chosen": -235.2249755859375, + "logps/rejected": -284.25543212890625, + "loss": 0.0896, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.834575176239014, + "rewards/margins": 7.375324249267578, + "rewards/rejected": -12.209898948669434, + "step": 2252 + }, + { + "epoch": 2.95, + "learning_rate": 4.059771352005686e-05, + "logits/chosen": -2.306246757507324, + "logits/rejected": -2.350177764892578, + "logps/chosen": -266.5599060058594, + "logps/rejected": -339.7952880859375, + "loss": 0.0466, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.026610851287842, + "rewards/margins": 8.363250732421875, + "rewards/rejected": -13.389862060546875, + "step": 2253 + }, + { + "epoch": 2.95, + "learning_rate": 4.0589571760608443e-05, + "logits/chosen": -1.755591869354248, + "logits/rejected": -1.799825668334961, + "logps/chosen": -215.43942260742188, + "logps/rejected": -324.50396728515625, + "loss": 0.095, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.952765941619873, + "rewards/margins": 9.143765449523926, + "rewards/rejected": -14.096531867980957, + "step": 2254 + }, + { + "epoch": 2.95, + "learning_rate": 4.058142729476471e-05, + "logits/chosen": -2.1598591804504395, + "logits/rejected": -2.207771062850952, + "logps/chosen": -218.27322387695312, + "logps/rejected": -281.4322509765625, + "loss": 0.0652, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.535760879516602, + "rewards/margins": 6.41301155090332, + "rewards/rejected": -12.948771476745605, + "step": 2255 + }, + { + "epoch": 2.95, + "learning_rate": 4.0573280123939574e-05, + "logits/chosen": -2.2826626300811768, + "logits/rejected": -2.242478609085083, + "logps/chosen": -225.7889404296875, + "logps/rejected": -294.6713562011719, + "loss": 0.1417, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.481799602508545, + "rewards/margins": 7.591139793395996, + "rewards/rejected": -12.0729398727417, + "step": 2256 + }, + { + "epoch": 2.95, + "learning_rate": 4.0565130249547396e-05, + "logits/chosen": -2.0708048343658447, + "logits/rejected": -2.065812110900879, + "logps/chosen": -216.64683532714844, + "logps/rejected": -314.45892333984375, + "loss": 0.0447, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.770445346832275, + "rewards/margins": 8.61684799194336, + "rewards/rejected": -13.387293815612793, + "step": 2257 + }, + { + "epoch": 2.96, + "learning_rate": 4.055697767300302e-05, + "logits/chosen": -2.217392683029175, + "logits/rejected": -2.2642881870269775, + "logps/chosen": -229.09271240234375, + "logps/rejected": -312.7999572753906, + "loss": 0.1442, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.301823139190674, + "rewards/margins": 7.752106666564941, + "rewards/rejected": -12.053930282592773, + "step": 2258 + }, + { + "epoch": 2.96, + "learning_rate": 4.054882239572176e-05, + "logits/chosen": -1.9833722114562988, + "logits/rejected": -2.1198909282684326, + "logps/chosen": -219.9600067138672, + "logps/rejected": -289.3666687011719, + "loss": 0.197, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.9837100505828857, + "rewards/margins": 9.081433296203613, + "rewards/rejected": -12.065143585205078, + "step": 2259 + }, + { + "epoch": 2.96, + "learning_rate": 4.054066441911939e-05, + "logits/chosen": -2.0739023685455322, + "logits/rejected": -2.0905725955963135, + "logps/chosen": -217.68226623535156, + "logps/rejected": -290.0678405761719, + "loss": 0.1176, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.164804935455322, + "rewards/margins": 6.717545986175537, + "rewards/rejected": -11.882349967956543, + "step": 2260 + }, + { + "epoch": 2.96, + "learning_rate": 4.053250374461217e-05, + "logits/chosen": -2.0488510131835938, + "logits/rejected": -2.0801382064819336, + "logps/chosen": -197.89596557617188, + "logps/rejected": -301.1357727050781, + "loss": 0.0015, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.089158535003662, + "rewards/margins": 8.928377151489258, + "rewards/rejected": -13.017536163330078, + "step": 2261 + }, + { + "epoch": 2.96, + "learning_rate": 4.052434037361681e-05, + "logits/chosen": -2.1752076148986816, + "logits/rejected": -2.2135376930236816, + "logps/chosen": -196.59396362304688, + "logps/rejected": -283.71185302734375, + "loss": 0.0888, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.9845967292785645, + "rewards/margins": 6.94215202331543, + "rewards/rejected": -11.926748275756836, + "step": 2262 + }, + { + "epoch": 2.96, + "learning_rate": 4.05161743075505e-05, + "logits/chosen": -1.9602102041244507, + "logits/rejected": -1.9223976135253906, + "logps/chosen": -233.06593322753906, + "logps/rejected": -308.6895751953125, + "loss": 0.0482, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.860264301300049, + "rewards/margins": 8.868692398071289, + "rewards/rejected": -13.72895622253418, + "step": 2263 + }, + { + "epoch": 2.96, + "learning_rate": 4.0508005547830884e-05, + "logits/chosen": -1.8972437381744385, + "logits/rejected": -2.00484561920166, + "logps/chosen": -246.32203674316406, + "logps/rejected": -347.1911926269531, + "loss": 0.0046, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.049254894256592, + "rewards/margins": 9.062969207763672, + "rewards/rejected": -14.112222671508789, + "step": 2264 + }, + { + "epoch": 2.96, + "learning_rate": 4.049983409587608e-05, + "logits/chosen": -1.9605618715286255, + "logits/rejected": -1.8788563013076782, + "logps/chosen": -222.1049041748047, + "logps/rejected": -309.7162170410156, + "loss": 0.0104, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.519625663757324, + "rewards/margins": 8.124978065490723, + "rewards/rejected": -13.644603729248047, + "step": 2265 + }, + { + "epoch": 2.97, + "learning_rate": 4.049165995310469e-05, + "logits/chosen": -1.9594004154205322, + "logits/rejected": -1.9734030961990356, + "logps/chosen": -238.72772216796875, + "logps/rejected": -329.5353698730469, + "loss": 0.0983, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.496771812438965, + "rewards/margins": 7.7921142578125, + "rewards/rejected": -13.288885116577148, + "step": 2266 + }, + { + "epoch": 2.97, + "learning_rate": 4.0483483120935766e-05, + "logits/chosen": -1.8387665748596191, + "logits/rejected": -1.8579622507095337, + "logps/chosen": -244.2477569580078, + "logps/rejected": -308.0921325683594, + "loss": 0.0678, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.856629848480225, + "rewards/margins": 7.74281644821167, + "rewards/rejected": -12.599447250366211, + "step": 2267 + }, + { + "epoch": 2.97, + "learning_rate": 4.0475303600788825e-05, + "logits/chosen": -2.012568712234497, + "logits/rejected": -2.0805506706237793, + "logps/chosen": -283.11785888671875, + "logps/rejected": -378.9236755371094, + "loss": 0.0938, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.643699645996094, + "rewards/margins": 8.028934478759766, + "rewards/rejected": -12.67263412475586, + "step": 2268 + }, + { + "epoch": 2.97, + "learning_rate": 4.0467121394083856e-05, + "logits/chosen": -2.284359931945801, + "logits/rejected": -2.301757574081421, + "logps/chosen": -254.15252685546875, + "logps/rejected": -321.6610412597656, + "loss": 0.1873, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.626988887786865, + "rewards/margins": 8.284836769104004, + "rewards/rejected": -12.911825180053711, + "step": 2269 + }, + { + "epoch": 2.97, + "learning_rate": 4.045893650224132e-05, + "logits/chosen": -2.092942237854004, + "logits/rejected": -2.1142632961273193, + "logps/chosen": -243.60618591308594, + "logps/rejected": -353.5018005371094, + "loss": 0.0064, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.826841831207275, + "rewards/margins": 8.407801628112793, + "rewards/rejected": -13.23464298248291, + "step": 2270 + }, + { + "epoch": 2.97, + "learning_rate": 4.0450748926682125e-05, + "logits/chosen": -1.9782465696334839, + "logits/rejected": -2.1163859367370605, + "logps/chosen": -222.07089233398438, + "logps/rejected": -366.1741943359375, + "loss": 0.0013, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.192220211029053, + "rewards/margins": 8.56313705444336, + "rewards/rejected": -13.75535774230957, + "step": 2271 + }, + { + "epoch": 2.97, + "learning_rate": 4.044255866882767e-05, + "logits/chosen": -2.07340669631958, + "logits/rejected": -2.100149154663086, + "logps/chosen": -299.82244873046875, + "logps/rejected": -415.00872802734375, + "loss": 0.0879, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.470133304595947, + "rewards/margins": 7.713799476623535, + "rewards/rejected": -12.18393325805664, + "step": 2272 + }, + { + "epoch": 2.97, + "learning_rate": 4.04343657300998e-05, + "logits/chosen": -2.0520966053009033, + "logits/rejected": -2.1131479740142822, + "logps/chosen": -200.32757568359375, + "logps/rejected": -331.4617614746094, + "loss": 0.0499, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.1466875076293945, + "rewards/margins": 9.14694881439209, + "rewards/rejected": -13.293636322021484, + "step": 2273 + }, + { + "epoch": 2.98, + "learning_rate": 4.042617011192085e-05, + "logits/chosen": -2.280841112136841, + "logits/rejected": -2.2835123538970947, + "logps/chosen": -237.56654357910156, + "logps/rejected": -305.79193115234375, + "loss": 0.1117, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.9321794509887695, + "rewards/margins": 6.701276779174805, + "rewards/rejected": -11.633456230163574, + "step": 2274 + }, + { + "epoch": 2.98, + "learning_rate": 4.0417971815713584e-05, + "logits/chosen": -2.2043027877807617, + "logits/rejected": -2.2083544731140137, + "logps/chosen": -245.98263549804688, + "logps/rejected": -355.17608642578125, + "loss": 0.0449, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.231986999511719, + "rewards/margins": 8.83634090423584, + "rewards/rejected": -14.068328857421875, + "step": 2275 + }, + { + "epoch": 2.98, + "learning_rate": 4.040977084290125e-05, + "logits/chosen": -2.291473627090454, + "logits/rejected": -2.2201426029205322, + "logps/chosen": -238.4390106201172, + "logps/rejected": -292.26910400390625, + "loss": 0.0063, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.370654582977295, + "rewards/margins": 7.422157287597656, + "rewards/rejected": -11.792811393737793, + "step": 2276 + }, + { + "epoch": 2.98, + "learning_rate": 4.040156719490757e-05, + "logits/chosen": -2.1759634017944336, + "logits/rejected": -2.283318042755127, + "logps/chosen": -215.19366455078125, + "logps/rejected": -313.5849914550781, + "loss": 0.0027, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.866866111755371, + "rewards/margins": 8.510933876037598, + "rewards/rejected": -12.377800941467285, + "step": 2277 + }, + { + "epoch": 2.98, + "learning_rate": 4.0393360873156727e-05, + "logits/chosen": -1.8939836025238037, + "logits/rejected": -1.9971206188201904, + "logps/chosen": -264.5501708984375, + "logps/rejected": -374.10809326171875, + "loss": 0.1771, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.189629554748535, + "rewards/margins": 8.558857917785645, + "rewards/rejected": -12.74848747253418, + "step": 2278 + }, + { + "epoch": 2.98, + "learning_rate": 4.038515187907335e-05, + "logits/chosen": -2.284393787384033, + "logits/rejected": -2.287358283996582, + "logps/chosen": -299.2924499511719, + "logps/rejected": -349.8957824707031, + "loss": 0.0971, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.67432975769043, + "rewards/margins": 7.653830528259277, + "rewards/rejected": -12.328161239624023, + "step": 2279 + }, + { + "epoch": 2.98, + "learning_rate": 4.037694021408255e-05, + "logits/chosen": -2.0360541343688965, + "logits/rejected": -2.105217933654785, + "logps/chosen": -187.9983367919922, + "logps/rejected": -275.1484069824219, + "loss": 0.061, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.26163387298584, + "rewards/margins": 7.959683418273926, + "rewards/rejected": -12.221317291259766, + "step": 2280 + }, + { + "epoch": 2.99, + "learning_rate": 4.0368725879609896e-05, + "logits/chosen": -1.9243905544281006, + "logits/rejected": -1.938022494316101, + "logps/chosen": -198.75108337402344, + "logps/rejected": -273.3779296875, + "loss": 0.0024, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.776736259460449, + "rewards/margins": 7.805794715881348, + "rewards/rejected": -11.582530975341797, + "step": 2281 + }, + { + "epoch": 2.99, + "learning_rate": 4.036050887708142e-05, + "logits/chosen": -2.002272605895996, + "logits/rejected": -2.020458936691284, + "logps/chosen": -238.6460723876953, + "logps/rejected": -387.5722961425781, + "loss": 0.0868, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.070218086242676, + "rewards/margins": 9.112555503845215, + "rewards/rejected": -13.18277359008789, + "step": 2282 + }, + { + "epoch": 2.99, + "learning_rate": 4.035228920792361e-05, + "logits/chosen": -2.1598801612854004, + "logits/rejected": -2.286231517791748, + "logps/chosen": -236.18081665039062, + "logps/rejected": -353.8681640625, + "loss": 0.0092, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.896045684814453, + "rewards/margins": 8.549945831298828, + "rewards/rejected": -13.445991516113281, + "step": 2283 + }, + { + "epoch": 2.99, + "learning_rate": 4.034406687356344e-05, + "logits/chosen": -2.0794379711151123, + "logits/rejected": -2.1407833099365234, + "logps/chosen": -217.3106689453125, + "logps/rejected": -331.027099609375, + "loss": 0.0021, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.971529006958008, + "rewards/margins": 9.772381782531738, + "rewards/rejected": -13.743910789489746, + "step": 2284 + }, + { + "epoch": 2.99, + "learning_rate": 4.0335841875428324e-05, + "logits/chosen": -1.941980004310608, + "logits/rejected": -2.0354809761047363, + "logps/chosen": -201.304443359375, + "logps/rejected": -340.4508361816406, + "loss": 0.0923, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.282627582550049, + "rewards/margins": 7.455456256866455, + "rewards/rejected": -11.738083839416504, + "step": 2285 + }, + { + "epoch": 2.99, + "learning_rate": 4.032761421494614e-05, + "logits/chosen": -2.0981268882751465, + "logits/rejected": -2.087101697921753, + "logps/chosen": -245.1864471435547, + "logps/rejected": -277.5719299316406, + "loss": 0.222, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.013238906860352, + "rewards/margins": 5.27143669128418, + "rewards/rejected": -10.284676551818848, + "step": 2286 + }, + { + "epoch": 2.99, + "learning_rate": 4.031938389354524e-05, + "logits/chosen": -2.0440993309020996, + "logits/rejected": -2.119518280029297, + "logps/chosen": -225.06170654296875, + "logps/rejected": -327.6748352050781, + "loss": 0.0893, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.085385799407959, + "rewards/margins": 8.826796531677246, + "rewards/rejected": -12.912181854248047, + "step": 2287 + }, + { + "epoch": 2.99, + "learning_rate": 4.031115091265444e-05, + "logits/chosen": -2.212399482727051, + "logits/rejected": -2.1920952796936035, + "logps/chosen": -297.997802734375, + "logps/rejected": -364.1531677246094, + "loss": 0.0443, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.721320152282715, + "rewards/margins": 7.526367664337158, + "rewards/rejected": -12.247687339782715, + "step": 2288 + }, + { + "epoch": 3.0, + "learning_rate": 4.030291527370299e-05, + "logits/chosen": -2.1944549083709717, + "logits/rejected": -2.1947126388549805, + "logps/chosen": -259.0519104003906, + "logps/rejected": -326.65972900390625, + "loss": 0.0211, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.198507308959961, + "rewards/margins": 8.169037818908691, + "rewards/rejected": -13.367544174194336, + "step": 2289 + }, + { + "epoch": 3.0, + "learning_rate": 4.029467697812064e-05, + "logits/chosen": -2.0671141147613525, + "logits/rejected": -2.0241549015045166, + "logps/chosen": -231.8189697265625, + "logps/rejected": -333.82305908203125, + "loss": 0.0455, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.156424045562744, + "rewards/margins": 9.836962699890137, + "rewards/rejected": -13.993387222290039, + "step": 2290 + }, + { + "epoch": 3.0, + "learning_rate": 4.028643602733756e-05, + "logits/chosen": -1.989417314529419, + "logits/rejected": -2.149822473526001, + "logps/chosen": -204.5302276611328, + "logps/rejected": -314.23602294921875, + "loss": 0.1316, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.630185604095459, + "rewards/margins": 9.480209350585938, + "rewards/rejected": -14.110394477844238, + "step": 2291 + }, + { + "epoch": 3.0, + "learning_rate": 4.027819242278443e-05, + "logits/chosen": -2.086125135421753, + "logits/rejected": -2.167119026184082, + "logps/chosen": -212.73826599121094, + "logps/rejected": -309.79736328125, + "loss": 0.1633, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.7124667167663574, + "rewards/margins": 7.573567867279053, + "rewards/rejected": -11.28603458404541, + "step": 2292 + }, + { + "epoch": 3.0, + "learning_rate": 4.026994616589235e-05, + "logits/chosen": -2.16123366355896, + "logits/rejected": -2.154041290283203, + "logps/chosen": -294.1865234375, + "logps/rejected": -340.05108642578125, + "loss": 0.0444, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.380455017089844, + "rewards/margins": 8.062877655029297, + "rewards/rejected": -13.44333267211914, + "step": 2293 + }, + { + "epoch": 3.0, + "learning_rate": 4.026169725809289e-05, + "logits/chosen": -2.189936399459839, + "logits/rejected": -2.1102802753448486, + "logps/chosen": -210.06727600097656, + "logps/rejected": -263.0583190917969, + "loss": 0.1329, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.176320552825928, + "rewards/margins": 7.507213592529297, + "rewards/rejected": -11.68353271484375, + "step": 2294 + }, + { + "epoch": 3.0, + "learning_rate": 4.0253445700818086e-05, + "logits/chosen": -2.2092671394348145, + "logits/rejected": -2.2503600120544434, + "logps/chosen": -234.4791259765625, + "logps/rejected": -332.8056640625, + "loss": 0.0437, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.380105018615723, + "rewards/margins": 9.157236099243164, + "rewards/rejected": -13.537342071533203, + "step": 2295 + }, + { + "epoch": 3.0, + "learning_rate": 4.0245191495500445e-05, + "logits/chosen": -2.053370714187622, + "logits/rejected": -2.0985865592956543, + "logps/chosen": -239.59951782226562, + "logps/rejected": -299.9369812011719, + "loss": 0.0443, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.36566162109375, + "rewards/margins": 7.33683967590332, + "rewards/rejected": -11.70250129699707, + "step": 2296 + }, + { + "epoch": 3.01, + "learning_rate": 4.023693464357291e-05, + "logits/chosen": -2.1033010482788086, + "logits/rejected": -2.139561176300049, + "logps/chosen": -252.38595581054688, + "logps/rejected": -339.8417053222656, + "loss": 0.088, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.987560749053955, + "rewards/margins": 8.050565719604492, + "rewards/rejected": -13.038125991821289, + "step": 2297 + }, + { + "epoch": 3.01, + "learning_rate": 4.02286751464689e-05, + "logits/chosen": -2.361865520477295, + "logits/rejected": -2.4237165451049805, + "logps/chosen": -236.89031982421875, + "logps/rejected": -370.56610107421875, + "loss": 0.0436, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.124394416809082, + "rewards/margins": 10.090132713317871, + "rewards/rejected": -14.21452808380127, + "step": 2298 + }, + { + "epoch": 3.01, + "learning_rate": 4.022041300562228e-05, + "logits/chosen": -1.9574178457260132, + "logits/rejected": -2.0618011951446533, + "logps/chosen": -223.8952178955078, + "logps/rejected": -323.16400146484375, + "loss": 0.045, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.20233678817749, + "rewards/margins": 10.032905578613281, + "rewards/rejected": -14.235241889953613, + "step": 2299 + }, + { + "epoch": 3.01, + "learning_rate": 4.021214822246739e-05, + "logits/chosen": -2.163879156112671, + "logits/rejected": -2.2356607913970947, + "logps/chosen": -258.11541748046875, + "logps/rejected": -380.8958435058594, + "loss": 0.046, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.227643966674805, + "rewards/margins": 8.98115062713623, + "rewards/rejected": -15.208795547485352, + "step": 2300 + }, + { + "epoch": 3.01, + "learning_rate": 4.020388079843902e-05, + "logits/chosen": -2.112044334411621, + "logits/rejected": -2.1053316593170166, + "logps/chosen": -280.8301696777344, + "logps/rejected": -317.64324951171875, + "loss": 0.0007, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.152070045471191, + "rewards/margins": 8.259082794189453, + "rewards/rejected": -12.411153793334961, + "step": 2301 + }, + { + "epoch": 3.01, + "learning_rate": 4.019561073497241e-05, + "logits/chosen": -2.4246768951416016, + "logits/rejected": -2.440532684326172, + "logps/chosen": -282.09765625, + "logps/rejected": -370.8497314453125, + "loss": 0.0868, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.180196762084961, + "rewards/margins": 8.825722694396973, + "rewards/rejected": -14.00592041015625, + "step": 2302 + }, + { + "epoch": 3.01, + "learning_rate": 4.0187338033503275e-05, + "logits/chosen": -2.1134700775146484, + "logits/rejected": -2.1776392459869385, + "logps/chosen": -189.08233642578125, + "logps/rejected": -281.25531005859375, + "loss": 0.044, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.9960646629333496, + "rewards/margins": 8.551589965820312, + "rewards/rejected": -11.547654151916504, + "step": 2303 + }, + { + "epoch": 3.02, + "learning_rate": 4.0179062695467784e-05, + "logits/chosen": -2.1593093872070312, + "logits/rejected": -2.1546666622161865, + "logps/chosen": -264.7270812988281, + "logps/rejected": -367.3416748046875, + "loss": 0.0453, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.1254730224609375, + "rewards/margins": 7.990652084350586, + "rewards/rejected": -12.11612606048584, + "step": 2304 + }, + { + "epoch": 3.02, + "learning_rate": 4.017078472230255e-05, + "logits/chosen": -2.1838388442993164, + "logits/rejected": -2.2894444465637207, + "logps/chosen": -263.6777038574219, + "logps/rejected": -403.84674072265625, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.472953796386719, + "rewards/margins": 11.139884948730469, + "rewards/rejected": -15.612838745117188, + "step": 2305 + }, + { + "epoch": 3.02, + "learning_rate": 4.016250411544466e-05, + "logits/chosen": -1.8764675855636597, + "logits/rejected": -1.9426155090332031, + "logps/chosen": -236.81805419921875, + "logps/rejected": -368.4296875, + "loss": 0.0013, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.013107776641846, + "rewards/margins": 10.904794692993164, + "rewards/rejected": -15.917903900146484, + "step": 2306 + }, + { + "epoch": 3.02, + "learning_rate": 4.015422087633165e-05, + "logits/chosen": -2.1358773708343506, + "logits/rejected": -2.325611114501953, + "logps/chosen": -219.749755859375, + "logps/rejected": -342.8475646972656, + "loss": 0.0439, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.907919883728027, + "rewards/margins": 9.476095199584961, + "rewards/rejected": -14.384015083312988, + "step": 2307 + }, + { + "epoch": 3.02, + "learning_rate": 4.0145935006401516e-05, + "logits/chosen": -2.153437376022339, + "logits/rejected": -2.0697827339172363, + "logps/chosen": -264.7963562011719, + "logps/rejected": -315.2132873535156, + "loss": 0.0468, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.331108093261719, + "rewards/margins": 7.732646465301514, + "rewards/rejected": -13.06375503540039, + "step": 2308 + }, + { + "epoch": 3.02, + "learning_rate": 4.0137646507092706e-05, + "logits/chosen": -2.2929344177246094, + "logits/rejected": -2.3997268676757812, + "logps/chosen": -272.797119140625, + "logps/rejected": -399.6045837402344, + "loss": 0.0002, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.711956977844238, + "rewards/margins": 11.528371810913086, + "rewards/rejected": -16.240327835083008, + "step": 2309 + }, + { + "epoch": 3.02, + "learning_rate": 4.012935537984414e-05, + "logits/chosen": -2.355783224105835, + "logits/rejected": -2.362971544265747, + "logps/chosen": -311.9186706542969, + "logps/rejected": -427.03094482421875, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.685152053833008, + "rewards/margins": 10.624361038208008, + "rewards/rejected": -16.309513092041016, + "step": 2310 + }, + { + "epoch": 3.02, + "learning_rate": 4.0121061626095155e-05, + "logits/chosen": -2.1310548782348633, + "logits/rejected": -2.0894548892974854, + "logps/chosen": -234.70578002929688, + "logps/rejected": -341.914794921875, + "loss": 0.0438, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.285902976989746, + "rewards/margins": 8.906684875488281, + "rewards/rejected": -14.192588806152344, + "step": 2311 + }, + { + "epoch": 3.03, + "learning_rate": 4.0112765247285595e-05, + "logits/chosen": -2.230635643005371, + "logits/rejected": -2.2372360229492188, + "logps/chosen": -205.48318481445312, + "logps/rejected": -288.3927917480469, + "loss": 0.044, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.535229206085205, + "rewards/margins": 8.466339111328125, + "rewards/rejected": -13.001565933227539, + "step": 2312 + }, + { + "epoch": 3.03, + "learning_rate": 4.010446624485572e-05, + "logits/chosen": -2.2720067501068115, + "logits/rejected": -2.2585554122924805, + "logps/chosen": -296.4164123535156, + "logps/rejected": -356.4053039550781, + "loss": 0.0438, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.370475769042969, + "rewards/margins": 8.456291198730469, + "rewards/rejected": -13.826766014099121, + "step": 2313 + }, + { + "epoch": 3.03, + "learning_rate": 4.009616462024628e-05, + "logits/chosen": -2.039180278778076, + "logits/rejected": -2.0593509674072266, + "logps/chosen": -213.3941650390625, + "logps/rejected": -323.753173828125, + "loss": 0.0873, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.562514781951904, + "rewards/margins": 8.90079402923584, + "rewards/rejected": -13.463308334350586, + "step": 2314 + }, + { + "epoch": 3.03, + "learning_rate": 4.0087860374898446e-05, + "logits/chosen": -2.219299793243408, + "logits/rejected": -2.2711002826690674, + "logps/chosen": -201.03753662109375, + "logps/rejected": -316.4630126953125, + "loss": 0.087, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.167980670928955, + "rewards/margins": 8.080424308776855, + "rewards/rejected": -12.248404502868652, + "step": 2315 + }, + { + "epoch": 3.03, + "learning_rate": 4.007955351025387e-05, + "logits/chosen": -2.040846824645996, + "logits/rejected": -2.157193183898926, + "logps/chosen": -245.75889587402344, + "logps/rejected": -385.9333801269531, + "loss": 0.001, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.451557159423828, + "rewards/margins": 11.200770378112793, + "rewards/rejected": -16.652326583862305, + "step": 2316 + }, + { + "epoch": 3.03, + "learning_rate": 4.007124402775463e-05, + "logits/chosen": -2.101086378097534, + "logits/rejected": -2.293389320373535, + "logps/chosen": -208.45314025878906, + "logps/rejected": -346.80401611328125, + "loss": 0.0011, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.186187744140625, + "rewards/margins": 10.941791534423828, + "rewards/rejected": -16.127979278564453, + "step": 2317 + }, + { + "epoch": 3.03, + "learning_rate": 4.00629319288433e-05, + "logits/chosen": -1.848655343055725, + "logits/rejected": -1.9465444087982178, + "logps/chosen": -278.0640563964844, + "logps/rejected": -320.0218200683594, + "loss": 0.0014, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.153502941131592, + "rewards/margins": 7.768182754516602, + "rewards/rejected": -12.921686172485352, + "step": 2318 + }, + { + "epoch": 3.03, + "learning_rate": 4.005461721496286e-05, + "logits/chosen": -1.8471152782440186, + "logits/rejected": -1.9216175079345703, + "logps/chosen": -219.87152099609375, + "logps/rejected": -335.74462890625, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.6926662921905518, + "rewards/margins": 10.610774993896484, + "rewards/rejected": -14.30344295501709, + "step": 2319 + }, + { + "epoch": 3.04, + "learning_rate": 4.00462998875568e-05, + "logits/chosen": -2.17388916015625, + "logits/rejected": -2.2358853816986084, + "logps/chosen": -244.40786743164062, + "logps/rejected": -344.83514404296875, + "loss": 0.0064, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.317914009094238, + "rewards/margins": 9.465472221374512, + "rewards/rejected": -14.78338623046875, + "step": 2320 + }, + { + "epoch": 3.04, + "learning_rate": 4.0037979948069003e-05, + "logits/chosen": -2.2021055221557617, + "logits/rejected": -2.250926971435547, + "logps/chosen": -189.5944366455078, + "logps/rejected": -303.94281005859375, + "loss": 0.0008, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.41383695602417, + "rewards/margins": 9.51795768737793, + "rewards/rejected": -13.931794166564941, + "step": 2321 + }, + { + "epoch": 3.04, + "learning_rate": 4.002965739794385e-05, + "logits/chosen": -1.741159439086914, + "logits/rejected": -1.768996238708496, + "logps/chosen": -258.5458984375, + "logps/rejected": -308.5641784667969, + "loss": 0.044, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.688573360443115, + "rewards/margins": 8.116438865661621, + "rewards/rejected": -12.805011749267578, + "step": 2322 + }, + { + "epoch": 3.04, + "learning_rate": 4.002133223862615e-05, + "logits/chosen": -2.1633663177490234, + "logits/rejected": -2.1048080921173096, + "logps/chosen": -210.6006317138672, + "logps/rejected": -328.53546142578125, + "loss": 0.045, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.308881759643555, + "rewards/margins": 9.225309371948242, + "rewards/rejected": -13.534191131591797, + "step": 2323 + }, + { + "epoch": 3.04, + "learning_rate": 4.001300447156119e-05, + "logits/chosen": -1.9731532335281372, + "logits/rejected": -2.082718849182129, + "logps/chosen": -184.78358459472656, + "logps/rejected": -326.98468017578125, + "loss": 0.0437, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.704720497131348, + "rewards/margins": 9.48631763458252, + "rewards/rejected": -14.191038131713867, + "step": 2324 + }, + { + "epoch": 3.04, + "learning_rate": 4.0004674098194675e-05, + "logits/chosen": -2.027649402618408, + "logits/rejected": -2.1711618900299072, + "logps/chosen": -218.40965270996094, + "logps/rejected": -353.84210205078125, + "loss": 0.0438, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.636758804321289, + "rewards/margins": 8.774707794189453, + "rewards/rejected": -14.411465644836426, + "step": 2325 + }, + { + "epoch": 3.04, + "learning_rate": 3.9996341119972805e-05, + "logits/chosen": -2.0298938751220703, + "logits/rejected": -2.186424732208252, + "logps/chosen": -188.10995483398438, + "logps/rejected": -320.994140625, + "loss": 0.0019, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.797072410583496, + "rewards/margins": 9.481252670288086, + "rewards/rejected": -14.278324127197266, + "step": 2326 + }, + { + "epoch": 3.05, + "learning_rate": 3.998800553834219e-05, + "logits/chosen": -1.8152039051055908, + "logits/rejected": -1.8558685779571533, + "logps/chosen": -211.7963409423828, + "logps/rejected": -309.7165832519531, + "loss": 0.0569, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.820761203765869, + "rewards/margins": 10.48230266571045, + "rewards/rejected": -15.30306339263916, + "step": 2327 + }, + { + "epoch": 3.05, + "learning_rate": 3.9979667354749917e-05, + "logits/chosen": -2.2250239849090576, + "logits/rejected": -2.2089879512786865, + "logps/chosen": -243.59071350097656, + "logps/rejected": -316.2306213378906, + "loss": 0.0489, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.653672218322754, + "rewards/margins": 9.42144775390625, + "rewards/rejected": -14.075119018554688, + "step": 2328 + }, + { + "epoch": 3.05, + "learning_rate": 3.997132657064353e-05, + "logits/chosen": -1.8166918754577637, + "logits/rejected": -1.8703713417053223, + "logps/chosen": -256.21514892578125, + "logps/rejected": -344.1813049316406, + "loss": 0.0011, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.947235107421875, + "rewards/margins": 9.703486442565918, + "rewards/rejected": -14.65072250366211, + "step": 2329 + }, + { + "epoch": 3.05, + "learning_rate": 3.9962983187470994e-05, + "logits/chosen": -2.1914587020874023, + "logits/rejected": -2.188868761062622, + "logps/chosen": -211.61862182617188, + "logps/rejected": -278.4850158691406, + "loss": 0.09, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.627830505371094, + "rewards/margins": 7.591188430786133, + "rewards/rejected": -12.219019889831543, + "step": 2330 + }, + { + "epoch": 3.05, + "learning_rate": 3.995463720668076e-05, + "logits/chosen": -2.0139284133911133, + "logits/rejected": -1.872101068496704, + "logps/chosen": -264.0347595214844, + "logps/rejected": -338.0701904296875, + "loss": 0.0435, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.595579624176025, + "rewards/margins": 9.703751564025879, + "rewards/rejected": -15.299330711364746, + "step": 2331 + }, + { + "epoch": 3.05, + "learning_rate": 3.994628862972172e-05, + "logits/chosen": -2.0300374031066895, + "logits/rejected": -2.114473342895508, + "logps/chosen": -224.14810180664062, + "logps/rejected": -363.4672546386719, + "loss": 0.0867, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.546936988830566, + "rewards/margins": 9.266657829284668, + "rewards/rejected": -14.813594818115234, + "step": 2332 + }, + { + "epoch": 3.05, + "learning_rate": 3.993793745804319e-05, + "logits/chosen": -2.044935941696167, + "logits/rejected": -2.18890380859375, + "logps/chosen": -250.87396240234375, + "logps/rejected": -357.27008056640625, + "loss": 0.0014, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.594356060028076, + "rewards/margins": 9.66875171661377, + "rewards/rejected": -15.263107299804688, + "step": 2333 + }, + { + "epoch": 3.05, + "learning_rate": 3.9929583693094967e-05, + "logits/chosen": -2.0749664306640625, + "logits/rejected": -2.10062575340271, + "logps/chosen": -223.5476837158203, + "logps/rejected": -336.767578125, + "loss": 0.0435, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.567058563232422, + "rewards/margins": 9.905509948730469, + "rewards/rejected": -14.47256851196289, + "step": 2334 + }, + { + "epoch": 3.06, + "learning_rate": 3.9921227336327293e-05, + "logits/chosen": -2.249016284942627, + "logits/rejected": -2.2628750801086426, + "logps/chosen": -282.121826171875, + "logps/rejected": -331.7041015625, + "loss": 0.0437, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.705267429351807, + "rewards/margins": 9.401190757751465, + "rewards/rejected": -14.10645866394043, + "step": 2335 + }, + { + "epoch": 3.06, + "learning_rate": 3.991286838919086e-05, + "logits/chosen": -2.0916032791137695, + "logits/rejected": -2.139112949371338, + "logps/chosen": -226.25941467285156, + "logps/rejected": -367.0303649902344, + "loss": 0.0439, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.4411211013793945, + "rewards/margins": 11.064277648925781, + "rewards/rejected": -15.505398750305176, + "step": 2336 + }, + { + "epoch": 3.06, + "learning_rate": 3.9904506853136796e-05, + "logits/chosen": -2.2027699947357178, + "logits/rejected": -2.300292730331421, + "logps/chosen": -265.5385437011719, + "logps/rejected": -397.46527099609375, + "loss": 0.0449, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.148678779602051, + "rewards/margins": 9.942577362060547, + "rewards/rejected": -16.091257095336914, + "step": 2337 + }, + { + "epoch": 3.06, + "learning_rate": 3.989614272961669e-05, + "logits/chosen": -2.186069965362549, + "logits/rejected": -2.189955234527588, + "logps/chosen": -238.25157165527344, + "logps/rejected": -309.65679931640625, + "loss": 0.0442, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.0187602043151855, + "rewards/margins": 7.825289249420166, + "rewards/rejected": -13.844049453735352, + "step": 2338 + }, + { + "epoch": 3.06, + "learning_rate": 3.988777602008259e-05, + "logits/chosen": -2.190187931060791, + "logits/rejected": -2.195145845413208, + "logps/chosen": -252.57415771484375, + "logps/rejected": -355.80804443359375, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.373104572296143, + "rewards/margins": 11.054689407348633, + "rewards/rejected": -15.427794456481934, + "step": 2339 + }, + { + "epoch": 3.06, + "learning_rate": 3.9879406725986965e-05, + "logits/chosen": -1.855334997177124, + "logits/rejected": -1.9873031377792358, + "logps/chosen": -205.4321746826172, + "logps/rejected": -339.5233154296875, + "loss": 0.0868, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.916016578674316, + "rewards/margins": 8.870521545410156, + "rewards/rejected": -13.786538124084473, + "step": 2340 + }, + { + "epoch": 3.06, + "learning_rate": 3.987103484878275e-05, + "logits/chosen": -2.270313262939453, + "logits/rejected": -2.252146005630493, + "logps/chosen": -253.8924102783203, + "logps/rejected": -356.2812194824219, + "loss": 0.0435, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.975635528564453, + "rewards/margins": 9.70572566986084, + "rewards/rejected": -15.681361198425293, + "step": 2341 + }, + { + "epoch": 3.06, + "learning_rate": 3.986266038992335e-05, + "logits/chosen": -2.0620858669281006, + "logits/rejected": -2.1372265815734863, + "logps/chosen": -224.60153198242188, + "logps/rejected": -314.62744140625, + "loss": 0.044, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.936737060546875, + "rewards/margins": 9.658467292785645, + "rewards/rejected": -14.595205307006836, + "step": 2342 + }, + { + "epoch": 3.07, + "learning_rate": 3.9854283350862564e-05, + "logits/chosen": -2.163487434387207, + "logits/rejected": -2.112785816192627, + "logps/chosen": -256.3828125, + "logps/rejected": -359.2196044921875, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.521401405334473, + "rewards/margins": 9.490942001342773, + "rewards/rejected": -15.012344360351562, + "step": 2343 + }, + { + "epoch": 3.07, + "learning_rate": 3.984590373305469e-05, + "logits/chosen": -2.2192397117614746, + "logits/rejected": -2.2327206134796143, + "logps/chosen": -219.90838623046875, + "logps/rejected": -313.4751892089844, + "loss": 0.0006, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.904820442199707, + "rewards/margins": 9.546765327453613, + "rewards/rejected": -14.45158576965332, + "step": 2344 + }, + { + "epoch": 3.07, + "learning_rate": 3.983752153795445e-05, + "logits/chosen": -2.0529139041900635, + "logits/rejected": -2.1299502849578857, + "logps/chosen": -209.40573120117188, + "logps/rejected": -321.980224609375, + "loss": 0.0872, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.771730899810791, + "rewards/margins": 8.564140319824219, + "rewards/rejected": -14.335870742797852, + "step": 2345 + }, + { + "epoch": 3.07, + "learning_rate": 3.982913676701702e-05, + "logits/chosen": -2.1710922718048096, + "logits/rejected": -2.21834135055542, + "logps/chosen": -253.80889892578125, + "logps/rejected": -362.06597900390625, + "loss": 0.0009, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.591020107269287, + "rewards/margins": 10.34268569946289, + "rewards/rejected": -15.93370532989502, + "step": 2346 + }, + { + "epoch": 3.07, + "learning_rate": 3.982074942169801e-05, + "logits/chosen": -2.0652060508728027, + "logits/rejected": -2.0282676219940186, + "logps/chosen": -297.8626708984375, + "logps/rejected": -414.75244140625, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.402783393859863, + "rewards/margins": 10.836182594299316, + "rewards/rejected": -16.23896598815918, + "step": 2347 + }, + { + "epoch": 3.07, + "learning_rate": 3.98123595034535e-05, + "logits/chosen": -1.8497111797332764, + "logits/rejected": -1.9697858095169067, + "logps/chosen": -196.43084716796875, + "logps/rejected": -325.74005126953125, + "loss": 0.0867, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.046850204467773, + "rewards/margins": 9.167963027954102, + "rewards/rejected": -14.214814186096191, + "step": 2348 + }, + { + "epoch": 3.07, + "learning_rate": 3.980396701374001e-05, + "logits/chosen": -2.0902960300445557, + "logits/rejected": -2.1278557777404785, + "logps/chosen": -211.67010498046875, + "logps/rejected": -327.716552734375, + "loss": 0.0019, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.313720226287842, + "rewards/margins": 9.680084228515625, + "rewards/rejected": -14.993803977966309, + "step": 2349 + }, + { + "epoch": 3.08, + "learning_rate": 3.979557195401447e-05, + "logits/chosen": -2.1743905544281006, + "logits/rejected": -2.2324106693267822, + "logps/chosen": -234.66226196289062, + "logps/rejected": -340.6358947753906, + "loss": 0.0438, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.393934726715088, + "rewards/margins": 8.925092697143555, + "rewards/rejected": -14.319025039672852, + "step": 2350 + }, + { + "epoch": 3.08, + "learning_rate": 3.978717432573432e-05, + "logits/chosen": -1.8415966033935547, + "logits/rejected": -1.935300350189209, + "logps/chosen": -213.11997985839844, + "logps/rejected": -358.7032470703125, + "loss": 0.003, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.9846391677856445, + "rewards/margins": 10.274259567260742, + "rewards/rejected": -16.258899688720703, + "step": 2351 + }, + { + "epoch": 3.08, + "learning_rate": 3.9778774130357396e-05, + "logits/chosen": -2.091505527496338, + "logits/rejected": -2.1661930084228516, + "logps/chosen": -209.1724853515625, + "logps/rejected": -393.89544677734375, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.482723236083984, + "rewards/margins": 10.429054260253906, + "rewards/rejected": -16.911779403686523, + "step": 2352 + }, + { + "epoch": 3.08, + "learning_rate": 3.977037136934201e-05, + "logits/chosen": -2.0253586769104004, + "logits/rejected": -2.018190622329712, + "logps/chosen": -230.11798095703125, + "logps/rejected": -301.07843017578125, + "loss": 0.0868, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.083752155303955, + "rewards/margins": 9.406051635742188, + "rewards/rejected": -14.489802360534668, + "step": 2353 + }, + { + "epoch": 3.08, + "learning_rate": 3.976196604414689e-05, + "logits/chosen": -1.952399730682373, + "logits/rejected": -1.9626762866973877, + "logps/chosen": -261.0067138671875, + "logps/rejected": -397.1845397949219, + "loss": 0.001, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.120563983917236, + "rewards/margins": 10.943238258361816, + "rewards/rejected": -16.06380271911621, + "step": 2354 + }, + { + "epoch": 3.08, + "learning_rate": 3.975355815623124e-05, + "logits/chosen": -1.919739007949829, + "logits/rejected": -1.9575756788253784, + "logps/chosen": -195.85140991210938, + "logps/rejected": -286.96673583984375, + "loss": 0.0439, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.722510814666748, + "rewards/margins": 8.700567245483398, + "rewards/rejected": -13.423078536987305, + "step": 2355 + }, + { + "epoch": 3.08, + "learning_rate": 3.974514770705468e-05, + "logits/chosen": -2.0669007301330566, + "logits/rejected": -1.9612680673599243, + "logps/chosen": -219.40257263183594, + "logps/rejected": -338.7507629394531, + "loss": 0.0479, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.551650524139404, + "rewards/margins": 9.081823348999023, + "rewards/rejected": -14.633474349975586, + "step": 2356 + }, + { + "epoch": 3.08, + "learning_rate": 3.973673469807729e-05, + "logits/chosen": -2.1750898361206055, + "logits/rejected": -2.220430612564087, + "logps/chosen": -227.869384765625, + "logps/rejected": -315.6466064453125, + "loss": 0.0012, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.440352916717529, + "rewards/margins": 9.903848648071289, + "rewards/rejected": -15.344200134277344, + "step": 2357 + }, + { + "epoch": 3.09, + "learning_rate": 3.972831913075961e-05, + "logits/chosen": -2.1814780235290527, + "logits/rejected": -2.2241060733795166, + "logps/chosen": -233.10333251953125, + "logps/rejected": -341.2297668457031, + "loss": 0.0528, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.034167289733887, + "rewards/margins": 9.180243492126465, + "rewards/rejected": -15.214411735534668, + "step": 2358 + }, + { + "epoch": 3.09, + "learning_rate": 3.971990100656258e-05, + "logits/chosen": -2.333792209625244, + "logits/rejected": -2.349745035171509, + "logps/chosen": -231.09255981445312, + "logps/rejected": -278.3640441894531, + "loss": 0.132, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.1729912757873535, + "rewards/margins": 7.631937026977539, + "rewards/rejected": -12.80492877960205, + "step": 2359 + }, + { + "epoch": 3.09, + "learning_rate": 3.9711480326947636e-05, + "logits/chosen": -1.8499326705932617, + "logits/rejected": -1.9601032733917236, + "logps/chosen": -144.95199584960938, + "logps/rejected": -281.6170654296875, + "loss": 0.1316, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.8826472759246826, + "rewards/margins": 8.919089317321777, + "rewards/rejected": -12.801735877990723, + "step": 2360 + }, + { + "epoch": 3.09, + "learning_rate": 3.970305709337662e-05, + "logits/chosen": -1.744267463684082, + "logits/rejected": -1.7960014343261719, + "logps/chosen": -278.4100646972656, + "logps/rejected": -379.42315673828125, + "loss": 0.05, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.190540790557861, + "rewards/margins": 8.991704940795898, + "rewards/rejected": -15.182245254516602, + "step": 2361 + }, + { + "epoch": 3.09, + "learning_rate": 3.969463130731183e-05, + "logits/chosen": -1.9131689071655273, + "logits/rejected": -1.9178646802902222, + "logps/chosen": -238.28317260742188, + "logps/rejected": -336.04132080078125, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.83204460144043, + "rewards/margins": 9.937345504760742, + "rewards/rejected": -15.769390106201172, + "step": 2362 + }, + { + "epoch": 3.09, + "learning_rate": 3.968620297021601e-05, + "logits/chosen": -2.2412827014923096, + "logits/rejected": -2.295560598373413, + "logps/chosen": -256.8214111328125, + "logps/rejected": -343.5921936035156, + "loss": 0.0568, + "rewards/accuracies": 0.9375, + "rewards/chosen": -7.132884502410889, + "rewards/margins": 8.71896743774414, + "rewards/rejected": -15.851852416992188, + "step": 2363 + }, + { + "epoch": 3.09, + "learning_rate": 3.9677772083552345e-05, + "logits/chosen": -1.8722002506256104, + "logits/rejected": -2.006021499633789, + "logps/chosen": -205.1575469970703, + "logps/rejected": -341.50830078125, + "loss": 0.0436, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.86874532699585, + "rewards/margins": 10.063451766967773, + "rewards/rejected": -14.932196617126465, + "step": 2364 + }, + { + "epoch": 3.1, + "learning_rate": 3.966933864878446e-05, + "logits/chosen": -2.046663522720337, + "logits/rejected": -2.1174910068511963, + "logps/chosen": -206.85365295410156, + "logps/rejected": -294.822998046875, + "loss": 0.1311, + "rewards/accuracies": 0.8125, + "rewards/chosen": -6.025838375091553, + "rewards/margins": 7.51859188079834, + "rewards/rejected": -13.54443073272705, + "step": 2365 + }, + { + "epoch": 3.1, + "learning_rate": 3.966090266737643e-05, + "logits/chosen": -2.019179582595825, + "logits/rejected": -1.9590449333190918, + "logps/chosen": -188.83555603027344, + "logps/rejected": -286.6895751953125, + "loss": 0.0448, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.439995288848877, + "rewards/margins": 9.44103717803955, + "rewards/rejected": -13.881032943725586, + "step": 2366 + }, + { + "epoch": 3.1, + "learning_rate": 3.9652464140792754e-05, + "logits/chosen": -1.8752981424331665, + "logits/rejected": -2.010054588317871, + "logps/chosen": -241.266845703125, + "logps/rejected": -383.7904357910156, + "loss": 0.0436, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.881939888000488, + "rewards/margins": 10.096649169921875, + "rewards/rejected": -16.978588104248047, + "step": 2367 + }, + { + "epoch": 3.1, + "learning_rate": 3.9644023070498394e-05, + "logits/chosen": -1.8323734998703003, + "logits/rejected": -1.8427077531814575, + "logps/chosen": -206.6799774169922, + "logps/rejected": -313.88092041015625, + "loss": 0.0877, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.858495235443115, + "rewards/margins": 8.448741912841797, + "rewards/rejected": -13.30723762512207, + "step": 2368 + }, + { + "epoch": 3.1, + "learning_rate": 3.963557945795874e-05, + "logits/chosen": -2.239041328430176, + "logits/rejected": -2.2368831634521484, + "logps/chosen": -275.11798095703125, + "logps/rejected": -377.6287841796875, + "loss": 0.0454, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.356086730957031, + "rewards/margins": 10.835125923156738, + "rewards/rejected": -17.191213607788086, + "step": 2369 + }, + { + "epoch": 3.1, + "learning_rate": 3.962713330463963e-05, + "logits/chosen": -2.091503620147705, + "logits/rejected": -2.0818939208984375, + "logps/chosen": -335.230224609375, + "logps/rejected": -415.01788330078125, + "loss": 0.0868, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.815808296203613, + "rewards/margins": 8.49116325378418, + "rewards/rejected": -14.306971549987793, + "step": 2370 + }, + { + "epoch": 3.1, + "learning_rate": 3.961868461200734e-05, + "logits/chosen": -2.1107940673828125, + "logits/rejected": -2.1707143783569336, + "logps/chosen": -236.53013610839844, + "logps/rejected": -324.61712646484375, + "loss": 0.0524, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.84749698638916, + "rewards/margins": 9.747594833374023, + "rewards/rejected": -15.595091819763184, + "step": 2371 + }, + { + "epoch": 3.1, + "learning_rate": 3.961023338152859e-05, + "logits/chosen": -2.094388008117676, + "logits/rejected": -2.1055898666381836, + "logps/chosen": -271.1788635253906, + "logps/rejected": -337.4134216308594, + "loss": 0.044, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.544390678405762, + "rewards/margins": 8.601953506469727, + "rewards/rejected": -14.146343231201172, + "step": 2372 + }, + { + "epoch": 3.11, + "learning_rate": 3.960177961467053e-05, + "logits/chosen": -1.7063359022140503, + "logits/rejected": -1.7682796716690063, + "logps/chosen": -256.10052490234375, + "logps/rejected": -396.14739990234375, + "loss": 0.0436, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.337587356567383, + "rewards/margins": 10.831178665161133, + "rewards/rejected": -17.168766021728516, + "step": 2373 + }, + { + "epoch": 3.11, + "learning_rate": 3.9593323312900776e-05, + "logits/chosen": -1.904474139213562, + "logits/rejected": -1.9390546083450317, + "logps/chosen": -229.9960174560547, + "logps/rejected": -338.8091735839844, + "loss": 0.0868, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.166641712188721, + "rewards/margins": 8.765958786010742, + "rewards/rejected": -14.932600975036621, + "step": 2374 + }, + { + "epoch": 3.11, + "learning_rate": 3.958486447768736e-05, + "logits/chosen": -2.048222064971924, + "logits/rejected": -2.0803263187408447, + "logps/chosen": -253.60183715820312, + "logps/rejected": -351.0816345214844, + "loss": 0.0014, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.731586933135986, + "rewards/margins": 10.426437377929688, + "rewards/rejected": -16.15802574157715, + "step": 2375 + }, + { + "epoch": 3.11, + "learning_rate": 3.957640311049874e-05, + "logits/chosen": -2.3471922874450684, + "logits/rejected": -2.3304343223571777, + "logps/chosen": -250.97804260253906, + "logps/rejected": -310.53143310546875, + "loss": 0.0462, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.121223449707031, + "rewards/margins": 8.97576904296875, + "rewards/rejected": -14.096992492675781, + "step": 2376 + }, + { + "epoch": 3.11, + "learning_rate": 3.956793921280386e-05, + "logits/chosen": -2.0674235820770264, + "logits/rejected": -2.17264461517334, + "logps/chosen": -251.26593017578125, + "logps/rejected": -397.5065612792969, + "loss": 0.0434, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.873909950256348, + "rewards/margins": 11.099771499633789, + "rewards/rejected": -16.973682403564453, + "step": 2377 + }, + { + "epoch": 3.11, + "learning_rate": 3.9559472786072064e-05, + "logits/chosen": -1.8911620378494263, + "logits/rejected": -2.07753324508667, + "logps/chosen": -220.3954620361328, + "logps/rejected": -392.901611328125, + "loss": 0.0436, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.4543890953063965, + "rewards/margins": 11.461380004882812, + "rewards/rejected": -16.915769577026367, + "step": 2378 + }, + { + "epoch": 3.11, + "learning_rate": 3.955100383177316e-05, + "logits/chosen": -2.0628786087036133, + "logits/rejected": -2.064690351486206, + "logps/chosen": -277.8653564453125, + "logps/rejected": -342.07745361328125, + "loss": 0.0885, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.13007926940918, + "rewards/margins": 7.721760272979736, + "rewards/rejected": -12.851840019226074, + "step": 2379 + }, + { + "epoch": 3.11, + "learning_rate": 3.954253235137737e-05, + "logits/chosen": -2.1326215267181396, + "logits/rejected": -2.1177854537963867, + "logps/chosen": -293.449462890625, + "logps/rejected": -377.8431091308594, + "loss": 0.0005, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.3262529373168945, + "rewards/margins": 10.278807640075684, + "rewards/rejected": -15.605061531066895, + "step": 2380 + }, + { + "epoch": 3.12, + "learning_rate": 3.9534058346355374e-05, + "logits/chosen": -2.0535805225372314, + "logits/rejected": -2.0691559314727783, + "logps/chosen": -250.04638671875, + "logps/rejected": -352.6424560546875, + "loss": 0.0873, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.745578765869141, + "rewards/margins": 8.711980819702148, + "rewards/rejected": -14.457559585571289, + "step": 2381 + }, + { + "epoch": 3.12, + "learning_rate": 3.952558181817829e-05, + "logits/chosen": -2.056554079055786, + "logits/rejected": -2.154934883117676, + "logps/chosen": -237.72950744628906, + "logps/rejected": -355.5850830078125, + "loss": 0.0875, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.244574546813965, + "rewards/margins": 8.941149711608887, + "rewards/rejected": -15.185724258422852, + "step": 2382 + }, + { + "epoch": 3.12, + "learning_rate": 3.9517102768317654e-05, + "logits/chosen": -1.9855201244354248, + "logits/rejected": -2.192671060562134, + "logps/chosen": -253.00885009765625, + "logps/rejected": -392.8494873046875, + "loss": 0.0435, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.224951267242432, + "rewards/margins": 10.622590065002441, + "rewards/rejected": -16.847543716430664, + "step": 2383 + }, + { + "epoch": 3.12, + "learning_rate": 3.9508621198245467e-05, + "logits/chosen": -2.1092135906219482, + "logits/rejected": -2.1657583713531494, + "logps/chosen": -221.89297485351562, + "logps/rejected": -368.66650390625, + "loss": 0.0435, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.002373695373535, + "rewards/margins": 10.422560691833496, + "rewards/rejected": -15.424935340881348, + "step": 2384 + }, + { + "epoch": 3.12, + "learning_rate": 3.950013710943415e-05, + "logits/chosen": -2.1366307735443115, + "logits/rejected": -2.127321720123291, + "logps/chosen": -258.19921875, + "logps/rejected": -347.7055358886719, + "loss": 0.0007, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.898001670837402, + "rewards/margins": 10.462735176086426, + "rewards/rejected": -15.360738754272461, + "step": 2385 + }, + { + "epoch": 3.12, + "learning_rate": 3.9491650503356556e-05, + "logits/chosen": -2.008326530456543, + "logits/rejected": -1.9478611946105957, + "logps/chosen": -232.56173706054688, + "logps/rejected": -329.6402893066406, + "loss": 0.0435, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.509620666503906, + "rewards/margins": 9.250005722045898, + "rewards/rejected": -14.759627342224121, + "step": 2386 + }, + { + "epoch": 3.12, + "learning_rate": 3.9483161381486e-05, + "logits/chosen": -1.8713055849075317, + "logits/rejected": -1.9137439727783203, + "logps/chosen": -239.3643798828125, + "logps/rejected": -343.9569396972656, + "loss": 0.1734, + "rewards/accuracies": 0.8125, + "rewards/chosen": -6.358916759490967, + "rewards/margins": 7.795135974884033, + "rewards/rejected": -14.154053688049316, + "step": 2387 + }, + { + "epoch": 3.13, + "learning_rate": 3.9474669745296225e-05, + "logits/chosen": -2.0420758724212646, + "logits/rejected": -2.1169066429138184, + "logps/chosen": -265.346923828125, + "logps/rejected": -364.44671630859375, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.088667392730713, + "rewards/margins": 9.89621353149414, + "rewards/rejected": -15.984879493713379, + "step": 2388 + }, + { + "epoch": 3.13, + "learning_rate": 3.946617559626137e-05, + "logits/chosen": -1.9690918922424316, + "logits/rejected": -1.896228313446045, + "logps/chosen": -244.475341796875, + "logps/rejected": -319.3477783203125, + "loss": 0.0868, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.9726781845092773, + "rewards/margins": 9.650399208068848, + "rewards/rejected": -13.623077392578125, + "step": 2389 + }, + { + "epoch": 3.13, + "learning_rate": 3.945767893585608e-05, + "logits/chosen": -1.9241656064987183, + "logits/rejected": -2.1013247966766357, + "logps/chosen": -210.3724365234375, + "logps/rejected": -335.97918701171875, + "loss": 0.0871, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.849796295166016, + "rewards/margins": 9.42531967163086, + "rewards/rejected": -15.275117874145508, + "step": 2390 + }, + { + "epoch": 3.13, + "learning_rate": 3.9449179765555375e-05, + "logits/chosen": -2.0780909061431885, + "logits/rejected": -2.051337718963623, + "logps/chosen": -258.7431335449219, + "logps/rejected": -354.255859375, + "loss": 0.0871, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.664587497711182, + "rewards/margins": 9.591286659240723, + "rewards/rejected": -15.25587272644043, + "step": 2391 + }, + { + "epoch": 3.13, + "learning_rate": 3.944067808683475e-05, + "logits/chosen": -2.2163031101226807, + "logits/rejected": -2.2428455352783203, + "logps/chosen": -234.6023712158203, + "logps/rejected": -338.6448059082031, + "loss": 0.0435, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.279127597808838, + "rewards/margins": 9.972273826599121, + "rewards/rejected": -16.251399993896484, + "step": 2392 + }, + { + "epoch": 3.13, + "learning_rate": 3.943217390117012e-05, + "logits/chosen": -2.0962891578674316, + "logits/rejected": -2.1283798217773438, + "logps/chosen": -218.8201141357422, + "logps/rejected": -331.392578125, + "loss": 0.0437, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.476904392242432, + "rewards/margins": 9.991121292114258, + "rewards/rejected": -15.468027114868164, + "step": 2393 + }, + { + "epoch": 3.13, + "learning_rate": 3.942366721003783e-05, + "logits/chosen": -2.068873405456543, + "logits/rejected": -2.2088582515716553, + "logps/chosen": -222.7272186279297, + "logps/rejected": -379.4482421875, + "loss": 0.0002, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.834370136260986, + "rewards/margins": 11.389117240905762, + "rewards/rejected": -16.223487854003906, + "step": 2394 + }, + { + "epoch": 3.13, + "learning_rate": 3.941515801491468e-05, + "logits/chosen": -2.146162509918213, + "logits/rejected": -2.0940680503845215, + "logps/chosen": -268.0540771484375, + "logps/rejected": -352.9101867675781, + "loss": 0.0438, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.3566179275512695, + "rewards/margins": 10.106650352478027, + "rewards/rejected": -15.463268280029297, + "step": 2395 + }, + { + "epoch": 3.14, + "learning_rate": 3.940664631727787e-05, + "logits/chosen": -2.075049638748169, + "logits/rejected": -2.1905934810638428, + "logps/chosen": -263.863037109375, + "logps/rejected": -367.4610595703125, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.505448341369629, + "rewards/margins": 10.4701566696167, + "rewards/rejected": -15.975607872009277, + "step": 2396 + }, + { + "epoch": 3.14, + "learning_rate": 3.939813211860507e-05, + "logits/chosen": -2.1090781688690186, + "logits/rejected": -2.1186399459838867, + "logps/chosen": -218.42019653320312, + "logps/rejected": -285.690185546875, + "loss": 0.0887, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.306107997894287, + "rewards/margins": 7.967566013336182, + "rewards/rejected": -13.273674964904785, + "step": 2397 + }, + { + "epoch": 3.14, + "learning_rate": 3.938961542037437e-05, + "logits/chosen": -1.9859368801116943, + "logits/rejected": -1.9667267799377441, + "logps/chosen": -231.34140014648438, + "logps/rejected": -309.3080749511719, + "loss": 0.0011, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.32011604309082, + "rewards/margins": 10.557493209838867, + "rewards/rejected": -14.877609252929688, + "step": 2398 + }, + { + "epoch": 3.14, + "learning_rate": 3.938109622406429e-05, + "logits/chosen": -1.7378034591674805, + "logits/rejected": -1.7265030145645142, + "logps/chosen": -252.15228271484375, + "logps/rejected": -382.1368408203125, + "loss": 0.0449, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.843169689178467, + "rewards/margins": 9.757123947143555, + "rewards/rejected": -14.60029411315918, + "step": 2399 + }, + { + "epoch": 3.14, + "learning_rate": 3.937257453115378e-05, + "logits/chosen": -1.988635540008545, + "logits/rejected": -1.9777966737747192, + "logps/chosen": -213.48365783691406, + "logps/rejected": -334.8724365234375, + "loss": 0.0012, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.516007423400879, + "rewards/margins": 9.487176895141602, + "rewards/rejected": -15.003185272216797, + "step": 2400 + }, + { + "epoch": 3.14, + "learning_rate": 3.9364050343122254e-05, + "logits/chosen": -1.9818344116210938, + "logits/rejected": -2.037508726119995, + "logps/chosen": -187.1419677734375, + "logps/rejected": -290.38812255859375, + "loss": 0.0052, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.9792304039001465, + "rewards/margins": 9.056032180786133, + "rewards/rejected": -14.035264015197754, + "step": 2401 + }, + { + "epoch": 3.14, + "learning_rate": 3.9355523661449515e-05, + "logits/chosen": -1.9677616357803345, + "logits/rejected": -2.0162582397460938, + "logps/chosen": -224.53501892089844, + "logps/rejected": -340.9477844238281, + "loss": 0.0436, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.842019557952881, + "rewards/margins": 12.349411010742188, + "rewards/rejected": -16.191429138183594, + "step": 2402 + }, + { + "epoch": 3.14, + "learning_rate": 3.9346994487615826e-05, + "logits/chosen": -2.144747495651245, + "logits/rejected": -2.187472105026245, + "logps/chosen": -257.41339111328125, + "logps/rejected": -309.4937438964844, + "loss": 0.0907, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.622319221496582, + "rewards/margins": 7.701298713684082, + "rewards/rejected": -14.323617935180664, + "step": 2403 + }, + { + "epoch": 3.15, + "learning_rate": 3.933846282310187e-05, + "logits/chosen": -2.13328218460083, + "logits/rejected": -2.134481906890869, + "logps/chosen": -246.9745330810547, + "logps/rejected": -361.6680908203125, + "loss": 0.0883, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.423940658569336, + "rewards/margins": 9.740814208984375, + "rewards/rejected": -16.16475486755371, + "step": 2404 + }, + { + "epoch": 3.15, + "learning_rate": 3.932992866938878e-05, + "logits/chosen": -2.129410743713379, + "logits/rejected": -2.046457052230835, + "logps/chosen": -315.9283752441406, + "logps/rejected": -406.8256530761719, + "loss": 0.0009, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.326528072357178, + "rewards/margins": 11.082746505737305, + "rewards/rejected": -17.40927505493164, + "step": 2405 + }, + { + "epoch": 3.15, + "learning_rate": 3.9321392027958116e-05, + "logits/chosen": -1.7999567985534668, + "logits/rejected": -1.8146796226501465, + "logps/chosen": -223.77041625976562, + "logps/rejected": -356.35107421875, + "loss": 0.0867, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.859269142150879, + "rewards/margins": 11.880956649780273, + "rewards/rejected": -15.740224838256836, + "step": 2406 + }, + { + "epoch": 3.15, + "learning_rate": 3.931285290029184e-05, + "logits/chosen": -2.105497360229492, + "logits/rejected": -2.0085368156433105, + "logps/chosen": -319.6090087890625, + "logps/rejected": -396.0473327636719, + "loss": 0.0869, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.423470497131348, + "rewards/margins": 8.27132511138916, + "rewards/rejected": -13.694795608520508, + "step": 2407 + }, + { + "epoch": 3.15, + "learning_rate": 3.930431128787239e-05, + "logits/chosen": -1.7433183193206787, + "logits/rejected": -1.8397947549819946, + "logps/chosen": -212.28330993652344, + "logps/rejected": -334.6816711425781, + "loss": 0.0873, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.340822696685791, + "rewards/margins": 9.099462509155273, + "rewards/rejected": -14.440284729003906, + "step": 2408 + }, + { + "epoch": 3.15, + "learning_rate": 3.929576719218261e-05, + "logits/chosen": -2.2288548946380615, + "logits/rejected": -2.113827705383301, + "logps/chosen": -289.6617126464844, + "logps/rejected": -354.66961669921875, + "loss": 0.0006, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.691329002380371, + "rewards/margins": 9.017830848693848, + "rewards/rejected": -15.709159851074219, + "step": 2409 + }, + { + "epoch": 3.15, + "learning_rate": 3.928722061470577e-05, + "logits/chosen": -2.0128843784332275, + "logits/rejected": -2.063493490219116, + "logps/chosen": -241.12109375, + "logps/rejected": -398.9261474609375, + "loss": 0.0435, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.745203495025635, + "rewards/margins": 12.559578895568848, + "rewards/rejected": -18.304780960083008, + "step": 2410 + }, + { + "epoch": 3.16, + "learning_rate": 3.9278671556925594e-05, + "logits/chosen": -1.885010838508606, + "logits/rejected": -2.011854410171509, + "logps/chosen": -254.62257385253906, + "logps/rejected": -302.7500915527344, + "loss": 0.087, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.52474308013916, + "rewards/margins": 8.861490249633789, + "rewards/rejected": -14.386234283447266, + "step": 2411 + }, + { + "epoch": 3.16, + "learning_rate": 3.927012002032622e-05, + "logits/chosen": -2.013869285583496, + "logits/rejected": -2.0908093452453613, + "logps/chosen": -255.79388427734375, + "logps/rejected": -385.9837951660156, + "loss": 0.0439, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.909495830535889, + "rewards/margins": 8.987797737121582, + "rewards/rejected": -15.897294044494629, + "step": 2412 + }, + { + "epoch": 3.16, + "learning_rate": 3.926156600639222e-05, + "logits/chosen": -1.8484530448913574, + "logits/rejected": -1.924649715423584, + "logps/chosen": -285.5309753417969, + "logps/rejected": -396.7544860839844, + "loss": 0.0869, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.703119277954102, + "rewards/margins": 9.599272727966309, + "rewards/rejected": -15.30239200592041, + "step": 2413 + }, + { + "epoch": 3.16, + "learning_rate": 3.925300951660858e-05, + "logits/chosen": -2.017725944519043, + "logits/rejected": -2.0484378337860107, + "logps/chosen": -249.70545959472656, + "logps/rejected": -306.10150146484375, + "loss": 0.0436, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.9575750827789307, + "rewards/margins": 10.65237808227539, + "rewards/rejected": -14.609952926635742, + "step": 2414 + }, + { + "epoch": 3.16, + "learning_rate": 3.924445055246076e-05, + "logits/chosen": -2.3275504112243652, + "logits/rejected": -2.4170687198638916, + "logps/chosen": -232.57530212402344, + "logps/rejected": -347.41302490234375, + "loss": 0.1307, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.247507095336914, + "rewards/margins": 7.428805351257324, + "rewards/rejected": -12.676311492919922, + "step": 2415 + }, + { + "epoch": 3.16, + "learning_rate": 3.923588911543461e-05, + "logits/chosen": -1.910735845565796, + "logits/rejected": -1.9588009119033813, + "logps/chosen": -210.57037353515625, + "logps/rejected": -340.1056823730469, + "loss": 0.0008, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.114107131958008, + "rewards/margins": 10.13116455078125, + "rewards/rejected": -15.245270729064941, + "step": 2416 + }, + { + "epoch": 3.16, + "learning_rate": 3.922732520701641e-05, + "logits/chosen": -1.9737759828567505, + "logits/rejected": -2.0073487758636475, + "logps/chosen": -234.25225830078125, + "logps/rejected": -329.5049133300781, + "loss": 0.0438, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.450007438659668, + "rewards/margins": 10.389273643493652, + "rewards/rejected": -15.839282989501953, + "step": 2417 + }, + { + "epoch": 3.16, + "learning_rate": 3.9218758828692885e-05, + "logits/chosen": -1.8982748985290527, + "logits/rejected": -2.017169713973999, + "logps/chosen": -219.2686004638672, + "logps/rejected": -355.2296447753906, + "loss": 0.0871, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.760926246643066, + "rewards/margins": 9.56212043762207, + "rewards/rejected": -15.323046684265137, + "step": 2418 + }, + { + "epoch": 3.17, + "learning_rate": 3.92101899819512e-05, + "logits/chosen": -2.09666109085083, + "logits/rejected": -2.1102030277252197, + "logps/chosen": -291.5617370605469, + "logps/rejected": -396.4189147949219, + "loss": 0.0436, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.530086994171143, + "rewards/margins": 10.002303123474121, + "rewards/rejected": -15.532389640808105, + "step": 2419 + }, + { + "epoch": 3.17, + "learning_rate": 3.920161866827889e-05, + "logits/chosen": -2.2040367126464844, + "logits/rejected": -2.184612989425659, + "logps/chosen": -201.29171752929688, + "logps/rejected": -306.67633056640625, + "loss": 0.1312, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.141445159912109, + "rewards/margins": 9.965995788574219, + "rewards/rejected": -15.107441902160645, + "step": 2420 + }, + { + "epoch": 3.17, + "learning_rate": 3.919304488916401e-05, + "logits/chosen": -2.039891004562378, + "logits/rejected": -2.046557903289795, + "logps/chosen": -279.17388916015625, + "logps/rejected": -368.23779296875, + "loss": 0.0643, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.187160015106201, + "rewards/margins": 9.28410530090332, + "rewards/rejected": -14.47126579284668, + "step": 2421 + }, + { + "epoch": 3.17, + "learning_rate": 3.918446864609496e-05, + "logits/chosen": -1.8513938188552856, + "logits/rejected": -1.8851006031036377, + "logps/chosen": -244.12559509277344, + "logps/rejected": -359.91925048828125, + "loss": 0.0434, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.379150390625, + "rewards/margins": 11.033041000366211, + "rewards/rejected": -17.41219139099121, + "step": 2422 + }, + { + "epoch": 3.17, + "learning_rate": 3.9175889940560626e-05, + "logits/chosen": -2.183446168899536, + "logits/rejected": -2.1999473571777344, + "logps/chosen": -245.6224365234375, + "logps/rejected": -338.06915283203125, + "loss": 0.0463, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.378600120544434, + "rewards/margins": 9.645833015441895, + "rewards/rejected": -15.024433135986328, + "step": 2423 + }, + { + "epoch": 3.17, + "learning_rate": 3.916730877405027e-05, + "logits/chosen": -2.3098859786987305, + "logits/rejected": -2.371561050415039, + "logps/chosen": -296.7130126953125, + "logps/rejected": -389.8238525390625, + "loss": 0.0448, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.905816555023193, + "rewards/margins": 10.16203784942627, + "rewards/rejected": -17.067853927612305, + "step": 2424 + }, + { + "epoch": 3.17, + "learning_rate": 3.9158725148053624e-05, + "logits/chosen": -1.7294484376907349, + "logits/rejected": -1.927701711654663, + "logps/chosen": -254.4145965576172, + "logps/rejected": -390.72906494140625, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.999884605407715, + "rewards/margins": 10.069520950317383, + "rewards/rejected": -17.06940460205078, + "step": 2425 + }, + { + "epoch": 3.17, + "learning_rate": 3.915013906406083e-05, + "logits/chosen": -2.015231132507324, + "logits/rejected": -1.955536961555481, + "logps/chosen": -217.324462890625, + "logps/rejected": -323.7929992675781, + "loss": 0.0873, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.474260330200195, + "rewards/margins": 8.68239974975586, + "rewards/rejected": -14.156660079956055, + "step": 2426 + }, + { + "epoch": 3.18, + "learning_rate": 3.9141550523562443e-05, + "logits/chosen": -2.0247368812561035, + "logits/rejected": -2.090301275253296, + "logps/chosen": -266.1253356933594, + "logps/rejected": -333.9754943847656, + "loss": 0.0873, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.677989482879639, + "rewards/margins": 8.69468879699707, + "rewards/rejected": -15.37267780303955, + "step": 2427 + }, + { + "epoch": 3.18, + "learning_rate": 3.9132959528049474e-05, + "logits/chosen": -1.9445977210998535, + "logits/rejected": -1.9694314002990723, + "logps/chosen": -233.66751098632812, + "logps/rejected": -312.2908630371094, + "loss": 0.0878, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.779075622558594, + "rewards/margins": 8.764878273010254, + "rewards/rejected": -14.543952941894531, + "step": 2428 + }, + { + "epoch": 3.18, + "learning_rate": 3.912436607901334e-05, + "logits/chosen": -1.892113447189331, + "logits/rejected": -1.903243899345398, + "logps/chosen": -230.07528686523438, + "logps/rejected": -279.7763366699219, + "loss": 0.0876, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.427990913391113, + "rewards/margins": 7.037283897399902, + "rewards/rejected": -12.465274810791016, + "step": 2429 + }, + { + "epoch": 3.18, + "learning_rate": 3.91157701779459e-05, + "logits/chosen": -2.0418152809143066, + "logits/rejected": -2.0431063175201416, + "logps/chosen": -289.6370849609375, + "logps/rejected": -371.258056640625, + "loss": 0.0869, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.584191799163818, + "rewards/margins": 9.527920722961426, + "rewards/rejected": -16.11211395263672, + "step": 2430 + }, + { + "epoch": 3.18, + "learning_rate": 3.91071718263394e-05, + "logits/chosen": -1.8952577114105225, + "logits/rejected": -1.975609302520752, + "logps/chosen": -277.6134033203125, + "logps/rejected": -369.93914794921875, + "loss": 0.044, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.555874824523926, + "rewards/margins": 9.395423889160156, + "rewards/rejected": -14.951298713684082, + "step": 2431 + }, + { + "epoch": 3.18, + "learning_rate": 3.909857102568657e-05, + "logits/chosen": -1.5970344543457031, + "logits/rejected": -1.5644360780715942, + "logps/chosen": -290.7686462402344, + "logps/rejected": -411.13165283203125, + "loss": 0.0438, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.298255443572998, + "rewards/margins": 10.077960968017578, + "rewards/rejected": -16.376216888427734, + "step": 2432 + }, + { + "epoch": 3.18, + "learning_rate": 3.908996777748051e-05, + "logits/chosen": -1.9386467933654785, + "logits/rejected": -2.052544593811035, + "logps/chosen": -192.90573120117188, + "logps/rejected": -322.02581787109375, + "loss": 0.0435, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.179001808166504, + "rewards/margins": 10.757402420043945, + "rewards/rejected": -14.936405181884766, + "step": 2433 + }, + { + "epoch": 3.19, + "learning_rate": 3.9081362083214784e-05, + "logits/chosen": -1.9067569971084595, + "logits/rejected": -1.8736732006072998, + "logps/chosen": -281.9491271972656, + "logps/rejected": -359.25262451171875, + "loss": 0.0007, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.667690277099609, + "rewards/margins": 9.953161239624023, + "rewards/rejected": -15.620849609375, + "step": 2434 + }, + { + "epoch": 3.19, + "learning_rate": 3.907275394438335e-05, + "logits/chosen": -2.115126609802246, + "logits/rejected": -2.1041605472564697, + "logps/chosen": -256.5445251464844, + "logps/rejected": -362.7337646484375, + "loss": 0.0436, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.392908096313477, + "rewards/margins": 9.380439758300781, + "rewards/rejected": -14.773348808288574, + "step": 2435 + }, + { + "epoch": 3.19, + "learning_rate": 3.9064143362480637e-05, + "logits/chosen": -1.8047089576721191, + "logits/rejected": -1.767399787902832, + "logps/chosen": -253.64195251464844, + "logps/rejected": -313.3564147949219, + "loss": 0.0494, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.76961088180542, + "rewards/margins": 6.63554573059082, + "rewards/rejected": -12.405157089233398, + "step": 2436 + }, + { + "epoch": 3.19, + "learning_rate": 3.905553033900144e-05, + "logits/chosen": -2.1431894302368164, + "logits/rejected": -2.1688289642333984, + "logps/chosen": -202.8959503173828, + "logps/rejected": -318.6661376953125, + "loss": 0.1241, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.242168426513672, + "rewards/margins": 9.04034423828125, + "rewards/rejected": -14.282512664794922, + "step": 2437 + }, + { + "epoch": 3.19, + "learning_rate": 3.904691487544101e-05, + "logits/chosen": -1.6103119850158691, + "logits/rejected": -1.6657888889312744, + "logps/chosen": -312.8416748046875, + "logps/rejected": -410.4940185546875, + "loss": 0.087, + "rewards/accuracies": 0.9375, + "rewards/chosen": -7.678059101104736, + "rewards/margins": 9.324478149414062, + "rewards/rejected": -17.00253677368164, + "step": 2438 + }, + { + "epoch": 3.19, + "learning_rate": 3.903829697329502e-05, + "logits/chosen": -2.0904810428619385, + "logits/rejected": -2.199312448501587, + "logps/chosen": -237.72813415527344, + "logps/rejected": -360.5721130371094, + "loss": 0.0435, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.821403503417969, + "rewards/margins": 9.57381534576416, + "rewards/rejected": -15.395220756530762, + "step": 2439 + }, + { + "epoch": 3.19, + "learning_rate": 3.902967663405956e-05, + "logits/chosen": -2.03212308883667, + "logits/rejected": -2.1629748344421387, + "logps/chosen": -267.4989929199219, + "logps/rejected": -342.18365478515625, + "loss": 0.0869, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.316411018371582, + "rewards/margins": 7.876006126403809, + "rewards/rejected": -13.19241714477539, + "step": 2440 + }, + { + "epoch": 3.19, + "learning_rate": 3.902105385923115e-05, + "logits/chosen": -1.9315513372421265, + "logits/rejected": -1.8531064987182617, + "logps/chosen": -275.96307373046875, + "logps/rejected": -345.4717712402344, + "loss": 0.1735, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.740131378173828, + "rewards/margins": 7.676638603210449, + "rewards/rejected": -13.416770935058594, + "step": 2441 + }, + { + "epoch": 3.2, + "learning_rate": 3.901242865030673e-05, + "logits/chosen": -1.7603967189788818, + "logits/rejected": -1.808661699295044, + "logps/chosen": -171.9634552001953, + "logps/rejected": -302.039794921875, + "loss": 0.0437, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.7204270362854, + "rewards/margins": 9.628297805786133, + "rewards/rejected": -14.348724365234375, + "step": 2442 + }, + { + "epoch": 3.2, + "learning_rate": 3.900380100878366e-05, + "logits/chosen": -1.9438949823379517, + "logits/rejected": -1.9663145542144775, + "logps/chosen": -211.74940490722656, + "logps/rejected": -284.43328857421875, + "loss": 0.1803, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.087337970733643, + "rewards/margins": 6.456294059753418, + "rewards/rejected": -11.543631553649902, + "step": 2443 + }, + { + "epoch": 3.2, + "learning_rate": 3.899517093615971e-05, + "logits/chosen": -1.6884608268737793, + "logits/rejected": -1.7831130027770996, + "logps/chosen": -256.3289489746094, + "logps/rejected": -407.73321533203125, + "loss": 0.0088, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.849040508270264, + "rewards/margins": 11.37957763671875, + "rewards/rejected": -17.228618621826172, + "step": 2444 + }, + { + "epoch": 3.2, + "learning_rate": 3.8986538433933116e-05, + "logits/chosen": -2.0082106590270996, + "logits/rejected": -2.0837900638580322, + "logps/chosen": -252.9528045654297, + "logps/rejected": -404.23822021484375, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.35462760925293, + "rewards/margins": 11.114782333374023, + "rewards/rejected": -17.469409942626953, + "step": 2445 + }, + { + "epoch": 3.2, + "learning_rate": 3.8977903503602476e-05, + "logits/chosen": -1.9969182014465332, + "logits/rejected": -2.0156984329223633, + "logps/chosen": -231.7480926513672, + "logps/rejected": -348.5244445800781, + "loss": 0.168, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.384876251220703, + "rewards/margins": 7.84205436706543, + "rewards/rejected": -12.22693157196045, + "step": 2446 + }, + { + "epoch": 3.2, + "learning_rate": 3.896926614666686e-05, + "logits/chosen": -1.9319980144500732, + "logits/rejected": -1.9487107992172241, + "logps/chosen": -269.6474609375, + "logps/rejected": -402.9222412109375, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.144097328186035, + "rewards/margins": 10.469097137451172, + "rewards/rejected": -17.61319351196289, + "step": 2447 + }, + { + "epoch": 3.2, + "learning_rate": 3.896062636462572e-05, + "logits/chosen": -1.6876417398452759, + "logits/rejected": -1.7451732158660889, + "logps/chosen": -255.68814086914062, + "logps/rejected": -343.18280029296875, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.484189987182617, + "rewards/margins": 9.643763542175293, + "rewards/rejected": -17.127952575683594, + "step": 2448 + }, + { + "epoch": 3.2, + "learning_rate": 3.895198415897896e-05, + "logits/chosen": -2.1559417247772217, + "logits/rejected": -2.2037086486816406, + "logps/chosen": -244.03125, + "logps/rejected": -376.2003173828125, + "loss": 0.0869, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.5298991203308105, + "rewards/margins": 8.450559616088867, + "rewards/rejected": -14.980459213256836, + "step": 2449 + }, + { + "epoch": 3.21, + "learning_rate": 3.8943339531226895e-05, + "logits/chosen": -2.036494255065918, + "logits/rejected": -2.1403274536132812, + "logps/chosen": -238.76034545898438, + "logps/rejected": -350.3614196777344, + "loss": 0.0051, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.96282958984375, + "rewards/margins": 9.401100158691406, + "rewards/rejected": -15.36392879486084, + "step": 2450 + }, + { + "epoch": 3.21, + "learning_rate": 3.893469248287025e-05, + "logits/chosen": -2.0279226303100586, + "logits/rejected": -2.021245241165161, + "logps/chosen": -231.94520568847656, + "logps/rejected": -349.776123046875, + "loss": 0.0435, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.093944549560547, + "rewards/margins": 11.85527229309082, + "rewards/rejected": -16.949216842651367, + "step": 2451 + }, + { + "epoch": 3.21, + "learning_rate": 3.892604301541018e-05, + "logits/chosen": -1.7343417406082153, + "logits/rejected": -1.8236770629882812, + "logps/chosen": -270.17864990234375, + "logps/rejected": -362.18426513671875, + "loss": 0.0868, + "rewards/accuracies": 0.9375, + "rewards/chosen": -7.230927467346191, + "rewards/margins": 8.540557861328125, + "rewards/rejected": -15.77148723602295, + "step": 2452 + }, + { + "epoch": 3.21, + "learning_rate": 3.891739113034826e-05, + "logits/chosen": -2.024186849594116, + "logits/rejected": -2.0081472396850586, + "logps/chosen": -245.2545166015625, + "logps/rejected": -332.8877868652344, + "loss": 0.0027, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.378724575042725, + "rewards/margins": 10.643712997436523, + "rewards/rejected": -16.022436141967773, + "step": 2453 + }, + { + "epoch": 3.21, + "learning_rate": 3.890873682918648e-05, + "logits/chosen": -1.7880804538726807, + "logits/rejected": -1.9576160907745361, + "logps/chosen": -184.98348999023438, + "logps/rejected": -287.5200500488281, + "loss": 0.0896, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.778811454772949, + "rewards/margins": 8.423736572265625, + "rewards/rejected": -13.202548027038574, + "step": 2454 + }, + { + "epoch": 3.21, + "learning_rate": 3.890008011342725e-05, + "logits/chosen": -1.8650615215301514, + "logits/rejected": -1.9369796514511108, + "logps/chosen": -280.738525390625, + "logps/rejected": -404.7315979003906, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.032692909240723, + "rewards/margins": 11.05531120300293, + "rewards/rejected": -17.088003158569336, + "step": 2455 + }, + { + "epoch": 3.21, + "learning_rate": 3.88914209845734e-05, + "logits/chosen": -2.0854010581970215, + "logits/rejected": -2.2006287574768066, + "logps/chosen": -240.37437438964844, + "logps/rejected": -410.825439453125, + "loss": 0.0002, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.425436019897461, + "rewards/margins": 10.704320907592773, + "rewards/rejected": -17.129756927490234, + "step": 2456 + }, + { + "epoch": 3.22, + "learning_rate": 3.888275944412819e-05, + "logits/chosen": -1.9605014324188232, + "logits/rejected": -1.9867416620254517, + "logps/chosen": -267.28179931640625, + "logps/rejected": -380.3411560058594, + "loss": 0.045, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.87191104888916, + "rewards/margins": 9.627467155456543, + "rewards/rejected": -16.499380111694336, + "step": 2457 + }, + { + "epoch": 3.22, + "learning_rate": 3.887409549359528e-05, + "logits/chosen": -2.3427610397338867, + "logits/rejected": -2.385375738143921, + "logps/chosen": -221.71961975097656, + "logps/rejected": -319.4991455078125, + "loss": 0.045, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.360358238220215, + "rewards/margins": 8.489831924438477, + "rewards/rejected": -13.850191116333008, + "step": 2458 + }, + { + "epoch": 3.22, + "learning_rate": 3.886542913447876e-05, + "logits/chosen": -2.1394031047821045, + "logits/rejected": -2.1759281158447266, + "logps/chosen": -254.70877075195312, + "logps/rejected": -348.2254943847656, + "loss": 0.0437, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.5420122146606445, + "rewards/margins": 10.061568260192871, + "rewards/rejected": -15.603580474853516, + "step": 2459 + }, + { + "epoch": 3.22, + "learning_rate": 3.8856760368283143e-05, + "logits/chosen": -1.9917089939117432, + "logits/rejected": -1.995586633682251, + "logps/chosen": -245.02655029296875, + "logps/rejected": -338.9930725097656, + "loss": 0.0436, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.434337139129639, + "rewards/margins": 9.70940113067627, + "rewards/rejected": -15.143738746643066, + "step": 2460 + }, + { + "epoch": 3.22, + "learning_rate": 3.884808919651334e-05, + "logits/chosen": -2.125701665878296, + "logits/rejected": -2.1440436840057373, + "logps/chosen": -251.43545532226562, + "logps/rejected": -355.54449462890625, + "loss": 0.0441, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.037803649902344, + "rewards/margins": 10.139799118041992, + "rewards/rejected": -16.177602767944336, + "step": 2461 + }, + { + "epoch": 3.22, + "learning_rate": 3.883941562067469e-05, + "logits/chosen": -1.7071962356567383, + "logits/rejected": -1.725158929824829, + "logps/chosen": -230.553466796875, + "logps/rejected": -310.40972900390625, + "loss": 0.0445, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.98706579208374, + "rewards/margins": 8.925557136535645, + "rewards/rejected": -14.912622451782227, + "step": 2462 + }, + { + "epoch": 3.22, + "learning_rate": 3.8830739642272965e-05, + "logits/chosen": -2.2549314498901367, + "logits/rejected": -2.256117105484009, + "logps/chosen": -252.87001037597656, + "logps/rejected": -331.2015380859375, + "loss": 0.0443, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.360776424407959, + "rewards/margins": 9.707989692687988, + "rewards/rejected": -15.068767547607422, + "step": 2463 + }, + { + "epoch": 3.22, + "learning_rate": 3.882206126281432e-05, + "logits/chosen": -1.7178093194961548, + "logits/rejected": -1.7864190340042114, + "logps/chosen": -261.02215576171875, + "logps/rejected": -351.16302490234375, + "loss": 0.0872, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.090516567230225, + "rewards/margins": 8.126142501831055, + "rewards/rejected": -14.216659545898438, + "step": 2464 + }, + { + "epoch": 3.23, + "learning_rate": 3.8813380483805374e-05, + "logits/chosen": -2.1009628772735596, + "logits/rejected": -2.0164217948913574, + "logps/chosen": -262.838134765625, + "logps/rejected": -359.9693603515625, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.574399948120117, + "rewards/margins": 11.433204650878906, + "rewards/rejected": -17.00760269165039, + "step": 2465 + }, + { + "epoch": 3.23, + "learning_rate": 3.8804697306753114e-05, + "logits/chosen": -2.1591877937316895, + "logits/rejected": -2.1212711334228516, + "logps/chosen": -222.40719604492188, + "logps/rejected": -327.9382019042969, + "loss": 0.0438, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.407850742340088, + "rewards/margins": 9.4583101272583, + "rewards/rejected": -14.866161346435547, + "step": 2466 + }, + { + "epoch": 3.23, + "learning_rate": 3.879601173316498e-05, + "logits/chosen": -1.9027129411697388, + "logits/rejected": -1.9284546375274658, + "logps/chosen": -216.7272186279297, + "logps/rejected": -337.1361083984375, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.403012752532959, + "rewards/margins": 12.751434326171875, + "rewards/rejected": -17.154449462890625, + "step": 2467 + }, + { + "epoch": 3.23, + "learning_rate": 3.87873237645488e-05, + "logits/chosen": -1.7814953327178955, + "logits/rejected": -1.8257054090499878, + "logps/chosen": -225.2574462890625, + "logps/rejected": -346.04974365234375, + "loss": 0.0044, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.834550380706787, + "rewards/margins": 10.236806869506836, + "rewards/rejected": -16.07135772705078, + "step": 2468 + }, + { + "epoch": 3.23, + "learning_rate": 3.8778633402412836e-05, + "logits/chosen": -2.073378324508667, + "logits/rejected": -2.1560871601104736, + "logps/chosen": -213.5805206298828, + "logps/rejected": -301.17755126953125, + "loss": 0.0031, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.958551406860352, + "rewards/margins": 9.559164047241211, + "rewards/rejected": -14.517715454101562, + "step": 2469 + }, + { + "epoch": 3.23, + "learning_rate": 3.876994064826576e-05, + "logits/chosen": -2.0379161834716797, + "logits/rejected": -2.056102752685547, + "logps/chosen": -275.51348876953125, + "logps/rejected": -376.2273864746094, + "loss": 0.0006, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.516529560089111, + "rewards/margins": 10.124565124511719, + "rewards/rejected": -16.641094207763672, + "step": 2470 + }, + { + "epoch": 3.23, + "learning_rate": 3.876124550361667e-05, + "logits/chosen": -2.012174129486084, + "logits/rejected": -1.9914698600769043, + "logps/chosen": -271.0264587402344, + "logps/rejected": -371.2334289550781, + "loss": 0.0027, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.945853233337402, + "rewards/margins": 10.607396125793457, + "rewards/rejected": -17.55324935913086, + "step": 2471 + }, + { + "epoch": 3.24, + "learning_rate": 3.875254796997505e-05, + "logits/chosen": -1.8690792322158813, + "logits/rejected": -1.9150246381759644, + "logps/chosen": -281.6956787109375, + "logps/rejected": -394.4105224609375, + "loss": 0.0005, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.801252365112305, + "rewards/margins": 10.3662748336792, + "rewards/rejected": -17.16752815246582, + "step": 2472 + }, + { + "epoch": 3.24, + "learning_rate": 3.874384804885082e-05, + "logits/chosen": -1.8989354372024536, + "logits/rejected": -1.9007232189178467, + "logps/chosen": -249.84799194335938, + "logps/rejected": -324.7830505371094, + "loss": 0.0439, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.7783098220825195, + "rewards/margins": 8.47901725769043, + "rewards/rejected": -14.257326126098633, + "step": 2473 + }, + { + "epoch": 3.24, + "learning_rate": 3.873514574175433e-05, + "logits/chosen": -1.9035624265670776, + "logits/rejected": -1.9880552291870117, + "logps/chosen": -225.6424560546875, + "logps/rejected": -357.0283203125, + "loss": 0.0868, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.232097625732422, + "rewards/margins": 9.460615158081055, + "rewards/rejected": -15.69271183013916, + "step": 2474 + }, + { + "epoch": 3.24, + "learning_rate": 3.8726441050196305e-05, + "logits/chosen": -2.067682981491089, + "logits/rejected": -2.197803258895874, + "logps/chosen": -312.9625549316406, + "logps/rejected": -411.094970703125, + "loss": 0.0436, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.463919639587402, + "rewards/margins": 10.368206977844238, + "rewards/rejected": -16.832128524780273, + "step": 2475 + }, + { + "epoch": 3.24, + "learning_rate": 3.871773397568792e-05, + "logits/chosen": -1.8097530603408813, + "logits/rejected": -1.8437132835388184, + "logps/chosen": -245.35177612304688, + "logps/rejected": -360.3444519042969, + "loss": 0.003, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.219241619110107, + "rewards/margins": 10.346756935119629, + "rewards/rejected": -15.565999031066895, + "step": 2476 + }, + { + "epoch": 3.24, + "learning_rate": 3.8709024519740735e-05, + "logits/chosen": -1.8741097450256348, + "logits/rejected": -1.864046335220337, + "logps/chosen": -236.70962524414062, + "logps/rejected": -333.8223571777344, + "loss": 0.0435, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.540530681610107, + "rewards/margins": 10.588478088378906, + "rewards/rejected": -15.129007339477539, + "step": 2477 + }, + { + "epoch": 3.24, + "learning_rate": 3.870031268386676e-05, + "logits/chosen": -2.0998189449310303, + "logits/rejected": -2.161912202835083, + "logps/chosen": -253.3290557861328, + "logps/rejected": -377.12567138671875, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.067862510681152, + "rewards/margins": 10.710309982299805, + "rewards/rejected": -16.77817153930664, + "step": 2478 + }, + { + "epoch": 3.24, + "learning_rate": 3.869159846957836e-05, + "logits/chosen": -1.9125173091888428, + "logits/rejected": -1.9453988075256348, + "logps/chosen": -246.0362548828125, + "logps/rejected": -325.9284973144531, + "loss": 0.0871, + "rewards/accuracies": 0.9375, + "rewards/chosen": -5.746542930603027, + "rewards/margins": 9.125516891479492, + "rewards/rejected": -14.872061729431152, + "step": 2479 + }, + { + "epoch": 3.25, + "learning_rate": 3.8682881878388376e-05, + "logits/chosen": -1.9923452138900757, + "logits/rejected": -2.0072903633117676, + "logps/chosen": -277.7795104980469, + "logps/rejected": -377.7403564453125, + "loss": 0.0438, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.331365585327148, + "rewards/margins": 10.37812614440918, + "rewards/rejected": -16.709491729736328, + "step": 2480 + }, + { + "epoch": 3.25, + "learning_rate": 3.867416291181003e-05, + "logits/chosen": -2.1541683673858643, + "logits/rejected": -2.184211492538452, + "logps/chosen": -259.4024658203125, + "logps/rejected": -336.68927001953125, + "loss": 0.0078, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.420302391052246, + "rewards/margins": 8.163504600524902, + "rewards/rejected": -15.583806991577148, + "step": 2481 + }, + { + "epoch": 3.25, + "learning_rate": 3.866544157135695e-05, + "logits/chosen": -2.1215081214904785, + "logits/rejected": -2.0577147006988525, + "logps/chosen": -278.74676513671875, + "logps/rejected": -354.3476867675781, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.952124118804932, + "rewards/margins": 9.676928520202637, + "rewards/rejected": -15.629051208496094, + "step": 2482 + }, + { + "epoch": 3.25, + "learning_rate": 3.8656717858543195e-05, + "logits/chosen": -1.8749539852142334, + "logits/rejected": -1.8486679792404175, + "logps/chosen": -215.23916625976562, + "logps/rejected": -323.09613037109375, + "loss": 0.0004, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.153021335601807, + "rewards/margins": 9.829922676086426, + "rewards/rejected": -15.98294448852539, + "step": 2483 + }, + { + "epoch": 3.25, + "learning_rate": 3.864799177488322e-05, + "logits/chosen": -2.078026294708252, + "logits/rejected": -2.025261402130127, + "logps/chosen": -285.80206298828125, + "logps/rejected": -350.468017578125, + "loss": 0.0498, + "rewards/accuracies": 0.9375, + "rewards/chosen": -6.340088844299316, + "rewards/margins": 8.425580024719238, + "rewards/rejected": -14.765668869018555, + "step": 2484 + }, + { + "epoch": 3.25, + "learning_rate": 3.863926332189191e-05, + "logits/chosen": -1.8989228010177612, + "logits/rejected": -1.887642502784729, + "logps/chosen": -209.74522399902344, + "logps/rejected": -319.71978759765625, + "loss": 0.0868, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.254289150238037, + "rewards/margins": 9.678909301757812, + "rewards/rejected": -14.933197975158691, + "step": 2485 + }, + { + "epoch": 3.25, + "learning_rate": 3.8630532501084535e-05, + "logits/chosen": -1.9497569799423218, + "logits/rejected": -2.0822038650512695, + "logps/chosen": -165.8619842529297, + "logps/rejected": -305.4637451171875, + "loss": 0.1316, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.822153091430664, + "rewards/margins": 9.318231582641602, + "rewards/rejected": -14.140385627746582, + "step": 2486 + }, + { + "epoch": 3.25, + "learning_rate": 3.8621799313976805e-05, + "logits/chosen": -1.9201487302780151, + "logits/rejected": -1.9919875860214233, + "logps/chosen": -236.57803344726562, + "logps/rejected": -345.8396301269531, + "loss": 0.0131, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.3776373863220215, + "rewards/margins": 10.207651138305664, + "rewards/rejected": -16.585289001464844, + "step": 2487 + }, + { + "epoch": 3.26, + "learning_rate": 3.861306376208481e-05, + "logits/chosen": -2.135756492614746, + "logits/rejected": -2.178420305252075, + "logps/chosen": -276.371337890625, + "logps/rejected": -428.98321533203125, + "loss": 0.0002, + "rewards/accuracies": 1.0, + "rewards/chosen": -7.153833866119385, + "rewards/margins": 11.597715377807617, + "rewards/rejected": -18.751548767089844, + "step": 2488 + }, + { + "epoch": 3.26, + "learning_rate": 3.860432584692509e-05, + "logits/chosen": -1.9746434688568115, + "logits/rejected": -2.049467086791992, + "logps/chosen": -244.74005126953125, + "logps/rejected": -384.7591247558594, + "loss": 0.0001, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.734624862670898, + "rewards/margins": 13.104833602905273, + "rewards/rejected": -18.839460372924805, + "step": 2489 + }, + { + "epoch": 3.26, + "learning_rate": 3.859558557001456e-05, + "logits/chosen": -1.8276326656341553, + "logits/rejected": -1.8128743171691895, + "logps/chosen": -242.62420654296875, + "logps/rejected": -370.6613464355469, + "loss": 0.0435, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.021103858947754, + "rewards/margins": 11.176106452941895, + "rewards/rejected": -17.19721031188965, + "step": 2490 + }, + { + "epoch": 3.26, + "learning_rate": 3.858684293287055e-05, + "logits/chosen": -1.827390193939209, + "logits/rejected": -1.8298470973968506, + "logps/chosen": -314.5787048339844, + "logps/rejected": -394.1010437011719, + "loss": 0.0868, + "rewards/accuracies": 0.875, + "rewards/chosen": -7.975650787353516, + "rewards/margins": 8.666006088256836, + "rewards/rejected": -16.641658782958984, + "step": 2491 + }, + { + "epoch": 3.26, + "learning_rate": 3.857809793701082e-05, + "logits/chosen": -1.9679362773895264, + "logits/rejected": -1.9574182033538818, + "logps/chosen": -205.41207885742188, + "logps/rejected": -321.79034423828125, + "loss": 0.1353, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.5130960941314697, + "rewards/margins": 10.656757354736328, + "rewards/rejected": -14.169854164123535, + "step": 2492 + }, + { + "epoch": 3.26, + "learning_rate": 3.856935058395352e-05, + "logits/chosen": -1.8985028266906738, + "logits/rejected": -1.9866564273834229, + "logps/chosen": -231.4768524169922, + "logps/rejected": -359.1987609863281, + "loss": 0.0868, + "rewards/accuracies": 0.875, + "rewards/chosen": -6.280361652374268, + "rewards/margins": 8.678674697875977, + "rewards/rejected": -14.959036827087402, + "step": 2493 + }, + { + "epoch": 3.26, + "learning_rate": 3.8560600875217235e-05, + "logits/chosen": -1.9721897840499878, + "logits/rejected": -2.0196175575256348, + "logps/chosen": -184.3533172607422, + "logps/rejected": -299.4060974121094, + "loss": 0.1734, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.074180603027344, + "rewards/margins": 9.483386039733887, + "rewards/rejected": -14.55756664276123, + "step": 2494 + }, + { + "epoch": 3.27, + "learning_rate": 3.8551848812320904e-05, + "logits/chosen": -1.945467233657837, + "logits/rejected": -1.9828685522079468, + "logps/chosen": -295.1348571777344, + "logps/rejected": -422.5150451660156, + "loss": 0.0877, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.004589080810547, + "rewards/margins": 10.746018409729004, + "rewards/rejected": -15.750608444213867, + "step": 2495 + }, + { + "epoch": 3.27, + "learning_rate": 3.854309439678394e-05, + "logits/chosen": -1.9645689725875854, + "logits/rejected": -1.9029303789138794, + "logps/chosen": -236.8740692138672, + "logps/rejected": -359.9755554199219, + "loss": 0.0002, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.681175708770752, + "rewards/margins": 11.418691635131836, + "rewards/rejected": -18.099868774414062, + "step": 2496 + }, + { + "epoch": 3.27, + "learning_rate": 3.853433763012612e-05, + "logits/chosen": -2.2054662704467773, + "logits/rejected": -2.1916756629943848, + "logps/chosen": -250.4889373779297, + "logps/rejected": -302.42645263671875, + "loss": 0.0927, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.19832706451416, + "rewards/margins": 9.23891830444336, + "rewards/rejected": -14.437244415283203, + "step": 2497 + }, + { + "epoch": 3.27, + "learning_rate": 3.852557851386766e-05, + "logits/chosen": -2.000141143798828, + "logits/rejected": -1.994922399520874, + "logps/chosen": -203.5865478515625, + "logps/rejected": -293.255615234375, + "loss": 0.1736, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.755192756652832, + "rewards/margins": 7.394274711608887, + "rewards/rejected": -13.149467468261719, + "step": 2498 + }, + { + "epoch": 3.27, + "learning_rate": 3.8516817049529144e-05, + "logits/chosen": -1.904963731765747, + "logits/rejected": -1.9929938316345215, + "logps/chosen": -228.28164672851562, + "logps/rejected": -359.9959411621094, + "loss": 0.0536, + "rewards/accuracies": 1.0, + "rewards/chosen": -6.353829383850098, + "rewards/margins": 9.504854202270508, + "rewards/rejected": -15.858683586120605, + "step": 2499 + }, + { + "epoch": 3.27, + "learning_rate": 3.8508053238631614e-05, + "logits/chosen": -1.7504050731658936, + "logits/rejected": -1.6863926649093628, + "logps/chosen": -251.31768798828125, + "logps/rejected": -340.1594543457031, + "loss": 0.0003, + "rewards/accuracies": 1.0, + "rewards/chosen": -5.837611675262451, + "rewards/margins": 9.95163631439209, + "rewards/rejected": -15.7892484664917, + "step": 2500 + } + ], + "logging_steps": 1, + "max_steps": 7640, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}