{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 3334, "global_step": 3334, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "approx. KL/chosen": 0.0, "approx. KL/rejected": 0.0, "epoch": 0.00029994001199760045, "grad_norm": 5.702817440032959, "learning_rate": 1e-05, "logp/chosen": -328.0, "logp/rejected": -346.0, "loss": 0.69140625, "reward/accuracy": 0.0, "reward/chosen": 0.0, "reward/margin": 0.0, "reward/rejected": 0.0, "step": 1 }, { "approx. KL/chosen": 0.921875, "approx. KL/rejected": 1.4765625, "epoch": 0.0005998800239952009, "grad_norm": 6.700874328613281, "learning_rate": 9.999997780227173e-06, "logp/chosen": -364.0, "logp/rejected": -398.0, "loss": 0.67333984375, "reward/accuracy": 0.4375, "reward/chosen": -0.0546875, "reward/margin": 0.04150390625, "reward/rejected": -0.09619140625, "step": 2 }, { "approx. KL/chosen": 1.9140625, "approx. KL/rejected": 1.40625, "epoch": 0.0008998200359928015, "grad_norm": 5.3937764167785645, "learning_rate": 9.99999112091066e-06, "logp/chosen": -414.0, "logp/rejected": -390.0, "loss": 0.68994140625, "reward/accuracy": 0.375, "reward/chosen": -0.06689453125, "reward/margin": 0.01220703125, "reward/rejected": -0.0791015625, "step": 3 }, { "approx. KL/chosen": 3.046875, "approx. KL/rejected": 1.4609375, "epoch": 0.0011997600479904018, "grad_norm": 5.950886249542236, "learning_rate": 9.999980022056375e-06, "logp/chosen": -212.0, "logp/rejected": -255.0, "loss": 0.71533203125, "reward/accuracy": 0.25, "reward/chosen": -0.09814453125, "reward/margin": -0.0308837890625, "reward/rejected": -0.0673828125, "step": 4 }, { "approx. KL/chosen": 1.53125, "approx. KL/rejected": 0.94921875, "epoch": 0.0014997000599880025, "grad_norm": 5.276072978973389, "learning_rate": 9.999964483674172e-06, "logp/chosen": -260.0, "logp/rejected": -320.0, "loss": 0.681640625, "reward/accuracy": 0.375, "reward/chosen": -0.033203125, "reward/margin": 0.03076171875, "reward/rejected": -0.06396484375, "step": 5 }, { "approx. KL/chosen": 2.078125, "approx. KL/rejected": 2.40625, "epoch": 0.001799640071985603, "grad_norm": 10.847053527832031, "learning_rate": 9.999944505777848e-06, "logp/chosen": -314.0, "logp/rejected": -312.0, "loss": 0.6640625, "reward/accuracy": 0.5625, "reward/chosen": -0.0849609375, "reward/margin": 0.0712890625, "reward/rejected": -0.15625, "step": 6 }, { "approx. KL/chosen": 2.515625, "approx. KL/rejected": 5.875, "epoch": 0.0020995800839832034, "grad_norm": 6.169078350067139, "learning_rate": 9.999920088385142e-06, "logp/chosen": -362.0, "logp/rejected": -332.0, "loss": 0.63720703125, "reward/accuracy": 0.625, "reward/chosen": -0.150390625, "reward/margin": 0.12890625, "reward/rejected": -0.279296875, "step": 7 }, { "approx. KL/chosen": 5.0625, "approx. KL/rejected": 8.625, "epoch": 0.0023995200959808036, "grad_norm": 5.282023906707764, "learning_rate": 9.999891231517736e-06, "logp/chosen": -382.0, "logp/rejected": -352.0, "loss": 0.6396484375, "reward/accuracy": 0.5, "reward/chosen": -0.2265625, "reward/margin": 0.126953125, "reward/rejected": -0.353515625, "step": 8 }, { "approx. KL/chosen": 8.8125, "approx. KL/rejected": 10.75, "epoch": 0.0026994601079784043, "grad_norm": 4.834122657775879, "learning_rate": 9.999857935201248e-06, "logp/chosen": -256.0, "logp/rejected": -282.0, "loss": 0.68310546875, "reward/accuracy": 0.5625, "reward/chosen": -0.267578125, "reward/margin": 0.051025390625, "reward/rejected": -0.318359375, "step": 9 }, { "approx. KL/chosen": 9.9375, "approx. KL/rejected": 15.5625, "epoch": 0.002999400119976005, "grad_norm": 5.905514240264893, "learning_rate": 9.999820199465246e-06, "logp/chosen": -292.0, "logp/rejected": -358.0, "loss": 0.6201171875, "reward/accuracy": 0.5625, "reward/chosen": -0.318359375, "reward/margin": 0.169921875, "reward/rejected": -0.48828125, "step": 10 }, { "approx. KL/chosen": 11.625, "approx. KL/rejected": 9.625, "epoch": 0.0032993401319736052, "grad_norm": 6.693594455718994, "learning_rate": 9.999778024343234e-06, "logp/chosen": -290.0, "logp/rejected": -284.0, "loss": 0.7939453125, "reward/accuracy": 0.25, "reward/chosen": -0.41015625, "reward/margin": -0.146484375, "reward/rejected": -0.263671875, "step": 11 }, { "approx. KL/chosen": 17.25, "approx. KL/rejected": 36.25, "epoch": 0.003599280143971206, "grad_norm": 5.624619483947754, "learning_rate": 9.99973140987266e-06, "logp/chosen": -400.0, "logp/rejected": -356.0, "loss": 0.615966796875, "reward/accuracy": 0.5625, "reward/chosen": -0.37109375, "reward/margin": 0.275390625, "reward/rejected": -0.6484375, "step": 12 }, { "approx. KL/chosen": 12.875, "approx. KL/rejected": 28.5, "epoch": 0.003899220155968806, "grad_norm": 6.40227746963501, "learning_rate": 9.999680356094915e-06, "logp/chosen": -266.0, "logp/rejected": -318.0, "loss": 0.68408203125, "reward/accuracy": 0.5625, "reward/chosen": -0.373046875, "reward/margin": 0.1328125, "reward/rejected": -0.50390625, "step": 13 }, { "approx. KL/chosen": 37.25, "approx. KL/rejected": 73.5, "epoch": 0.004199160167966407, "grad_norm": 5.500258445739746, "learning_rate": 9.999624863055327e-06, "logp/chosen": -446.0, "logp/rejected": -372.0, "loss": 0.572509765625, "reward/accuracy": 0.6875, "reward/chosen": -0.609375, "reward/margin": 0.412109375, "reward/rejected": -1.0234375, "step": 14 }, { "approx. KL/chosen": 11.9375, "approx. KL/rejected": 32.0, "epoch": 0.004499100179964007, "grad_norm": 4.564111232757568, "learning_rate": 9.999564930803172e-06, "logp/chosen": -278.0, "logp/rejected": -290.0, "loss": 0.614501953125, "reward/accuracy": 0.5625, "reward/chosen": -0.380859375, "reward/margin": 0.248046875, "reward/rejected": -0.62890625, "step": 15 }, { "approx. KL/chosen": 15.375, "approx. KL/rejected": 25.5, "epoch": 0.004799040191961607, "grad_norm": 4.36024808883667, "learning_rate": 9.999500559391661e-06, "logp/chosen": -348.0, "logp/rejected": -274.0, "loss": 0.56298828125, "reward/accuracy": 0.875, "reward/chosen": -0.287109375, "reward/margin": 0.34765625, "reward/rejected": -0.63671875, "step": 16 }, { "approx. KL/chosen": 12.875, "approx. KL/rejected": 28.625, "epoch": 0.005098980203959208, "grad_norm": 5.061554431915283, "learning_rate": 9.999431748877954e-06, "logp/chosen": -196.0, "logp/rejected": -252.0, "loss": 0.6123046875, "reward/accuracy": 0.5625, "reward/chosen": -0.302734375, "reward/margin": 0.2451171875, "reward/rejected": -0.546875, "step": 17 }, { "approx. KL/chosen": 14.0625, "approx. KL/rejected": 34.75, "epoch": 0.005398920215956809, "grad_norm": 4.499447345733643, "learning_rate": 9.999358499323145e-06, "logp/chosen": -480.0, "logp/rejected": -362.0, "loss": 0.631591796875, "reward/accuracy": 0.75, "reward/chosen": -0.40625, "reward/margin": 0.2275390625, "reward/rejected": -0.6328125, "step": 18 }, { "approx. KL/chosen": 26.75, "approx. KL/rejected": 60.0, "epoch": 0.005698860227954409, "grad_norm": 6.246758460998535, "learning_rate": 9.999280810792275e-06, "logp/chosen": -209.0, "logp/rejected": -292.0, "loss": 0.55810546875, "reward/accuracy": 0.6875, "reward/chosen": -0.51953125, "reward/margin": 0.4140625, "reward/rejected": -0.93359375, "step": 19 }, { "approx. KL/chosen": 54.25, "approx. KL/rejected": 77.5, "epoch": 0.00599880023995201, "grad_norm": 7.326489448547363, "learning_rate": 9.999198683354324e-06, "logp/chosen": -334.0, "logp/rejected": -340.0, "loss": 0.5771484375, "reward/accuracy": 0.8125, "reward/chosen": -0.62109375, "reward/margin": 0.427734375, "reward/rejected": -1.046875, "step": 20 }, { "approx. KL/chosen": 23.875, "approx. KL/rejected": 134.0, "epoch": 0.00629874025194961, "grad_norm": 4.65969181060791, "learning_rate": 9.999112117082212e-06, "logp/chosen": -258.0, "logp/rejected": -342.0, "loss": 0.557861328125, "reward/accuracy": 0.625, "reward/chosen": -0.482421875, "reward/margin": 0.6484375, "reward/rejected": -1.1328125, "step": 21 }, { "approx. KL/chosen": 74.5, "approx. KL/rejected": 68.0, "epoch": 0.0065986802639472104, "grad_norm": 7.481565475463867, "learning_rate": 9.999021112052804e-06, "logp/chosen": -312.0, "logp/rejected": -268.0, "loss": 0.746337890625, "reward/accuracy": 0.5625, "reward/chosen": -0.75, "reward/margin": 0.1865234375, "reward/rejected": -0.9375, "step": 22 }, { "approx. KL/chosen": 23.25, "approx. KL/rejected": 74.0, "epoch": 0.006898620275944811, "grad_norm": 4.906702518463135, "learning_rate": 9.998925668346902e-06, "logp/chosen": -360.0, "logp/rejected": -272.0, "loss": 0.62109375, "reward/accuracy": 0.5625, "reward/chosen": -0.5390625, "reward/margin": 0.349609375, "reward/rejected": -0.890625, "step": 23 }, { "approx. KL/chosen": 113.5, "approx. KL/rejected": 81.5, "epoch": 0.007198560287942412, "grad_norm": 8.695887565612793, "learning_rate": 9.998825786049255e-06, "logp/chosen": -276.0, "logp/rejected": -272.0, "loss": 0.822021484375, "reward/accuracy": 0.5, "reward/chosen": -0.98828125, "reward/margin": -0.031982421875, "reward/rejected": -0.95703125, "step": 24 }, { "approx. KL/chosen": 30.875, "approx. KL/rejected": 121.5, "epoch": 0.007498500299940012, "grad_norm": 4.589423656463623, "learning_rate": 9.998721465248546e-06, "logp/chosen": -318.0, "logp/rejected": -290.0, "loss": 0.5545654296875, "reward/accuracy": 0.6875, "reward/chosen": -0.60546875, "reward/margin": 0.5234375, "reward/rejected": -1.125, "step": 25 }, { "approx. KL/chosen": 35.75, "approx. KL/rejected": 86.0, "epoch": 0.007798440311937612, "grad_norm": 4.761041641235352, "learning_rate": 9.998612706037405e-06, "logp/chosen": -382.0, "logp/rejected": -358.0, "loss": 0.5379638671875, "reward/accuracy": 0.75, "reward/chosen": -0.58984375, "reward/margin": 0.54296875, "reward/rejected": -1.1328125, "step": 26 }, { "approx. KL/chosen": 47.75, "approx. KL/rejected": 154.0, "epoch": 0.008098380323935212, "grad_norm": 4.91733980178833, "learning_rate": 9.998499508512398e-06, "logp/chosen": -212.0, "logp/rejected": -306.0, "loss": 0.434814453125, "reward/accuracy": 0.8125, "reward/chosen": -0.796875, "reward/margin": 0.859375, "reward/rejected": -1.65625, "step": 27 }, { "approx. KL/chosen": 42.25, "approx. KL/rejected": 71.5, "epoch": 0.008398320335932814, "grad_norm": 5.545990943908691, "learning_rate": 9.998381872774035e-06, "logp/chosen": -251.0, "logp/rejected": -356.0, "loss": 0.499267578125, "reward/accuracy": 0.875, "reward/chosen": -0.51171875, "reward/margin": 0.54296875, "reward/rejected": -1.0546875, "step": 28 }, { "approx. KL/chosen": 45.75, "approx. KL/rejected": 214.0, "epoch": 0.008698260347930415, "grad_norm": 3.706907033920288, "learning_rate": 9.998259798926766e-06, "logp/chosen": -364.0, "logp/rejected": -304.0, "loss": 0.3447265625, "reward/accuracy": 1.0, "reward/chosen": -0.431640625, "reward/margin": 1.1015625, "reward/rejected": -1.53125, "step": 29 }, { "approx. KL/chosen": 62.0, "approx. KL/rejected": 102.0, "epoch": 0.008998200359928014, "grad_norm": 6.414937973022461, "learning_rate": 9.998133287078982e-06, "logp/chosen": -210.0, "logp/rejected": -306.0, "loss": 0.610107421875, "reward/accuracy": 0.6875, "reward/chosen": -0.7890625, "reward/margin": 0.423828125, "reward/rejected": -1.2109375, "step": 30 }, { "approx. KL/chosen": 115.0, "approx. KL/rejected": 236.0, "epoch": 0.009298140371925615, "grad_norm": 6.313652515411377, "learning_rate": 9.998002337343013e-06, "logp/chosen": -258.0, "logp/rejected": -306.0, "loss": 0.561920166015625, "reward/accuracy": 0.8125, "reward/chosen": -0.61328125, "reward/margin": 1.0859375, "reward/rejected": -1.703125, "step": 31 }, { "approx. KL/chosen": 42.75, "approx. KL/rejected": 147.0, "epoch": 0.009598080383923215, "grad_norm": 4.9897565841674805, "learning_rate": 9.99786694983513e-06, "logp/chosen": -282.0, "logp/rejected": -310.0, "loss": 0.4049072265625, "reward/accuracy": 0.75, "reward/chosen": -0.46484375, "reward/margin": 0.953125, "reward/rejected": -1.4140625, "step": 32 }, { "approx. KL/chosen": 21.0, "approx. KL/rejected": 120.5, "epoch": 0.009898020395920816, "grad_norm": 4.14672327041626, "learning_rate": 9.997727124675547e-06, "logp/chosen": -252.0, "logp/rejected": -237.0, "loss": 0.44140625, "reward/accuracy": 0.6875, "reward/chosen": -0.43359375, "reward/margin": 0.83203125, "reward/rejected": -1.265625, "step": 33 }, { "approx. KL/chosen": 95.0, "approx. KL/rejected": 166.0, "epoch": 0.010197960407918417, "grad_norm": 6.66151237487793, "learning_rate": 9.997582861988415e-06, "logp/chosen": -394.0, "logp/rejected": -360.0, "loss": 0.666259765625, "reward/accuracy": 0.625, "reward/chosen": -0.9453125, "reward/margin": 0.43359375, "reward/rejected": -1.375, "step": 34 }, { "approx. KL/chosen": 42.25, "approx. KL/rejected": 97.5, "epoch": 0.010497900419916016, "grad_norm": 5.090958118438721, "learning_rate": 9.997434161901825e-06, "logp/chosen": -356.0, "logp/rejected": -378.0, "loss": 0.5224609375, "reward/accuracy": 0.75, "reward/chosen": -0.578125, "reward/margin": 0.56640625, "reward/rejected": -1.140625, "step": 35 }, { "approx. KL/chosen": 44.0, "approx. KL/rejected": 124.0, "epoch": 0.010797840431913617, "grad_norm": 4.764455795288086, "learning_rate": 9.99728102454781e-06, "logp/chosen": -336.0, "logp/rejected": -350.0, "loss": 0.5341796875, "reward/accuracy": 0.625, "reward/chosen": -0.447265625, "reward/margin": 0.6484375, "reward/rejected": -1.09375, "step": 36 }, { "approx. KL/chosen": 38.25, "approx. KL/rejected": 624.0, "epoch": 0.011097780443911218, "grad_norm": 4.795415878295898, "learning_rate": 9.997123450062343e-06, "logp/chosen": -370.0, "logp/rejected": -388.0, "loss": 0.51409912109375, "reward/accuracy": 0.8125, "reward/chosen": -0.330078125, "reward/margin": 1.5, "reward/rejected": -1.828125, "step": 37 }, { "approx. KL/chosen": 68.5, "approx. KL/rejected": 185.0, "epoch": 0.011397720455908818, "grad_norm": 4.909962177276611, "learning_rate": 9.996961438585335e-06, "logp/chosen": -316.0, "logp/rejected": -348.0, "loss": 0.59912109375, "reward/accuracy": 0.6875, "reward/chosen": -0.7734375, "reward/margin": 0.66796875, "reward/rejected": -1.4453125, "step": 38 }, { "approx. KL/chosen": 53.0, "approx. KL/rejected": 177.0, "epoch": 0.011697660467906419, "grad_norm": 4.553750991821289, "learning_rate": 9.996794990260636e-06, "logp/chosen": -238.0, "logp/rejected": -290.0, "loss": 0.545166015625, "reward/accuracy": 0.6875, "reward/chosen": -0.484375, "reward/margin": 0.87890625, "reward/rejected": -1.3671875, "step": 39 }, { "approx. KL/chosen": 55.75, "approx. KL/rejected": 215.0, "epoch": 0.01199760047990402, "grad_norm": 5.085227012634277, "learning_rate": 9.99662410523604e-06, "logp/chosen": -239.0, "logp/rejected": -334.0, "loss": 0.548095703125, "reward/accuracy": 0.5625, "reward/chosen": -0.60546875, "reward/margin": 0.81640625, "reward/rejected": -1.421875, "step": 40 }, { "approx. KL/chosen": 40.5, "approx. KL/rejected": 211.0, "epoch": 0.01229754049190162, "grad_norm": 3.752643585205078, "learning_rate": 9.996448783663274e-06, "logp/chosen": -346.0, "logp/rejected": -352.0, "loss": 0.36041259765625, "reward/accuracy": 0.8125, "reward/chosen": -0.404296875, "reward/margin": 1.28125, "reward/rejected": -1.6875, "step": 41 }, { "approx. KL/chosen": 108.5, "approx. KL/rejected": 173.0, "epoch": 0.01259748050389922, "grad_norm": 6.684460163116455, "learning_rate": 9.99626902569801e-06, "logp/chosen": -306.0, "logp/rejected": -296.0, "loss": 0.708740234375, "reward/accuracy": 0.5, "reward/chosen": -1.1875, "reward/margin": 0.2734375, "reward/rejected": -1.4609375, "step": 42 }, { "approx. KL/chosen": 113.0, "approx. KL/rejected": 153.0, "epoch": 0.01289742051589682, "grad_norm": 7.0753493309021, "learning_rate": 9.996084831499857e-06, "logp/chosen": -358.0, "logp/rejected": -298.0, "loss": 0.8282470703125, "reward/accuracy": 0.5, "reward/chosen": -1.234375, "reward/margin": 0.0673828125, "reward/rejected": -1.3046875, "step": 43 }, { "approx. KL/chosen": 58.0, "approx. KL/rejected": 210.0, "epoch": 0.013197360527894421, "grad_norm": 4.3497538566589355, "learning_rate": 9.99589620123236e-06, "logp/chosen": -336.0, "logp/rejected": -310.0, "loss": 0.454833984375, "reward/accuracy": 0.875, "reward/chosen": -0.7734375, "reward/margin": 0.859375, "reward/rejected": -1.6328125, "step": 44 }, { "approx. KL/chosen": 94.0, "approx. KL/rejected": 414.0, "epoch": 0.013497300539892022, "grad_norm": 4.9340314865112305, "learning_rate": 9.99570313506301e-06, "logp/chosen": -284.0, "logp/rejected": -322.0, "loss": 0.41265869140625, "reward/accuracy": 0.75, "reward/chosen": -1.140625, "reward/margin": 1.375, "reward/rejected": -2.515625, "step": 45 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 268.0, "epoch": 0.013797240551889621, "grad_norm": 8.416193008422852, "learning_rate": 9.995505633163226e-06, "logp/chosen": -284.0, "logp/rejected": -230.0, "loss": 0.667724609375, "reward/accuracy": 0.625, "reward/chosen": -1.5625, "reward/margin": 0.328125, "reward/rejected": -1.890625, "step": 46 }, { "approx. KL/chosen": 81.5, "approx. KL/rejected": 408.0, "epoch": 0.014097180563887222, "grad_norm": 4.852382183074951, "learning_rate": 9.995303695708379e-06, "logp/chosen": -360.0, "logp/rejected": -418.0, "loss": 0.4515380859375, "reward/accuracy": 0.75, "reward/chosen": -0.87109375, "reward/margin": 1.1875, "reward/rejected": -2.0625, "step": 47 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 147.0, "epoch": 0.014397120575884824, "grad_norm": 11.603910446166992, "learning_rate": 9.995097322877765e-06, "logp/chosen": -422.0, "logp/rejected": -300.0, "loss": 1.0589599609375, "reward/accuracy": 0.75, "reward/chosen": -1.390625, "reward/margin": 0.126953125, "reward/rejected": -1.515625, "step": 48 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 326.0, "epoch": 0.014697060587882423, "grad_norm": 6.911006450653076, "learning_rate": 9.99488651485463e-06, "logp/chosen": -324.0, "logp/rejected": -264.0, "loss": 0.767852783203125, "reward/accuracy": 0.625, "reward/chosen": -1.4765625, "reward/margin": 0.59375, "reward/rejected": -2.0625, "step": 49 }, { "approx. KL/chosen": 60.5, "approx. KL/rejected": 107.5, "epoch": 0.014997000599880024, "grad_norm": 6.320631980895996, "learning_rate": 9.994671271826146e-06, "logp/chosen": -243.0, "logp/rejected": -282.0, "loss": 0.68310546875, "reward/accuracy": 0.5, "reward/chosen": -0.89453125, "reward/margin": 0.283203125, "reward/rejected": -1.1796875, "step": 50 }, { "approx. KL/chosen": 65.5, "approx. KL/rejected": 229.0, "epoch": 0.015296940611877625, "grad_norm": 5.408466815948486, "learning_rate": 9.994451593983433e-06, "logp/chosen": -278.0, "logp/rejected": -306.0, "loss": 0.440185546875, "reward/accuracy": 0.8125, "reward/chosen": -0.74609375, "reward/margin": 0.96484375, "reward/rejected": -1.7109375, "step": 51 }, { "approx. KL/chosen": 45.5, "approx. KL/rejected": 1208.0, "epoch": 0.015596880623875225, "grad_norm": 4.7575907707214355, "learning_rate": 9.994227481521544e-06, "logp/chosen": -213.0, "logp/rejected": -480.0, "loss": 0.451171875, "reward/accuracy": 0.6875, "reward/chosen": -0.65625, "reward/margin": 1.8046875, "reward/rejected": -2.453125, "step": 52 }, { "approx. KL/chosen": 74.0, "approx. KL/rejected": 708.0, "epoch": 0.015896820635872826, "grad_norm": 4.015054702758789, "learning_rate": 9.993998934639474e-06, "logp/chosen": -388.0, "logp/rejected": -352.0, "loss": 0.4254150390625, "reward/accuracy": 0.8125, "reward/chosen": -0.75, "reward/margin": 1.78125, "reward/rejected": -2.53125, "step": 53 }, { "approx. KL/chosen": 47.5, "approx. KL/rejected": 102.5, "epoch": 0.016196760647870425, "grad_norm": 3.9811277389526367, "learning_rate": 9.993765953540144e-06, "logp/chosen": -294.0, "logp/rejected": -318.0, "loss": 0.588134765625, "reward/accuracy": 0.6875, "reward/chosen": -0.6015625, "reward/margin": 0.478515625, "reward/rejected": -1.078125, "step": 54 }, { "approx. KL/chosen": 58.5, "approx. KL/rejected": 308.0, "epoch": 0.016496700659868028, "grad_norm": 4.716079235076904, "learning_rate": 9.993528538430427e-06, "logp/chosen": -422.0, "logp/rejected": -416.0, "loss": 0.484466552734375, "reward/accuracy": 0.625, "reward/chosen": -0.609375, "reward/margin": 1.1484375, "reward/rejected": -1.7578125, "step": 55 }, { "approx. KL/chosen": 52.0, "approx. KL/rejected": 728.0, "epoch": 0.016796640671865627, "grad_norm": 3.1597392559051514, "learning_rate": 9.993286689521124e-06, "logp/chosen": -374.0, "logp/rejected": -372.0, "loss": 0.25244140625, "reward/accuracy": 0.9375, "reward/chosen": -0.5703125, "reward/margin": 2.1875, "reward/rejected": -2.75, "step": 56 }, { "approx. KL/chosen": 74.5, "approx. KL/rejected": 251.0, "epoch": 0.017096580683863227, "grad_norm": 4.344589710235596, "learning_rate": 9.993040407026975e-06, "logp/chosen": -268.0, "logp/rejected": -262.0, "loss": 0.4754638671875, "reward/accuracy": 0.75, "reward/chosen": -0.9609375, "reward/margin": 0.9296875, "reward/rejected": -1.890625, "step": 57 }, { "approx. KL/chosen": 114.0, "approx. KL/rejected": 556.0, "epoch": 0.01739652069586083, "grad_norm": 5.938233852386475, "learning_rate": 9.992789691166654e-06, "logp/chosen": -344.0, "logp/rejected": -366.0, "loss": 0.50091552734375, "reward/accuracy": 0.75, "reward/chosen": -1.1015625, "reward/margin": 1.359375, "reward/rejected": -2.453125, "step": 58 }, { "approx. KL/chosen": 87.5, "approx. KL/rejected": 424.0, "epoch": 0.01769646070785843, "grad_norm": 9.594034194946289, "learning_rate": 9.992534542162778e-06, "logp/chosen": -294.0, "logp/rejected": -340.0, "loss": 0.514495849609375, "reward/accuracy": 0.625, "reward/chosen": -0.875, "reward/margin": 1.2265625, "reward/rejected": -2.09375, "step": 59 }, { "approx. KL/chosen": 65.0, "approx. KL/rejected": 136.0, "epoch": 0.017996400719856028, "grad_norm": 4.865693092346191, "learning_rate": 9.992274960241894e-06, "logp/chosen": -454.0, "logp/rejected": -314.0, "loss": 0.5029296875, "reward/accuracy": 0.8125, "reward/chosen": -0.78125, "reward/margin": 0.69140625, "reward/rejected": -1.46875, "step": 60 }, { "approx. KL/chosen": 120.5, "approx. KL/rejected": 486.0, "epoch": 0.01829634073185363, "grad_norm": 6.311225414276123, "learning_rate": 9.992010945634487e-06, "logp/chosen": -195.0, "logp/rejected": -292.0, "loss": 0.7635498046875, "reward/accuracy": 0.5625, "reward/chosen": -1.2734375, "reward/margin": 0.76953125, "reward/rejected": -2.046875, "step": 61 }, { "approx. KL/chosen": 33.0, "approx. KL/rejected": 274.0, "epoch": 0.01859628074385123, "grad_norm": 3.6240177154541016, "learning_rate": 9.99174249857498e-06, "logp/chosen": -368.0, "logp/rejected": -354.0, "loss": 0.315582275390625, "reward/accuracy": 0.75, "reward/chosen": -0.49609375, "reward/margin": 1.4375, "reward/rejected": -1.9296875, "step": 62 }, { "approx. KL/chosen": 107.0, "approx. KL/rejected": 209.0, "epoch": 0.01889622075584883, "grad_norm": 5.273666858673096, "learning_rate": 9.991469619301723e-06, "logp/chosen": -470.0, "logp/rejected": -312.0, "loss": 0.5147705078125, "reward/accuracy": 0.75, "reward/chosen": -1.0390625, "reward/margin": 0.7578125, "reward/rejected": -1.796875, "step": 63 }, { "approx. KL/chosen": 416.0, "approx. KL/rejected": 151.0, "epoch": 0.01919616076784643, "grad_norm": 9.599977493286133, "learning_rate": 9.991192308057018e-06, "logp/chosen": -436.0, "logp/rejected": -352.0, "loss": 1.2757568359375, "reward/accuracy": 0.5, "reward/chosen": -1.53125, "reward/margin": -0.1455078125, "reward/rejected": -1.390625, "step": 64 }, { "approx. KL/chosen": 98.5, "approx. KL/rejected": 344.0, "epoch": 0.019496100779844032, "grad_norm": 4.793121337890625, "learning_rate": 9.990910565087084e-06, "logp/chosen": -422.0, "logp/rejected": -406.0, "loss": 0.484619140625, "reward/accuracy": 0.8125, "reward/chosen": -1.0625, "reward/margin": 1.0859375, "reward/rejected": -2.140625, "step": 65 }, { "approx. KL/chosen": 126.5, "approx. KL/rejected": 241.0, "epoch": 0.01979604079184163, "grad_norm": 4.962535858154297, "learning_rate": 9.990624390642088e-06, "logp/chosen": -316.0, "logp/rejected": -410.0, "loss": 0.46661376953125, "reward/accuracy": 0.8125, "reward/chosen": -1.078125, "reward/margin": 0.84375, "reward/rejected": -1.921875, "step": 66 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 516.0, "epoch": 0.02009598080383923, "grad_norm": 4.790317058563232, "learning_rate": 9.990333784976125e-06, "logp/chosen": -388.0, "logp/rejected": -384.0, "loss": 0.48516845703125, "reward/accuracy": 0.75, "reward/chosen": -1.015625, "reward/margin": 1.3203125, "reward/rejected": -2.328125, "step": 67 }, { "approx. KL/chosen": 108.5, "approx. KL/rejected": 360.0, "epoch": 0.020395920815836834, "grad_norm": 4.453270435333252, "learning_rate": 9.990038748347226e-06, "logp/chosen": -344.0, "logp/rejected": -340.0, "loss": 0.56268310546875, "reward/accuracy": 0.6875, "reward/chosen": -1.1875, "reward/margin": 1.0390625, "reward/rejected": -2.21875, "step": 68 }, { "approx. KL/chosen": 402.0, "approx. KL/rejected": 350.0, "epoch": 0.020695860827834433, "grad_norm": 10.195316314697266, "learning_rate": 9.989739281017356e-06, "logp/chosen": -432.0, "logp/rejected": -422.0, "loss": 0.89453125, "reward/accuracy": 0.4375, "reward/chosen": -1.7578125, "reward/margin": 0.52734375, "reward/rejected": -2.28125, "step": 69 }, { "approx. KL/chosen": 101.0, "approx. KL/rejected": 260.0, "epoch": 0.020995800839832032, "grad_norm": 4.615387439727783, "learning_rate": 9.989435383252419e-06, "logp/chosen": -378.0, "logp/rejected": -388.0, "loss": 0.508056640625, "reward/accuracy": 0.75, "reward/chosen": -1.046875, "reward/margin": 0.96875, "reward/rejected": -2.015625, "step": 70 }, { "approx. KL/chosen": 167.0, "approx. KL/rejected": 258.0, "epoch": 0.021295740851829635, "grad_norm": 6.032623767852783, "learning_rate": 9.989127055322244e-06, "logp/chosen": -396.0, "logp/rejected": -374.0, "loss": 0.494873046875, "reward/accuracy": 0.6875, "reward/chosen": -1.03125, "reward/margin": 0.9765625, "reward/rejected": -2.0, "step": 71 }, { "approx. KL/chosen": 72.5, "approx. KL/rejected": 223.0, "epoch": 0.021595680863827234, "grad_norm": 4.957800388336182, "learning_rate": 9.9888142975006e-06, "logp/chosen": -298.0, "logp/rejected": -292.0, "loss": 0.513916015625, "reward/accuracy": 0.625, "reward/chosen": -0.74609375, "reward/margin": 0.89453125, "reward/rejected": -1.640625, "step": 72 }, { "approx. KL/chosen": 69.0, "approx. KL/rejected": 174.0, "epoch": 0.021895620875824834, "grad_norm": 5.465092182159424, "learning_rate": 9.988497110065187e-06, "logp/chosen": -398.0, "logp/rejected": -352.0, "loss": 0.6683349609375, "reward/accuracy": 0.6875, "reward/chosen": -0.93359375, "reward/margin": 0.58203125, "reward/rejected": -1.515625, "step": 73 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 274.0, "epoch": 0.022195560887822437, "grad_norm": 7.835497856140137, "learning_rate": 9.988175493297638e-06, "logp/chosen": -298.0, "logp/rejected": -312.0, "loss": 0.710052490234375, "reward/accuracy": 0.625, "reward/chosen": -1.5078125, "reward/margin": 0.6484375, "reward/rejected": -2.15625, "step": 74 }, { "approx. KL/chosen": 55.0, "approx. KL/rejected": 316.0, "epoch": 0.022495500899820036, "grad_norm": 3.422612428665161, "learning_rate": 9.987849447483522e-06, "logp/chosen": -332.0, "logp/rejected": -298.0, "loss": 0.3388671875, "reward/accuracy": 0.8125, "reward/chosen": -0.75, "reward/margin": 1.4765625, "reward/rejected": -2.21875, "step": 75 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 832.0, "epoch": 0.022795440911817635, "grad_norm": 6.4288716316223145, "learning_rate": 9.987518972912336e-06, "logp/chosen": -322.0, "logp/rejected": -404.0, "loss": 0.480712890625, "reward/accuracy": 0.8125, "reward/chosen": -1.296875, "reward/margin": 1.4453125, "reward/rejected": -2.75, "step": 76 }, { "approx. KL/chosen": 92.0, "approx. KL/rejected": 116.5, "epoch": 0.02309538092381524, "grad_norm": 6.257024765014648, "learning_rate": 9.987184069877514e-06, "logp/chosen": -490.0, "logp/rejected": -410.0, "loss": 0.765625, "reward/accuracy": 0.5625, "reward/chosen": -0.80859375, "reward/margin": 0.404296875, "reward/rejected": -1.2109375, "step": 77 }, { "approx. KL/chosen": 238.0, "approx. KL/rejected": 128.0, "epoch": 0.023395320935812838, "grad_norm": 12.369327545166016, "learning_rate": 9.986844738676414e-06, "logp/chosen": -284.0, "logp/rejected": -236.0, "loss": 1.0870361328125, "reward/accuracy": 0.4375, "reward/chosen": -1.6171875, "reward/margin": -0.255859375, "reward/rejected": -1.359375, "step": 78 }, { "approx. KL/chosen": 62.25, "approx. KL/rejected": 282.0, "epoch": 0.023695260947810437, "grad_norm": 2.963195323944092, "learning_rate": 9.986500979610336e-06, "logp/chosen": -294.0, "logp/rejected": -282.0, "loss": 0.298248291015625, "reward/accuracy": 0.9375, "reward/chosen": -0.470703125, "reward/margin": 1.4375, "reward/rejected": -1.90625, "step": 79 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 426.0, "epoch": 0.02399520095980804, "grad_norm": 5.973742485046387, "learning_rate": 9.986152792984506e-06, "logp/chosen": -392.0, "logp/rejected": -334.0, "loss": 0.46124267578125, "reward/accuracy": 0.625, "reward/chosen": -1.0, "reward/margin": 1.328125, "reward/rejected": -2.328125, "step": 80 }, { "approx. KL/chosen": 91.5, "approx. KL/rejected": 191.0, "epoch": 0.02429514097180564, "grad_norm": 5.843038558959961, "learning_rate": 9.98580017910808e-06, "logp/chosen": -280.0, "logp/rejected": -266.0, "loss": 0.4871826171875, "reward/accuracy": 0.75, "reward/chosen": -0.890625, "reward/margin": 0.75390625, "reward/rejected": -1.6484375, "step": 81 }, { "approx. KL/chosen": 45.5, "approx. KL/rejected": 243.0, "epoch": 0.02459508098380324, "grad_norm": 3.4459831714630127, "learning_rate": 9.985443138294151e-06, "logp/chosen": -258.0, "logp/rejected": -270.0, "loss": 0.3756103515625, "reward/accuracy": 0.875, "reward/chosen": -0.5625, "reward/margin": 1.25, "reward/rejected": -1.8046875, "step": 82 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 334.0, "epoch": 0.02489502099580084, "grad_norm": 7.862899303436279, "learning_rate": 9.985081670859735e-06, "logp/chosen": -454.0, "logp/rejected": -418.0, "loss": 0.632293701171875, "reward/accuracy": 0.75, "reward/chosen": -0.93359375, "reward/margin": 1.1015625, "reward/rejected": -2.03125, "step": 83 }, { "approx. KL/chosen": 74.0, "approx. KL/rejected": 262.0, "epoch": 0.02519496100779844, "grad_norm": 5.881145000457764, "learning_rate": 9.984715777125784e-06, "logp/chosen": -356.0, "logp/rejected": -354.0, "loss": 0.5294189453125, "reward/accuracy": 0.625, "reward/chosen": -0.9375, "reward/margin": 0.96484375, "reward/rejected": -1.8984375, "step": 84 }, { "approx. KL/chosen": 58.25, "approx. KL/rejected": 236.0, "epoch": 0.02549490101979604, "grad_norm": 4.589963436126709, "learning_rate": 9.984345457417177e-06, "logp/chosen": -408.0, "logp/rejected": -326.0, "loss": 0.483642578125, "reward/accuracy": 0.6875, "reward/chosen": -0.7578125, "reward/margin": 1.0, "reward/rejected": -1.7578125, "step": 85 }, { "approx. KL/chosen": 59.5, "approx. KL/rejected": 276.0, "epoch": 0.02579484103179364, "grad_norm": 4.458577632904053, "learning_rate": 9.983970712062727e-06, "logp/chosen": -456.0, "logp/rejected": -344.0, "loss": 0.4307861328125, "reward/accuracy": 0.75, "reward/chosen": -0.71875, "reward/margin": 1.3203125, "reward/rejected": -2.046875, "step": 86 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 202.0, "epoch": 0.026094781043791242, "grad_norm": 5.860467433929443, "learning_rate": 9.983591541395171e-06, "logp/chosen": -460.0, "logp/rejected": -428.0, "loss": 0.624755859375, "reward/accuracy": 0.6875, "reward/chosen": -1.3671875, "reward/margin": 0.3125, "reward/rejected": -1.6796875, "step": 87 }, { "approx. KL/chosen": 88.0, "approx. KL/rejected": 314.0, "epoch": 0.026394721055788842, "grad_norm": 4.581920623779297, "learning_rate": 9.983207945751178e-06, "logp/chosen": -290.0, "logp/rejected": -422.0, "loss": 0.6087646484375, "reward/accuracy": 0.6875, "reward/chosen": -1.0703125, "reward/margin": 0.78515625, "reward/rejected": -1.859375, "step": 88 }, { "approx. KL/chosen": 103.5, "approx. KL/rejected": 416.0, "epoch": 0.02669466106778644, "grad_norm": 5.078807830810547, "learning_rate": 9.98281992547135e-06, "logp/chosen": -290.0, "logp/rejected": -438.0, "loss": 0.40313720703125, "reward/accuracy": 0.875, "reward/chosen": -1.2578125, "reward/margin": 1.21875, "reward/rejected": -2.484375, "step": 89 }, { "approx. KL/chosen": 96.5, "approx. KL/rejected": 228.0, "epoch": 0.026994601079784044, "grad_norm": 4.634148597717285, "learning_rate": 9.98242748090021e-06, "logp/chosen": -332.0, "logp/rejected": -278.0, "loss": 0.48675537109375, "reward/accuracy": 0.8125, "reward/chosen": -1.1015625, "reward/margin": 0.8359375, "reward/rejected": -1.9375, "step": 90 }, { "approx. KL/chosen": 121.5, "approx. KL/rejected": 458.0, "epoch": 0.027294541091781643, "grad_norm": 4.218292236328125, "learning_rate": 9.982030612386214e-06, "logp/chosen": -398.0, "logp/rejected": -382.0, "loss": 0.39990234375, "reward/accuracy": 0.8125, "reward/chosen": -1.1640625, "reward/margin": 1.4140625, "reward/rejected": -2.578125, "step": 91 }, { "approx. KL/chosen": 67.0, "approx. KL/rejected": 193.0, "epoch": 0.027594481103779243, "grad_norm": 5.350975036621094, "learning_rate": 9.981629320281745e-06, "logp/chosen": -368.0, "logp/rejected": -322.0, "loss": 0.513916015625, "reward/accuracy": 0.75, "reward/chosen": -0.81640625, "reward/margin": 0.90625, "reward/rejected": -1.7265625, "step": 92 }, { "approx. KL/chosen": 109.5, "approx. KL/rejected": 352.0, "epoch": 0.027894421115776846, "grad_norm": 3.763650417327881, "learning_rate": 9.981223604943113e-06, "logp/chosen": -286.0, "logp/rejected": -207.0, "loss": 0.3497314453125, "reward/accuracy": 0.9375, "reward/chosen": -1.265625, "reward/margin": 1.21875, "reward/rejected": -2.484375, "step": 93 }, { "approx. KL/chosen": 147.0, "approx. KL/rejected": 234.0, "epoch": 0.028194361127774445, "grad_norm": 6.644477367401123, "learning_rate": 9.98081346673056e-06, "logp/chosen": -324.0, "logp/rejected": -233.0, "loss": 0.6683349609375, "reward/accuracy": 0.6875, "reward/chosen": -1.2890625, "reward/margin": 0.609375, "reward/rejected": -1.8984375, "step": 94 }, { "approx. KL/chosen": 73.0, "approx. KL/rejected": 332.0, "epoch": 0.028494301139772044, "grad_norm": 4.6056294441223145, "learning_rate": 9.980398906008248e-06, "logp/chosen": -316.0, "logp/rejected": -384.0, "loss": 0.420166015625, "reward/accuracy": 0.8125, "reward/chosen": -0.95703125, "reward/margin": 1.140625, "reward/rejected": -2.09375, "step": 95 }, { "approx. KL/chosen": 36.0, "approx. KL/rejected": 119.0, "epoch": 0.028794241151769647, "grad_norm": 4.929848670959473, "learning_rate": 9.97997992314427e-06, "logp/chosen": -346.0, "logp/rejected": -300.0, "loss": 0.545166015625, "reward/accuracy": 0.75, "reward/chosen": -0.69921875, "reward/margin": 0.55078125, "reward/rejected": -1.25, "step": 96 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 178.0, "epoch": 0.029094181163767247, "grad_norm": 7.569291114807129, "learning_rate": 9.979556518510646e-06, "logp/chosen": -288.0, "logp/rejected": -244.0, "loss": 0.680908203125, "reward/accuracy": 0.5625, "reward/chosen": -1.0546875, "reward/margin": 0.4375, "reward/rejected": -1.4921875, "step": 97 }, { "approx. KL/chosen": 66.0, "approx. KL/rejected": 364.0, "epoch": 0.029394121175764846, "grad_norm": 3.658496379852295, "learning_rate": 9.97912869248332e-06, "logp/chosen": -374.0, "logp/rejected": -382.0, "loss": 0.40045166015625, "reward/accuracy": 0.75, "reward/chosen": -0.7265625, "reward/margin": 1.2734375, "reward/rejected": -2.0, "step": 98 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 372.0, "epoch": 0.02969406118776245, "grad_norm": 11.60836124420166, "learning_rate": 9.978696445442161e-06, "logp/chosen": -438.0, "logp/rejected": -326.0, "loss": 0.92059326171875, "reward/accuracy": 0.625, "reward/chosen": -1.484375, "reward/margin": 0.54296875, "reward/rejected": -2.03125, "step": 99 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 372.0, "epoch": 0.029994001199760048, "grad_norm": 5.182374477386475, "learning_rate": 9.978259777770967e-06, "logp/chosen": -352.0, "logp/rejected": -310.0, "loss": 0.470947265625, "reward/accuracy": 0.6875, "reward/chosen": -1.0546875, "reward/margin": 1.0234375, "reward/rejected": -2.078125, "step": 100 }, { "approx. KL/chosen": 63.25, "approx. KL/rejected": 150.0, "epoch": 0.030293941211757647, "grad_norm": 5.194439888000488, "learning_rate": 9.97781868985746e-06, "logp/chosen": -428.0, "logp/rejected": -410.0, "loss": 0.6552734375, "reward/accuracy": 0.625, "reward/chosen": -0.7421875, "reward/margin": 0.451171875, "reward/rejected": -1.1953125, "step": 101 }, { "approx. KL/chosen": 89.5, "approx. KL/rejected": 96.0, "epoch": 0.03059388122375525, "grad_norm": 6.613986492156982, "learning_rate": 9.977373182093284e-06, "logp/chosen": -390.0, "logp/rejected": -320.0, "loss": 0.849853515625, "reward/accuracy": 0.625, "reward/chosen": -0.7421875, "reward/margin": 0.1435546875, "reward/rejected": -0.88671875, "step": 102 }, { "approx. KL/chosen": 19.5, "approx. KL/rejected": 195.0, "epoch": 0.03089382123575285, "grad_norm": 3.518120050430298, "learning_rate": 9.976923254874008e-06, "logp/chosen": -294.0, "logp/rejected": -346.0, "loss": 0.372802734375, "reward/accuracy": 0.8125, "reward/chosen": -0.203125, "reward/margin": 1.2890625, "reward/rejected": -1.4921875, "step": 103 }, { "approx. KL/chosen": 71.0, "approx. KL/rejected": 212.0, "epoch": 0.03119376124775045, "grad_norm": 4.096585750579834, "learning_rate": 9.976468908599133e-06, "logp/chosen": -312.0, "logp/rejected": -320.0, "loss": 0.5028076171875, "reward/accuracy": 0.8125, "reward/chosen": -0.89453125, "reward/margin": 0.7421875, "reward/rejected": -1.640625, "step": 104 }, { "approx. KL/chosen": 40.0, "approx. KL/rejected": 208.0, "epoch": 0.03149370125974805, "grad_norm": 3.706989049911499, "learning_rate": 9.97601014367207e-06, "logp/chosen": -356.0, "logp/rejected": -274.0, "loss": 0.2861328125, "reward/accuracy": 0.875, "reward/chosen": -0.396484375, "reward/margin": 1.4921875, "reward/rejected": -1.890625, "step": 105 }, { "approx. KL/chosen": 52.5, "approx. KL/rejected": 171.0, "epoch": 0.03179364127174565, "grad_norm": 4.492401123046875, "learning_rate": 9.975546960500165e-06, "logp/chosen": -284.0, "logp/rejected": -266.0, "loss": 0.3997802734375, "reward/accuracy": 0.875, "reward/chosen": -0.76171875, "reward/margin": 0.87890625, "reward/rejected": -1.640625, "step": 106 }, { "approx. KL/chosen": 62.0, "approx. KL/rejected": 184.0, "epoch": 0.032093581283743254, "grad_norm": 4.061866760253906, "learning_rate": 9.975079359494679e-06, "logp/chosen": -314.0, "logp/rejected": -350.0, "loss": 0.4295654296875, "reward/accuracy": 0.8125, "reward/chosen": -0.80859375, "reward/margin": 0.8828125, "reward/rejected": -1.6875, "step": 107 }, { "approx. KL/chosen": 70.0, "approx. KL/rejected": 219.0, "epoch": 0.03239352129574085, "grad_norm": 4.2008280754089355, "learning_rate": 9.974607341070803e-06, "logp/chosen": -304.0, "logp/rejected": -264.0, "loss": 0.466064453125, "reward/accuracy": 0.75, "reward/chosen": -0.75, "reward/margin": 0.93359375, "reward/rejected": -1.6796875, "step": 108 }, { "approx. KL/chosen": 38.75, "approx. KL/rejected": 292.0, "epoch": 0.03269346130773845, "grad_norm": 3.719769239425659, "learning_rate": 9.974130905647644e-06, "logp/chosen": -386.0, "logp/rejected": -336.0, "loss": 0.31610107421875, "reward/accuracy": 0.875, "reward/chosen": -0.5, "reward/margin": 1.4609375, "reward/rejected": -1.9609375, "step": 109 }, { "approx. KL/chosen": 83.5, "approx. KL/rejected": 230.0, "epoch": 0.032993401319736056, "grad_norm": 5.226969242095947, "learning_rate": 9.973650053648235e-06, "logp/chosen": -378.0, "logp/rejected": -400.0, "loss": 0.5792236328125, "reward/accuracy": 0.625, "reward/chosen": -0.9375, "reward/margin": 0.69140625, "reward/rejected": -1.6328125, "step": 110 }, { "approx. KL/chosen": 146.0, "approx. KL/rejected": 185.0, "epoch": 0.03329334133173365, "grad_norm": 6.04603910446167, "learning_rate": 9.973164785499527e-06, "logp/chosen": -346.0, "logp/rejected": -430.0, "loss": 0.60302734375, "reward/accuracy": 0.6875, "reward/chosen": -1.2265625, "reward/margin": 0.396484375, "reward/rejected": -1.625, "step": 111 }, { "approx. KL/chosen": 64.5, "approx. KL/rejected": 332.0, "epoch": 0.033593281343731254, "grad_norm": 4.835385322570801, "learning_rate": 9.972675101632395e-06, "logp/chosen": -370.0, "logp/rejected": -430.0, "loss": 0.46148681640625, "reward/accuracy": 0.6875, "reward/chosen": -0.734375, "reward/margin": 1.2890625, "reward/rejected": -2.015625, "step": 112 }, { "approx. KL/chosen": 81.5, "approx. KL/rejected": 600.0, "epoch": 0.03389322135572886, "grad_norm": 3.9141337871551514, "learning_rate": 9.972181002481634e-06, "logp/chosen": -236.0, "logp/rejected": -264.0, "loss": 0.3719940185546875, "reward/accuracy": 0.75, "reward/chosen": -0.94921875, "reward/margin": 2.0625, "reward/rejected": -3.015625, "step": 113 }, { "approx. KL/chosen": 172.0, "approx. KL/rejected": 420.0, "epoch": 0.03419316136772645, "grad_norm": 5.604537487030029, "learning_rate": 9.97168248848596e-06, "logp/chosen": -424.0, "logp/rejected": -372.0, "loss": 0.597137451171875, "reward/accuracy": 0.625, "reward/chosen": -1.3671875, "reward/margin": 1.109375, "reward/rejected": -2.484375, "step": 114 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 262.0, "epoch": 0.034493101379724056, "grad_norm": 6.7771687507629395, "learning_rate": 9.971179560088005e-06, "logp/chosen": -486.0, "logp/rejected": -360.0, "loss": 0.76397705078125, "reward/accuracy": 0.625, "reward/chosen": -1.375, "reward/margin": 0.470703125, "reward/rejected": -1.84375, "step": 115 }, { "approx. KL/chosen": 104.0, "approx. KL/rejected": 292.0, "epoch": 0.03479304139172166, "grad_norm": 5.316931247711182, "learning_rate": 9.970672217734328e-06, "logp/chosen": -272.0, "logp/rejected": -288.0, "loss": 0.57940673828125, "reward/accuracy": 0.75, "reward/chosen": -0.99609375, "reward/margin": 0.96875, "reward/rejected": -1.9609375, "step": 116 }, { "approx. KL/chosen": 205.0, "approx. KL/rejected": 364.0, "epoch": 0.035092981403719255, "grad_norm": 7.2279276847839355, "learning_rate": 9.970160461875398e-06, "logp/chosen": -328.0, "logp/rejected": -370.0, "loss": 0.73046875, "reward/accuracy": 0.5625, "reward/chosen": -1.7578125, "reward/margin": 0.546875, "reward/rejected": -2.296875, "step": 117 }, { "approx. KL/chosen": 142.0, "approx. KL/rejected": 378.0, "epoch": 0.03539292141571686, "grad_norm": 6.802712440490723, "learning_rate": 9.969644292965612e-06, "logp/chosen": -210.0, "logp/rejected": -296.0, "loss": 0.566131591796875, "reward/accuracy": 0.875, "reward/chosen": -1.3359375, "reward/margin": 0.93359375, "reward/rejected": -2.265625, "step": 118 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 352.0, "epoch": 0.03569286142771446, "grad_norm": 5.23526668548584, "learning_rate": 9.969123711463278e-06, "logp/chosen": -340.0, "logp/rejected": -328.0, "loss": 0.54571533203125, "reward/accuracy": 0.625, "reward/chosen": -1.25, "reward/margin": 0.94140625, "reward/rejected": -2.1875, "step": 119 }, { "approx. KL/chosen": 102.0, "approx. KL/rejected": 320.0, "epoch": 0.035992801439712056, "grad_norm": 6.058311462402344, "learning_rate": 9.968598717830629e-06, "logp/chosen": -318.0, "logp/rejected": -294.0, "loss": 0.5535888671875, "reward/accuracy": 0.75, "reward/chosen": -1.0390625, "reward/margin": 0.90625, "reward/rejected": -1.9453125, "step": 120 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 490.0, "epoch": 0.03629274145170966, "grad_norm": 5.099196434020996, "learning_rate": 9.968069312533806e-06, "logp/chosen": -348.0, "logp/rejected": -370.0, "loss": 0.4586181640625, "reward/accuracy": 0.875, "reward/chosen": -1.2578125, "reward/margin": 1.2578125, "reward/rejected": -2.515625, "step": 121 }, { "approx. KL/chosen": 110.5, "approx. KL/rejected": 334.0, "epoch": 0.03659268146370726, "grad_norm": 4.361644744873047, "learning_rate": 9.967535496042878e-06, "logp/chosen": -330.0, "logp/rejected": -364.0, "loss": 0.5360107421875, "reward/accuracy": 0.8125, "reward/chosen": -1.0546875, "reward/margin": 1.0859375, "reward/rejected": -2.140625, "step": 122 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 490.0, "epoch": 0.03689262147570486, "grad_norm": 5.824214935302734, "learning_rate": 9.966997268831825e-06, "logp/chosen": -396.0, "logp/rejected": -426.0, "loss": 0.613037109375, "reward/accuracy": 0.5625, "reward/chosen": -1.7578125, "reward/margin": 0.69140625, "reward/rejected": -2.453125, "step": 123 }, { "approx. KL/chosen": 110.5, "approx. KL/rejected": 249.0, "epoch": 0.03719256148770246, "grad_norm": 5.914198398590088, "learning_rate": 9.96645463137854e-06, "logp/chosen": -408.0, "logp/rejected": -300.0, "loss": 0.622314453125, "reward/accuracy": 0.75, "reward/chosen": -0.8515625, "reward/margin": 1.0703125, "reward/rejected": -1.921875, "step": 124 }, { "approx. KL/chosen": 61.5, "approx. KL/rejected": 232.0, "epoch": 0.03749250149970006, "grad_norm": 4.5050153732299805, "learning_rate": 9.965907584164837e-06, "logp/chosen": -288.0, "logp/rejected": -272.0, "loss": 0.5081787109375, "reward/accuracy": 0.6875, "reward/chosen": -0.7421875, "reward/margin": 0.93359375, "reward/rejected": -1.6796875, "step": 125 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 338.0, "epoch": 0.03779244151169766, "grad_norm": 7.242123126983643, "learning_rate": 9.965356127676448e-06, "logp/chosen": -304.0, "logp/rejected": -306.0, "loss": 0.50732421875, "reward/accuracy": 0.75, "reward/chosen": -1.5390625, "reward/margin": 0.76953125, "reward/rejected": -2.3125, "step": 126 }, { "approx. KL/chosen": 31.75, "approx. KL/rejected": 580.0, "epoch": 0.03809238152369526, "grad_norm": 3.3079984188079834, "learning_rate": 9.96480026240301e-06, "logp/chosen": -338.0, "logp/rejected": -458.0, "loss": 0.2884521484375, "reward/accuracy": 0.875, "reward/chosen": -0.2138671875, "reward/margin": 2.046875, "reward/rejected": -2.265625, "step": 127 }, { "approx. KL/chosen": 237.0, "approx. KL/rejected": 508.0, "epoch": 0.03839232153569286, "grad_norm": 4.874882221221924, "learning_rate": 9.96423998883809e-06, "logp/chosen": -348.0, "logp/rejected": -360.0, "loss": 0.4766845703125, "reward/accuracy": 0.6875, "reward/chosen": -1.5859375, "reward/margin": 1.2265625, "reward/rejected": -2.8125, "step": 128 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 255.0, "epoch": 0.03869226154769046, "grad_norm": 4.317200183868408, "learning_rate": 9.96367530747915e-06, "logp/chosen": -286.0, "logp/rejected": -312.0, "loss": 0.46630859375, "reward/accuracy": 0.75, "reward/chosen": -0.984375, "reward/margin": 0.87109375, "reward/rejected": -1.859375, "step": 129 }, { "approx. KL/chosen": 48.75, "approx. KL/rejected": 304.0, "epoch": 0.038992201559688064, "grad_norm": 3.7544658184051514, "learning_rate": 9.963106218827582e-06, "logp/chosen": -420.0, "logp/rejected": -243.0, "loss": 0.35382080078125, "reward/accuracy": 0.8125, "reward/chosen": -0.5390625, "reward/margin": 1.4921875, "reward/rejected": -2.03125, "step": 130 }, { "approx. KL/chosen": 65.5, "approx. KL/rejected": 207.0, "epoch": 0.03929214157168566, "grad_norm": 4.332869052886963, "learning_rate": 9.962532723388682e-06, "logp/chosen": -348.0, "logp/rejected": -330.0, "loss": 0.5849609375, "reward/accuracy": 0.6875, "reward/chosen": -0.6953125, "reward/margin": 0.81640625, "reward/rejected": -1.5078125, "step": 131 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 384.0, "epoch": 0.03959208158368326, "grad_norm": 5.384690761566162, "learning_rate": 9.961954821671664e-06, "logp/chosen": -264.0, "logp/rejected": -372.0, "loss": 0.6103515625, "reward/accuracy": 0.625, "reward/chosen": -0.94921875, "reward/margin": 0.9375, "reward/rejected": -1.890625, "step": 132 }, { "approx. KL/chosen": 174.0, "approx. KL/rejected": 286.0, "epoch": 0.039892021595680865, "grad_norm": 11.702309608459473, "learning_rate": 9.96137251418965e-06, "logp/chosen": -280.0, "logp/rejected": -354.0, "loss": 0.65673828125, "reward/accuracy": 0.5625, "reward/chosen": -1.1875, "reward/margin": 0.70703125, "reward/rejected": -1.890625, "step": 133 }, { "approx. KL/chosen": 174.0, "approx. KL/rejected": 356.0, "epoch": 0.04019196160767846, "grad_norm": 5.1906609535217285, "learning_rate": 9.960785801459678e-06, "logp/chosen": -404.0, "logp/rejected": -424.0, "loss": 0.712890625, "reward/accuracy": 0.75, "reward/chosen": -1.296875, "reward/margin": 0.81640625, "reward/rejected": -2.109375, "step": 134 }, { "approx. KL/chosen": 76.0, "approx. KL/rejected": 105.0, "epoch": 0.040491901619676064, "grad_norm": 6.189085006713867, "learning_rate": 9.960194684002695e-06, "logp/chosen": -241.0, "logp/rejected": -266.0, "loss": 0.7386474609375, "reward/accuracy": 0.5625, "reward/chosen": -0.625, "reward/margin": 0.306640625, "reward/rejected": -0.93359375, "step": 135 }, { "approx. KL/chosen": 31.125, "approx. KL/rejected": 167.0, "epoch": 0.04079184163167367, "grad_norm": 4.871098041534424, "learning_rate": 9.95959916234356e-06, "logp/chosen": -482.0, "logp/rejected": -372.0, "loss": 0.37237548828125, "reward/accuracy": 0.8125, "reward/chosen": 0.01123046875, "reward/margin": 1.375, "reward/rejected": -1.3671875, "step": 136 }, { "approx. KL/chosen": 51.5, "approx. KL/rejected": 175.0, "epoch": 0.04109178164367126, "grad_norm": 4.621551036834717, "learning_rate": 9.958999237011041e-06, "logp/chosen": -324.0, "logp/rejected": -340.0, "loss": 0.4942626953125, "reward/accuracy": 0.75, "reward/chosen": -0.431640625, "reward/margin": 0.98828125, "reward/rejected": -1.421875, "step": 137 }, { "approx. KL/chosen": 71.0, "approx. KL/rejected": 150.0, "epoch": 0.041391721655668866, "grad_norm": 6.113485813140869, "learning_rate": 9.958394908537818e-06, "logp/chosen": -312.0, "logp/rejected": -266.0, "loss": 0.545379638671875, "reward/accuracy": 0.75, "reward/chosen": -0.515625, "reward/margin": 0.78515625, "reward/rejected": -1.296875, "step": 138 }, { "approx. KL/chosen": 41.0, "approx. KL/rejected": 410.0, "epoch": 0.04169166166766647, "grad_norm": 5.142750263214111, "learning_rate": 9.95778617746048e-06, "logp/chosen": -262.0, "logp/rejected": -314.0, "loss": 0.31524658203125, "reward/accuracy": 0.9375, "reward/chosen": -0.390625, "reward/margin": 1.8515625, "reward/rejected": -2.25, "step": 139 }, { "approx. KL/chosen": 38.0, "approx. KL/rejected": 229.0, "epoch": 0.041991601679664065, "grad_norm": 3.1154820919036865, "learning_rate": 9.957173044319523e-06, "logp/chosen": -251.0, "logp/rejected": -318.0, "loss": 0.373291015625, "reward/accuracy": 0.75, "reward/chosen": -0.392578125, "reward/margin": 1.328125, "reward/rejected": -1.71875, "step": 140 }, { "approx. KL/chosen": 61.25, "approx. KL/rejected": 166.0, "epoch": 0.04229154169166167, "grad_norm": 5.907874584197998, "learning_rate": 9.956555509659358e-06, "logp/chosen": -448.0, "logp/rejected": -246.0, "loss": 0.60107421875, "reward/accuracy": 0.625, "reward/chosen": -0.765625, "reward/margin": 0.66015625, "reward/rejected": -1.4296875, "step": 141 }, { "approx. KL/chosen": 66.0, "approx. KL/rejected": 240.0, "epoch": 0.04259148170365927, "grad_norm": 5.139753341674805, "learning_rate": 9.955933574028293e-06, "logp/chosen": -274.0, "logp/rejected": -268.0, "loss": 0.47540283203125, "reward/accuracy": 0.75, "reward/chosen": -0.64453125, "reward/margin": 1.03125, "reward/rejected": -1.6796875, "step": 142 }, { "approx. KL/chosen": 42.5, "approx. KL/rejected": 118.5, "epoch": 0.042891421715656866, "grad_norm": 4.8386006355285645, "learning_rate": 9.955307237978557e-06, "logp/chosen": -410.0, "logp/rejected": -376.0, "loss": 0.5333251953125, "reward/accuracy": 0.5625, "reward/chosen": -0.30859375, "reward/margin": 0.6484375, "reward/rejected": -0.95703125, "step": 143 }, { "approx. KL/chosen": 101.5, "approx. KL/rejected": 298.0, "epoch": 0.04319136172765447, "grad_norm": 3.7212910652160645, "learning_rate": 9.954676502066275e-06, "logp/chosen": -348.0, "logp/rejected": -416.0, "loss": 0.3741455078125, "reward/accuracy": 0.8125, "reward/chosen": -0.546875, "reward/margin": 1.3515625, "reward/rejected": -1.8984375, "step": 144 }, { "approx. KL/chosen": 68.0, "approx. KL/rejected": 282.0, "epoch": 0.04349130173965207, "grad_norm": 4.460813522338867, "learning_rate": 9.954041366851486e-06, "logp/chosen": -274.0, "logp/rejected": -316.0, "loss": 0.48541259765625, "reward/accuracy": 0.8125, "reward/chosen": -0.55859375, "reward/margin": 1.0234375, "reward/rejected": -1.578125, "step": 145 }, { "approx. KL/chosen": 72.5, "approx. KL/rejected": 512.0, "epoch": 0.04379124175164967, "grad_norm": 4.300189971923828, "learning_rate": 9.953401832898129e-06, "logp/chosen": -102.5, "logp/rejected": -217.0, "loss": 0.41741943359375, "reward/accuracy": 0.875, "reward/chosen": -0.9453125, "reward/margin": 1.625, "reward/rejected": -2.578125, "step": 146 }, { "approx. KL/chosen": 73.0, "approx. KL/rejected": 236.0, "epoch": 0.04409118176364727, "grad_norm": 3.991074800491333, "learning_rate": 9.952757900774055e-06, "logp/chosen": -446.0, "logp/rejected": -364.0, "loss": 0.391204833984375, "reward/accuracy": 0.75, "reward/chosen": -0.25390625, "reward/margin": 1.4375, "reward/rejected": -1.6875, "step": 147 }, { "approx. KL/chosen": 125.5, "approx. KL/rejected": 167.0, "epoch": 0.04439112177564487, "grad_norm": 5.749067783355713, "learning_rate": 9.952109571051016e-06, "logp/chosen": -294.0, "logp/rejected": -306.0, "loss": 0.77392578125, "reward/accuracy": 0.75, "reward/chosen": -1.2109375, "reward/margin": 0.33203125, "reward/rejected": -1.5390625, "step": 148 }, { "approx. KL/chosen": 81.5, "approx. KL/rejected": 240.0, "epoch": 0.04469106178764247, "grad_norm": 4.353954315185547, "learning_rate": 9.95145684430467e-06, "logp/chosen": -338.0, "logp/rejected": -310.0, "loss": 0.4908447265625, "reward/accuracy": 0.6875, "reward/chosen": -0.9375, "reward/margin": 0.875, "reward/rejected": -1.8125, "step": 149 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 326.0, "epoch": 0.04499100179964007, "grad_norm": 6.3343000411987305, "learning_rate": 9.95079972111458e-06, "logp/chosen": -296.0, "logp/rejected": -282.0, "loss": 0.529541015625, "reward/accuracy": 0.75, "reward/chosen": -1.1796875, "reward/margin": 0.91796875, "reward/rejected": -2.09375, "step": 150 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 352.0, "epoch": 0.045290941811637675, "grad_norm": 4.411291599273682, "learning_rate": 9.95013820206421e-06, "logp/chosen": -310.0, "logp/rejected": -376.0, "loss": 0.462158203125, "reward/accuracy": 0.8125, "reward/chosen": -1.3046875, "reward/margin": 1.0390625, "reward/rejected": -2.34375, "step": 151 }, { "approx. KL/chosen": 120.5, "approx. KL/rejected": 414.0, "epoch": 0.04559088182363527, "grad_norm": 3.859604597091675, "learning_rate": 9.949472287740931e-06, "logp/chosen": -294.0, "logp/rejected": -398.0, "loss": 0.39129638671875, "reward/accuracy": 0.75, "reward/chosen": -1.1953125, "reward/margin": 1.375, "reward/rejected": -2.5625, "step": 152 }, { "approx. KL/chosen": 129.0, "approx. KL/rejected": 450.0, "epoch": 0.045890821835632874, "grad_norm": 3.7446601390838623, "learning_rate": 9.948801978736011e-06, "logp/chosen": -268.0, "logp/rejected": -258.0, "loss": 0.470123291015625, "reward/accuracy": 0.6875, "reward/chosen": -1.0234375, "reward/margin": 1.390625, "reward/rejected": -2.421875, "step": 153 }, { "approx. KL/chosen": 99.5, "approx. KL/rejected": 604.0, "epoch": 0.04619076184763048, "grad_norm": 3.586885929107666, "learning_rate": 9.948127275644627e-06, "logp/chosen": -268.0, "logp/rejected": -376.0, "loss": 0.330352783203125, "reward/accuracy": 0.9375, "reward/chosen": -0.8828125, "reward/margin": 1.7265625, "reward/rejected": -2.609375, "step": 154 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 330.0, "epoch": 0.04649070185962807, "grad_norm": 6.577398300170898, "learning_rate": 9.947448179065854e-06, "logp/chosen": -334.0, "logp/rejected": -332.0, "loss": 0.7913818359375, "reward/accuracy": 0.6875, "reward/chosen": -1.4921875, "reward/margin": 0.58984375, "reward/rejected": -2.078125, "step": 155 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 406.0, "epoch": 0.046790641871625675, "grad_norm": 6.341791152954102, "learning_rate": 9.946764689602664e-06, "logp/chosen": -270.0, "logp/rejected": -290.0, "loss": 0.6767578125, "reward/accuracy": 0.625, "reward/chosen": -1.6875, "reward/margin": 0.8359375, "reward/rejected": -2.53125, "step": 156 }, { "approx. KL/chosen": 109.5, "approx. KL/rejected": 288.0, "epoch": 0.04709058188362328, "grad_norm": 4.793351173400879, "learning_rate": 9.946076807861936e-06, "logp/chosen": -340.0, "logp/rejected": -338.0, "loss": 0.37890625, "reward/accuracy": 0.8125, "reward/chosen": -0.796875, "reward/margin": 1.1875, "reward/rejected": -1.984375, "step": 157 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 254.0, "epoch": 0.047390521895620874, "grad_norm": 7.1717376708984375, "learning_rate": 9.945384534454448e-06, "logp/chosen": -262.0, "logp/rejected": -326.0, "loss": 0.739990234375, "reward/accuracy": 0.5625, "reward/chosen": -1.2578125, "reward/margin": 0.6328125, "reward/rejected": -1.890625, "step": 158 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 324.0, "epoch": 0.04769046190761848, "grad_norm": 7.609292507171631, "learning_rate": 9.944687869994875e-06, "logp/chosen": -350.0, "logp/rejected": -354.0, "loss": 0.6195068359375, "reward/accuracy": 0.5625, "reward/chosen": -1.6640625, "reward/margin": 0.609375, "reward/rejected": -2.28125, "step": 159 }, { "approx. KL/chosen": 138.0, "approx. KL/rejected": 332.0, "epoch": 0.04799040191961608, "grad_norm": 6.688152313232422, "learning_rate": 9.943986815101789e-06, "logp/chosen": -314.0, "logp/rejected": -241.0, "loss": 0.4866943359375, "reward/accuracy": 0.625, "reward/chosen": -1.0, "reward/margin": 1.1015625, "reward/rejected": -2.09375, "step": 160 }, { "approx. KL/chosen": 229.0, "approx. KL/rejected": 724.0, "epoch": 0.048290341931613676, "grad_norm": 4.333008766174316, "learning_rate": 9.943281370397669e-06, "logp/chosen": -316.0, "logp/rejected": -370.0, "loss": 0.458038330078125, "reward/accuracy": 0.75, "reward/chosen": -1.1640625, "reward/margin": 1.7421875, "reward/rejected": -2.90625, "step": 161 }, { "approx. KL/chosen": 93.0, "approx. KL/rejected": 188.0, "epoch": 0.04859028194361128, "grad_norm": 5.394521713256836, "learning_rate": 9.942571536508877e-06, "logp/chosen": -294.0, "logp/rejected": -270.0, "loss": 0.57470703125, "reward/accuracy": 0.6875, "reward/chosen": -0.98828125, "reward/margin": 0.68359375, "reward/rejected": -1.671875, "step": 162 }, { "approx. KL/chosen": 93.5, "approx. KL/rejected": 161.0, "epoch": 0.04889022195560888, "grad_norm": 6.1381120681762695, "learning_rate": 9.941857314065691e-06, "logp/chosen": -390.0, "logp/rejected": -342.0, "loss": 0.65948486328125, "reward/accuracy": 0.625, "reward/chosen": -0.84375, "reward/margin": 0.4296875, "reward/rejected": -1.2734375, "step": 163 }, { "approx. KL/chosen": 66.0, "approx. KL/rejected": 239.0, "epoch": 0.04919016196760648, "grad_norm": 4.103336334228516, "learning_rate": 9.941138703702268e-06, "logp/chosen": -358.0, "logp/rejected": -348.0, "loss": 0.36328125, "reward/accuracy": 0.8125, "reward/chosen": -0.51953125, "reward/margin": 1.3515625, "reward/rejected": -1.8671875, "step": 164 }, { "approx. KL/chosen": 69.0, "approx. KL/rejected": 211.0, "epoch": 0.04949010197960408, "grad_norm": 4.571175575256348, "learning_rate": 9.940415706056673e-06, "logp/chosen": -414.0, "logp/rejected": -334.0, "loss": 0.5384521484375, "reward/accuracy": 0.625, "reward/chosen": -0.73046875, "reward/margin": 0.88671875, "reward/rejected": -1.6171875, "step": 165 }, { "approx. KL/chosen": 40.25, "approx. KL/rejected": 260.0, "epoch": 0.04979004199160168, "grad_norm": 3.72478985786438, "learning_rate": 9.93968832177086e-06, "logp/chosen": -142.0, "logp/rejected": -213.0, "loss": 0.38189697265625, "reward/accuracy": 0.8125, "reward/chosen": -0.44140625, "reward/margin": 1.3125, "reward/rejected": -1.75, "step": 166 }, { "approx. KL/chosen": 108.0, "approx. KL/rejected": 516.0, "epoch": 0.05008998200359928, "grad_norm": 6.049056529998779, "learning_rate": 9.93895655149068e-06, "logp/chosen": -360.0, "logp/rejected": -338.0, "loss": 0.5954551696777344, "reward/accuracy": 0.6875, "reward/chosen": -0.87890625, "reward/margin": 1.6484375, "reward/rejected": -2.53125, "step": 167 }, { "approx. KL/chosen": 117.0, "approx. KL/rejected": 230.0, "epoch": 0.05038992201559688, "grad_norm": 4.741345405578613, "learning_rate": 9.938220395865882e-06, "logp/chosen": -390.0, "logp/rejected": -342.0, "loss": 0.51904296875, "reward/accuracy": 0.6875, "reward/chosen": -0.8515625, "reward/margin": 0.9140625, "reward/rejected": -1.765625, "step": 168 }, { "approx. KL/chosen": 58.5, "approx. KL/rejected": 348.0, "epoch": 0.050689862027594484, "grad_norm": 4.825052738189697, "learning_rate": 9.9374798555501e-06, "logp/chosen": -268.0, "logp/rejected": -380.0, "loss": 0.40985107421875, "reward/accuracy": 0.75, "reward/chosen": -0.58984375, "reward/margin": 1.5390625, "reward/rejected": -2.125, "step": 169 }, { "approx. KL/chosen": 48.75, "approx. KL/rejected": 241.0, "epoch": 0.05098980203959208, "grad_norm": 5.843135833740234, "learning_rate": 9.936734931200869e-06, "logp/chosen": -304.0, "logp/rejected": -340.0, "loss": 0.4913330078125, "reward/accuracy": 0.75, "reward/chosen": -0.5546875, "reward/margin": 1.09375, "reward/rejected": -1.6484375, "step": 170 }, { "approx. KL/chosen": 46.75, "approx. KL/rejected": 406.0, "epoch": 0.05128974205158968, "grad_norm": 5.654855251312256, "learning_rate": 9.935985623479614e-06, "logp/chosen": -286.0, "logp/rejected": -464.0, "loss": 0.5328369140625, "reward/accuracy": 0.8125, "reward/chosen": -0.484375, "reward/margin": 1.3828125, "reward/rejected": -1.8671875, "step": 171 }, { "approx. KL/chosen": 40.75, "approx. KL/rejected": 184.0, "epoch": 0.05158968206358728, "grad_norm": 5.0057597160339355, "learning_rate": 9.935231933051654e-06, "logp/chosen": -247.0, "logp/rejected": -227.0, "loss": 0.4332275390625, "reward/accuracy": 0.6875, "reward/chosen": -0.38671875, "reward/margin": 1.1328125, "reward/rejected": -1.515625, "step": 172 }, { "approx. KL/chosen": 33.75, "approx. KL/rejected": 242.0, "epoch": 0.05188962207558488, "grad_norm": 3.814138174057007, "learning_rate": 9.934473860586195e-06, "logp/chosen": -376.0, "logp/rejected": -260.0, "loss": 0.387451171875, "reward/accuracy": 0.75, "reward/chosen": -0.333984375, "reward/margin": 1.28125, "reward/rejected": -1.6171875, "step": 173 }, { "approx. KL/chosen": 33.75, "approx. KL/rejected": 253.0, "epoch": 0.052189562087582485, "grad_norm": 3.29087233543396, "learning_rate": 9.933711406756339e-06, "logp/chosen": -308.0, "logp/rejected": -372.0, "loss": 0.35430908203125, "reward/accuracy": 0.6875, "reward/chosen": -0.455078125, "reward/margin": 1.484375, "reward/rejected": -1.9375, "step": 174 }, { "approx. KL/chosen": 51.25, "approx. KL/rejected": 316.0, "epoch": 0.05248950209958008, "grad_norm": 3.814889669418335, "learning_rate": 9.932944572239072e-06, "logp/chosen": -376.0, "logp/rejected": -244.0, "loss": 0.2855224609375, "reward/accuracy": 0.875, "reward/chosen": -0.35546875, "reward/margin": 1.8203125, "reward/rejected": -2.171875, "step": 175 }, { "approx. KL/chosen": 16.875, "approx. KL/rejected": 288.0, "epoch": 0.052789442111577684, "grad_norm": 3.22928524017334, "learning_rate": 9.932173357715277e-06, "logp/chosen": -390.0, "logp/rejected": -364.0, "loss": 0.32708740234375, "reward/accuracy": 0.8125, "reward/chosen": -0.1533203125, "reward/margin": 1.7265625, "reward/rejected": -1.8828125, "step": 176 }, { "approx. KL/chosen": 67.0, "approx. KL/rejected": 223.0, "epoch": 0.053089382123575286, "grad_norm": 4.078185081481934, "learning_rate": 9.93139776386972e-06, "logp/chosen": -270.0, "logp/rejected": -344.0, "loss": 0.4517822265625, "reward/accuracy": 0.8125, "reward/chosen": -0.5859375, "reward/margin": 1.0859375, "reward/rejected": -1.671875, "step": 177 }, { "approx. KL/chosen": 47.0, "approx. KL/rejected": 240.0, "epoch": 0.05338932213557288, "grad_norm": 4.326745510101318, "learning_rate": 9.93061779139106e-06, "logp/chosen": -250.0, "logp/rejected": -292.0, "loss": 0.45648193359375, "reward/accuracy": 0.75, "reward/chosen": -0.51171875, "reward/margin": 1.265625, "reward/rejected": -1.78125, "step": 178 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 394.0, "epoch": 0.053689262147570485, "grad_norm": 7.762580871582031, "learning_rate": 9.929833440971842e-06, "logp/chosen": -308.0, "logp/rejected": -276.0, "loss": 0.8115234375, "reward/accuracy": 0.6875, "reward/chosen": -1.4140625, "reward/margin": 0.78515625, "reward/rejected": -2.203125, "step": 179 }, { "approx. KL/chosen": 104.5, "approx. KL/rejected": 316.0, "epoch": 0.05398920215956809, "grad_norm": 7.155220985412598, "learning_rate": 9.929044713308493e-06, "logp/chosen": -342.0, "logp/rejected": -312.0, "loss": 0.493804931640625, "reward/accuracy": 0.6875, "reward/chosen": -0.78515625, "reward/margin": 1.3203125, "reward/rejected": -2.109375, "step": 180 }, { "approx. KL/chosen": 147.0, "approx. KL/rejected": 255.0, "epoch": 0.054289142171565684, "grad_norm": 5.864964962005615, "learning_rate": 9.928251609101337e-06, "logp/chosen": -306.0, "logp/rejected": -256.0, "loss": 0.681396484375, "reward/accuracy": 0.625, "reward/chosen": -1.3984375, "reward/margin": 0.337890625, "reward/rejected": -1.734375, "step": 181 }, { "approx. KL/chosen": 86.5, "approx. KL/rejected": 274.0, "epoch": 0.05458908218356329, "grad_norm": 4.656021595001221, "learning_rate": 9.927454129054577e-06, "logp/chosen": -286.0, "logp/rejected": -418.0, "loss": 0.510498046875, "reward/accuracy": 0.8125, "reward/chosen": -0.86328125, "reward/margin": 1.0546875, "reward/rejected": -1.921875, "step": 182 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 200.0, "epoch": 0.05488902219556089, "grad_norm": 5.721610069274902, "learning_rate": 9.9266522738763e-06, "logp/chosen": -358.0, "logp/rejected": -320.0, "loss": 0.5504150390625, "reward/accuracy": 0.875, "reward/chosen": -0.8359375, "reward/margin": 0.68359375, "reward/rejected": -1.5234375, "step": 183 }, { "approx. KL/chosen": 141.0, "approx. KL/rejected": 572.0, "epoch": 0.055188962207558485, "grad_norm": 4.591719150543213, "learning_rate": 9.925846044278485e-06, "logp/chosen": -360.0, "logp/rejected": -396.0, "loss": 0.34912109375, "reward/accuracy": 0.75, "reward/chosen": -1.171875, "reward/margin": 1.5390625, "reward/rejected": -2.71875, "step": 184 }, { "approx. KL/chosen": 104.5, "approx. KL/rejected": 324.0, "epoch": 0.05548890221955609, "grad_norm": 4.519774436950684, "learning_rate": 9.925035440976987e-06, "logp/chosen": -466.0, "logp/rejected": -452.0, "loss": 0.41015625, "reward/accuracy": 0.75, "reward/chosen": -0.8828125, "reward/margin": 1.3125, "reward/rejected": -2.203125, "step": 185 }, { "approx. KL/chosen": 95.5, "approx. KL/rejected": 352.0, "epoch": 0.05578884223155369, "grad_norm": 4.03705358505249, "learning_rate": 9.92422046469155e-06, "logp/chosen": -352.0, "logp/rejected": -376.0, "loss": 0.38079833984375, "reward/accuracy": 0.75, "reward/chosen": -0.55078125, "reward/margin": 1.6796875, "reward/rejected": -2.234375, "step": 186 }, { "approx. KL/chosen": 213.0, "approx. KL/rejected": 302.0, "epoch": 0.05608878224355129, "grad_norm": 5.843655109405518, "learning_rate": 9.923401116145796e-06, "logp/chosen": -330.0, "logp/rejected": -225.0, "loss": 0.614501953125, "reward/accuracy": 0.6875, "reward/chosen": -1.1171875, "reward/margin": 0.9296875, "reward/rejected": -2.046875, "step": 187 }, { "approx. KL/chosen": 113.5, "approx. KL/rejected": 276.0, "epoch": 0.05638872225554889, "grad_norm": 4.819321632385254, "learning_rate": 9.922577396067239e-06, "logp/chosen": -320.0, "logp/rejected": -278.0, "loss": 0.5528564453125, "reward/accuracy": 0.75, "reward/chosen": -0.94140625, "reward/margin": 0.8359375, "reward/rejected": -1.7734375, "step": 188 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 414.0, "epoch": 0.05668866226754649, "grad_norm": 5.3695549964904785, "learning_rate": 9.92174930518726e-06, "logp/chosen": -312.0, "logp/rejected": -324.0, "loss": 0.66064453125, "reward/accuracy": 0.5625, "reward/chosen": -1.3125, "reward/margin": 1.1875, "reward/rejected": -2.5, "step": 189 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 516.0, "epoch": 0.05698860227954409, "grad_norm": 5.251297950744629, "learning_rate": 9.92091684424113e-06, "logp/chosen": -330.0, "logp/rejected": -332.0, "loss": 0.534271240234375, "reward/accuracy": 0.6875, "reward/chosen": -1.453125, "reward/margin": 1.4765625, "reward/rejected": -2.921875, "step": 190 }, { "approx. KL/chosen": 55.5, "approx. KL/rejected": 326.0, "epoch": 0.05728854229154169, "grad_norm": 3.52413010597229, "learning_rate": 9.920080013968002e-06, "logp/chosen": -336.0, "logp/rejected": -310.0, "loss": 0.38525390625, "reward/accuracy": 0.8125, "reward/chosen": -0.76953125, "reward/margin": 1.2109375, "reward/rejected": -1.984375, "step": 191 }, { "approx. KL/chosen": 53.75, "approx. KL/rejected": 408.0, "epoch": 0.057588482303539294, "grad_norm": 3.362736463546753, "learning_rate": 9.919238815110903e-06, "logp/chosen": -334.0, "logp/rejected": -328.0, "loss": 0.3239898681640625, "reward/accuracy": 0.75, "reward/chosen": -0.76953125, "reward/margin": 1.59375, "reward/rejected": -2.359375, "step": 192 }, { "approx. KL/chosen": 76.0, "approx. KL/rejected": 280.0, "epoch": 0.05788842231553689, "grad_norm": 6.042901039123535, "learning_rate": 9.918393248416743e-06, "logp/chosen": -326.0, "logp/rejected": -342.0, "loss": 0.57867431640625, "reward/accuracy": 0.6875, "reward/chosen": -0.68359375, "reward/margin": 1.234375, "reward/rejected": -1.921875, "step": 193 }, { "approx. KL/chosen": 72.0, "approx. KL/rejected": 272.0, "epoch": 0.05818836232753449, "grad_norm": 4.593782901763916, "learning_rate": 9.917543314636306e-06, "logp/chosen": -332.0, "logp/rejected": -336.0, "loss": 0.4959716796875, "reward/accuracy": 0.6875, "reward/chosen": -0.75390625, "reward/margin": 1.15625, "reward/rejected": -1.9140625, "step": 194 }, { "approx. KL/chosen": 54.75, "approx. KL/rejected": 346.0, "epoch": 0.058488302339532096, "grad_norm": 3.4253346920013428, "learning_rate": 9.916689014524256e-06, "logp/chosen": -254.0, "logp/rejected": -225.0, "loss": 0.33071136474609375, "reward/accuracy": 0.9375, "reward/chosen": -0.4375, "reward/margin": 1.71875, "reward/rejected": -2.15625, "step": 195 }, { "approx. KL/chosen": 46.5, "approx. KL/rejected": 304.0, "epoch": 0.05878824235152969, "grad_norm": 2.956453323364258, "learning_rate": 9.915830348839135e-06, "logp/chosen": -199.0, "logp/rejected": -200.0, "loss": 0.31524658203125, "reward/accuracy": 0.9375, "reward/chosen": -0.7578125, "reward/margin": 1.3515625, "reward/rejected": -2.109375, "step": 196 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 436.0, "epoch": 0.059088182363527295, "grad_norm": 8.39277172088623, "learning_rate": 9.914967318343361e-06, "logp/chosen": -244.0, "logp/rejected": -388.0, "loss": 0.7532958984375, "reward/accuracy": 0.625, "reward/chosen": -1.1328125, "reward/margin": 0.80078125, "reward/rejected": -1.9375, "step": 197 }, { "approx. KL/chosen": 240.0, "approx. KL/rejected": 350.0, "epoch": 0.0593881223755249, "grad_norm": 5.073069095611572, "learning_rate": 9.914099923803224e-06, "logp/chosen": -336.0, "logp/rejected": -384.0, "loss": 0.708251953125, "reward/accuracy": 0.4375, "reward/chosen": -1.8828125, "reward/margin": 0.45703125, "reward/rejected": -2.34375, "step": 198 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 552.0, "epoch": 0.05968806238752249, "grad_norm": 4.7852654457092285, "learning_rate": 9.913228165988893e-06, "logp/chosen": -418.0, "logp/rejected": -318.0, "loss": 0.544921875, "reward/accuracy": 0.75, "reward/chosen": -1.765625, "reward/margin": 1.2265625, "reward/rejected": -2.984375, "step": 199 }, { "approx. KL/chosen": 125.0, "approx. KL/rejected": 292.0, "epoch": 0.059988002399520096, "grad_norm": 4.827452182769775, "learning_rate": 9.91235204567441e-06, "logp/chosen": -350.0, "logp/rejected": -310.0, "loss": 0.481201171875, "reward/accuracy": 0.6875, "reward/chosen": -1.34375, "reward/margin": 0.8046875, "reward/rejected": -2.140625, "step": 200 }, { "approx. KL/chosen": 137.0, "approx. KL/rejected": 416.0, "epoch": 0.0602879424115177, "grad_norm": 3.8496830463409424, "learning_rate": 9.91147156363769e-06, "logp/chosen": -420.0, "logp/rejected": -318.0, "loss": 0.44049072265625, "reward/accuracy": 0.6875, "reward/chosen": -1.0859375, "reward/margin": 1.3515625, "reward/rejected": -2.4375, "step": 201 }, { "approx. KL/chosen": 223.0, "approx. KL/rejected": 484.0, "epoch": 0.060587882423515295, "grad_norm": 5.396090507507324, "learning_rate": 9.910586720660522e-06, "logp/chosen": -360.0, "logp/rejected": -348.0, "loss": 0.519805908203125, "reward/accuracy": 0.6875, "reward/chosen": -1.8125, "reward/margin": 1.0703125, "reward/rejected": -2.875, "step": 202 }, { "approx. KL/chosen": 133.0, "approx. KL/rejected": 352.0, "epoch": 0.0608878224355129, "grad_norm": 6.619226932525635, "learning_rate": 9.909697517528565e-06, "logp/chosen": -278.0, "logp/rejected": -322.0, "loss": 0.67474365234375, "reward/accuracy": 0.6875, "reward/chosen": -1.4609375, "reward/margin": 0.76171875, "reward/rejected": -2.21875, "step": 203 }, { "approx. KL/chosen": 175.0, "approx. KL/rejected": 466.0, "epoch": 0.0611877624475105, "grad_norm": 3.2073068618774414, "learning_rate": 9.90880395503135e-06, "logp/chosen": -352.0, "logp/rejected": -346.0, "loss": 0.32177734375, "reward/accuracy": 0.9375, "reward/chosen": -1.4453125, "reward/margin": 1.1953125, "reward/rejected": -2.640625, "step": 204 }, { "approx. KL/chosen": 184.0, "approx. KL/rejected": 536.0, "epoch": 0.061487702459508096, "grad_norm": 3.630640745162964, "learning_rate": 9.907906033962281e-06, "logp/chosen": -356.0, "logp/rejected": -324.0, "loss": 0.36175537109375, "reward/accuracy": 0.8125, "reward/chosen": -1.4921875, "reward/margin": 1.5234375, "reward/rejected": -3.015625, "step": 205 }, { "approx. KL/chosen": 348.0, "approx. KL/rejected": 362.0, "epoch": 0.0617876424715057, "grad_norm": 7.228971004486084, "learning_rate": 9.907003755118627e-06, "logp/chosen": -314.0, "logp/rejected": -414.0, "loss": 0.71142578125, "reward/accuracy": 0.75, "reward/chosen": -2.234375, "reward/margin": 0.189453125, "reward/rejected": -2.421875, "step": 206 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 225.0, "epoch": 0.0620875824835033, "grad_norm": 5.347393989562988, "learning_rate": 9.906097119301535e-06, "logp/chosen": -300.0, "logp/rejected": -298.0, "loss": 0.63818359375, "reward/accuracy": 0.5625, "reward/chosen": -1.65625, "reward/margin": 0.259765625, "reward/rejected": -1.921875, "step": 207 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 508.0, "epoch": 0.0623875224955009, "grad_norm": 5.269659519195557, "learning_rate": 9.90518612731601e-06, "logp/chosen": -247.0, "logp/rejected": -344.0, "loss": 0.57342529296875, "reward/accuracy": 0.6875, "reward/chosen": -1.515625, "reward/margin": 1.2109375, "reward/rejected": -2.71875, "step": 208 }, { "approx. KL/chosen": 106.0, "approx. KL/rejected": 450.0, "epoch": 0.0626874625074985, "grad_norm": 4.183182239532471, "learning_rate": 9.904270779970932e-06, "logp/chosen": -225.0, "logp/rejected": -270.0, "loss": 0.4346923828125, "reward/accuracy": 0.6875, "reward/chosen": -1.078125, "reward/margin": 1.328125, "reward/rejected": -2.40625, "step": 209 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 524.0, "epoch": 0.0629874025194961, "grad_norm": 3.9586567878723145, "learning_rate": 9.903351078079048e-06, "logp/chosen": -378.0, "logp/rejected": -306.0, "loss": 0.46417236328125, "reward/accuracy": 0.6875, "reward/chosen": -1.6328125, "reward/margin": 1.28125, "reward/rejected": -2.90625, "step": 210 }, { "approx. KL/chosen": 66.5, "approx. KL/rejected": 272.0, "epoch": 0.0632873425314937, "grad_norm": 4.281298637390137, "learning_rate": 9.902427022456967e-06, "logp/chosen": -452.0, "logp/rejected": -446.0, "loss": 0.4957275390625, "reward/accuracy": 0.8125, "reward/chosen": -0.77734375, "reward/margin": 1.0625, "reward/rejected": -1.84375, "step": 211 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 336.0, "epoch": 0.0635872825434913, "grad_norm": 7.94545316696167, "learning_rate": 9.901498613925168e-06, "logp/chosen": -364.0, "logp/rejected": -334.0, "loss": 0.93310546875, "reward/accuracy": 0.375, "reward/chosen": -1.875, "reward/margin": 0.126953125, "reward/rejected": -2.0, "step": 212 }, { "approx. KL/chosen": 54.25, "approx. KL/rejected": 324.0, "epoch": 0.0638872225554889, "grad_norm": 3.620624303817749, "learning_rate": 9.900565853307995e-06, "logp/chosen": -354.0, "logp/rejected": -370.0, "loss": 0.3360595703125, "reward/accuracy": 0.8125, "reward/chosen": -0.75390625, "reward/margin": 1.4609375, "reward/rejected": -2.21875, "step": 213 }, { "approx. KL/chosen": 125.5, "approx. KL/rejected": 300.0, "epoch": 0.06418716256748651, "grad_norm": 4.636273384094238, "learning_rate": 9.89962874143365e-06, "logp/chosen": -256.0, "logp/rejected": -292.0, "loss": 0.528076171875, "reward/accuracy": 0.625, "reward/chosen": -1.0390625, "reward/margin": 1.0859375, "reward/rejected": -2.125, "step": 214 }, { "approx. KL/chosen": 86.5, "approx. KL/rejected": 231.0, "epoch": 0.0644871025794841, "grad_norm": 3.9985015392303467, "learning_rate": 9.898687279134209e-06, "logp/chosen": -396.0, "logp/rejected": -520.0, "loss": 0.44677734375, "reward/accuracy": 0.8125, "reward/chosen": -0.9296875, "reward/margin": 1.03125, "reward/rejected": -1.9609375, "step": 215 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 376.0, "epoch": 0.0647870425914817, "grad_norm": 3.86558198928833, "learning_rate": 9.897741467245599e-06, "logp/chosen": -382.0, "logp/rejected": -338.0, "loss": 0.34375, "reward/accuracy": 0.875, "reward/chosen": -1.25, "reward/margin": 1.2109375, "reward/rejected": -2.453125, "step": 216 }, { "approx. KL/chosen": 171.0, "approx. KL/rejected": 240.0, "epoch": 0.06508698260347931, "grad_norm": 4.96244478225708, "learning_rate": 9.896791306607621e-06, "logp/chosen": -286.0, "logp/rejected": -298.0, "loss": 0.7218017578125, "reward/accuracy": 0.6875, "reward/chosen": -1.390625, "reward/margin": 0.515625, "reward/rejected": -1.90625, "step": 217 }, { "approx. KL/chosen": 102.0, "approx. KL/rejected": 394.0, "epoch": 0.0653869226154769, "grad_norm": 4.59776496887207, "learning_rate": 9.895836798063924e-06, "logp/chosen": -424.0, "logp/rejected": -356.0, "loss": 0.33905029296875, "reward/accuracy": 0.8125, "reward/chosen": -1.0078125, "reward/margin": 1.5, "reward/rejected": -2.515625, "step": 218 }, { "approx. KL/chosen": 106.0, "approx. KL/rejected": 165.0, "epoch": 0.0656868626274745, "grad_norm": 10.217972755432129, "learning_rate": 9.894877942462033e-06, "logp/chosen": -288.0, "logp/rejected": -304.0, "loss": 0.7330322265625, "reward/accuracy": 0.6875, "reward/chosen": -0.91015625, "reward/margin": 0.5703125, "reward/rejected": -1.4765625, "step": 219 }, { "approx. KL/chosen": 109.0, "approx. KL/rejected": 358.0, "epoch": 0.06598680263947211, "grad_norm": 4.1306352615356445, "learning_rate": 9.893914740653319e-06, "logp/chosen": -414.0, "logp/rejected": -430.0, "loss": 0.39508056640625, "reward/accuracy": 0.875, "reward/chosen": -0.921875, "reward/margin": 1.421875, "reward/rejected": -2.34375, "step": 220 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 205.0, "epoch": 0.06628674265146971, "grad_norm": 6.566727638244629, "learning_rate": 9.892947193493022e-06, "logp/chosen": -402.0, "logp/rejected": -418.0, "loss": 0.72216796875, "reward/accuracy": 0.6875, "reward/chosen": -1.0546875, "reward/margin": 0.5625, "reward/rejected": -1.6171875, "step": 221 }, { "approx. KL/chosen": 82.5, "approx. KL/rejected": 278.0, "epoch": 0.0665866826634673, "grad_norm": 4.1373724937438965, "learning_rate": 9.891975301840229e-06, "logp/chosen": -394.0, "logp/rejected": -354.0, "loss": 0.4554443359375, "reward/accuracy": 0.75, "reward/chosen": -0.92578125, "reward/margin": 0.92578125, "reward/rejected": -1.8515625, "step": 222 }, { "approx. KL/chosen": 148.0, "approx. KL/rejected": 362.0, "epoch": 0.06688662267546491, "grad_norm": 3.8777077198028564, "learning_rate": 9.890999066557899e-06, "logp/chosen": -235.0, "logp/rejected": -276.0, "loss": 0.4227294921875, "reward/accuracy": 0.8125, "reward/chosen": -1.203125, "reward/margin": 1.078125, "reward/rejected": -2.28125, "step": 223 }, { "approx. KL/chosen": 64.0, "approx. KL/rejected": 192.0, "epoch": 0.06718656268746251, "grad_norm": 3.900768280029297, "learning_rate": 9.890018488512836e-06, "logp/chosen": -200.0, "logp/rejected": -288.0, "loss": 0.4207763671875, "reward/accuracy": 0.75, "reward/chosen": -0.79296875, "reward/margin": 0.94140625, "reward/rejected": -1.734375, "step": 224 }, { "approx. KL/chosen": 110.5, "approx. KL/rejected": 416.0, "epoch": 0.0674865026994601, "grad_norm": 4.602071762084961, "learning_rate": 9.889033568575706e-06, "logp/chosen": -352.0, "logp/rejected": -356.0, "loss": 0.337371826171875, "reward/accuracy": 0.875, "reward/chosen": -1.0859375, "reward/margin": 1.546875, "reward/rejected": -2.625, "step": 225 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 306.0, "epoch": 0.06778644271145771, "grad_norm": 4.3739519119262695, "learning_rate": 9.888044307621026e-06, "logp/chosen": -330.0, "logp/rejected": -360.0, "loss": 0.5382080078125, "reward/accuracy": 0.6875, "reward/chosen": -1.4140625, "reward/margin": 0.7421875, "reward/rejected": -2.15625, "step": 226 }, { "approx. KL/chosen": 83.5, "approx. KL/rejected": 181.0, "epoch": 0.06808638272345531, "grad_norm": 5.605779647827148, "learning_rate": 9.887050706527173e-06, "logp/chosen": -342.0, "logp/rejected": -342.0, "loss": 0.67578125, "reward/accuracy": 0.5, "reward/chosen": -0.921875, "reward/margin": 0.458984375, "reward/rejected": -1.3828125, "step": 227 }, { "approx. KL/chosen": 68.0, "approx. KL/rejected": 422.0, "epoch": 0.0683863227354529, "grad_norm": 3.0835797786712646, "learning_rate": 9.886052766176374e-06, "logp/chosen": -338.0, "logp/rejected": -360.0, "loss": 0.270965576171875, "reward/accuracy": 0.875, "reward/chosen": -0.64453125, "reward/margin": 1.6953125, "reward/rejected": -2.34375, "step": 228 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 346.0, "epoch": 0.06868626274745052, "grad_norm": 4.855175971984863, "learning_rate": 9.885050487454706e-06, "logp/chosen": -284.0, "logp/rejected": -334.0, "loss": 0.49920654296875, "reward/accuracy": 0.8125, "reward/chosen": -1.0625, "reward/margin": 1.359375, "reward/rejected": -2.421875, "step": 229 }, { "approx. KL/chosen": 109.0, "approx. KL/rejected": 266.0, "epoch": 0.06898620275944811, "grad_norm": 7.692544460296631, "learning_rate": 9.884043871252104e-06, "logp/chosen": -316.0, "logp/rejected": -266.0, "loss": 0.577880859375, "reward/accuracy": 0.75, "reward/chosen": -0.97265625, "reward/margin": 0.8046875, "reward/rejected": -1.7734375, "step": 230 }, { "approx. KL/chosen": 113.0, "approx. KL/rejected": 318.0, "epoch": 0.06928614277144571, "grad_norm": 4.768011093139648, "learning_rate": 9.883032918462352e-06, "logp/chosen": -286.0, "logp/rejected": -233.0, "loss": 0.544921875, "reward/accuracy": 0.8125, "reward/chosen": -1.1796875, "reward/margin": 0.86328125, "reward/rejected": -2.046875, "step": 231 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 336.0, "epoch": 0.06958608278344332, "grad_norm": 8.347225189208984, "learning_rate": 9.882017629983085e-06, "logp/chosen": -322.0, "logp/rejected": -336.0, "loss": 0.72314453125, "reward/accuracy": 0.875, "reward/chosen": -1.1953125, "reward/margin": 0.890625, "reward/rejected": -2.09375, "step": 232 }, { "approx. KL/chosen": 49.25, "approx. KL/rejected": 376.0, "epoch": 0.06988602279544091, "grad_norm": 5.756846904754639, "learning_rate": 9.880998006715786e-06, "logp/chosen": -334.0, "logp/rejected": -294.0, "loss": 0.487518310546875, "reward/accuracy": 0.6875, "reward/chosen": -0.75390625, "reward/margin": 1.4453125, "reward/rejected": -2.203125, "step": 233 }, { "approx. KL/chosen": 77.5, "approx. KL/rejected": 368.0, "epoch": 0.07018596280743851, "grad_norm": 3.340064525604248, "learning_rate": 9.879974049565785e-06, "logp/chosen": -332.0, "logp/rejected": -308.0, "loss": 0.3370513916015625, "reward/accuracy": 0.9375, "reward/chosen": -0.609375, "reward/margin": 1.4765625, "reward/rejected": -2.078125, "step": 234 }, { "approx. KL/chosen": 92.5, "approx. KL/rejected": 354.0, "epoch": 0.07048590281943612, "grad_norm": 3.5333917140960693, "learning_rate": 9.878945759442267e-06, "logp/chosen": -254.0, "logp/rejected": -424.0, "loss": 0.4017333984375, "reward/accuracy": 0.75, "reward/chosen": -0.82421875, "reward/margin": 1.328125, "reward/rejected": -2.15625, "step": 235 }, { "approx. KL/chosen": 118.0, "approx. KL/rejected": 338.0, "epoch": 0.07078584283143372, "grad_norm": 5.234951019287109, "learning_rate": 9.877913137258257e-06, "logp/chosen": -374.0, "logp/rejected": -374.0, "loss": 0.5020751953125, "reward/accuracy": 0.8125, "reward/chosen": -1.1015625, "reward/margin": 1.1171875, "reward/rejected": -2.21875, "step": 236 }, { "approx. KL/chosen": 87.5, "approx. KL/rejected": 338.0, "epoch": 0.07108578284343131, "grad_norm": 5.060104846954346, "learning_rate": 9.876876183930632e-06, "logp/chosen": -220.0, "logp/rejected": -360.0, "loss": 0.36322021484375, "reward/accuracy": 0.875, "reward/chosen": -0.77734375, "reward/margin": 1.5546875, "reward/rejected": -2.328125, "step": 237 }, { "approx. KL/chosen": 51.5, "approx. KL/rejected": 266.0, "epoch": 0.07138572285542892, "grad_norm": 3.4046716690063477, "learning_rate": 9.875834900380113e-06, "logp/chosen": -201.0, "logp/rejected": -286.0, "loss": 0.35467529296875, "reward/accuracy": 0.875, "reward/chosen": -0.61328125, "reward/margin": 1.3671875, "reward/rejected": -1.9765625, "step": 238 }, { "approx. KL/chosen": 79.0, "approx. KL/rejected": 174.0, "epoch": 0.07168566286742652, "grad_norm": 4.5507307052612305, "learning_rate": 9.874789287531262e-06, "logp/chosen": -384.0, "logp/rejected": -402.0, "loss": 0.4351806640625, "reward/accuracy": 0.75, "reward/chosen": -0.61328125, "reward/margin": 1.078125, "reward/rejected": -1.6875, "step": 239 }, { "approx. KL/chosen": 29.875, "approx. KL/rejected": 312.0, "epoch": 0.07198560287942411, "grad_norm": 4.153167724609375, "learning_rate": 9.87373934631249e-06, "logp/chosen": -398.0, "logp/rejected": -426.0, "loss": 0.50048828125, "reward/accuracy": 0.6875, "reward/chosen": -0.58984375, "reward/margin": 1.03125, "reward/rejected": -1.625, "step": 240 }, { "approx. KL/chosen": 121.5, "approx. KL/rejected": 292.0, "epoch": 0.07228554289142172, "grad_norm": 4.755293846130371, "learning_rate": 9.87268507765605e-06, "logp/chosen": -320.0, "logp/rejected": -296.0, "loss": 0.579345703125, "reward/accuracy": 0.75, "reward/chosen": -1.1640625, "reward/margin": 0.75, "reward/rejected": -1.9140625, "step": 241 }, { "approx. KL/chosen": 102.0, "approx. KL/rejected": 528.0, "epoch": 0.07258548290341932, "grad_norm": 4.326462268829346, "learning_rate": 9.871626482498036e-06, "logp/chosen": -374.0, "logp/rejected": -448.0, "loss": 0.4219970703125, "reward/accuracy": 0.8125, "reward/chosen": -1.140625, "reward/margin": 1.34375, "reward/rejected": -2.484375, "step": 242 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 316.0, "epoch": 0.07288542291541691, "grad_norm": 4.76361608505249, "learning_rate": 9.870563561778385e-06, "logp/chosen": -342.0, "logp/rejected": -296.0, "loss": 0.4817962646484375, "reward/accuracy": 0.75, "reward/chosen": -1.0078125, "reward/margin": 1.0625, "reward/rejected": -2.0625, "step": 243 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 428.0, "epoch": 0.07318536292741452, "grad_norm": 4.510195255279541, "learning_rate": 9.869496316440872e-06, "logp/chosen": -251.0, "logp/rejected": -368.0, "loss": 0.5286865234375, "reward/accuracy": 0.75, "reward/chosen": -1.15625, "reward/margin": 1.125, "reward/rejected": -2.28125, "step": 244 }, { "approx. KL/chosen": 101.0, "approx. KL/rejected": 524.0, "epoch": 0.07348530293941212, "grad_norm": 4.054311275482178, "learning_rate": 9.868424747433117e-06, "logp/chosen": -454.0, "logp/rejected": -392.0, "loss": 0.42340087890625, "reward/accuracy": 0.75, "reward/chosen": -0.76171875, "reward/margin": 1.6328125, "reward/rejected": -2.390625, "step": 245 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 428.0, "epoch": 0.07378524295140972, "grad_norm": 5.513525485992432, "learning_rate": 9.86734885570657e-06, "logp/chosen": -344.0, "logp/rejected": -364.0, "loss": 0.519775390625, "reward/accuracy": 0.75, "reward/chosen": -1.6015625, "reward/margin": 1.0, "reward/rejected": -2.609375, "step": 246 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 422.0, "epoch": 0.07408518296340731, "grad_norm": 5.864447116851807, "learning_rate": 9.866268642216535e-06, "logp/chosen": -228.0, "logp/rejected": -294.0, "loss": 0.485015869140625, "reward/accuracy": 0.75, "reward/chosen": -1.2109375, "reward/margin": 1.296875, "reward/rejected": -2.5, "step": 247 }, { "approx. KL/chosen": 154.0, "approx. KL/rejected": 422.0, "epoch": 0.07438512297540492, "grad_norm": 4.468801498413086, "learning_rate": 9.865184107922134e-06, "logp/chosen": -342.0, "logp/rejected": -292.0, "loss": 0.53521728515625, "reward/accuracy": 0.75, "reward/chosen": -1.2421875, "reward/margin": 1.3359375, "reward/rejected": -2.578125, "step": 248 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 458.0, "epoch": 0.07468506298740252, "grad_norm": 3.5539751052856445, "learning_rate": 9.864095253786339e-06, "logp/chosen": -308.0, "logp/rejected": -324.0, "loss": 0.4044189453125, "reward/accuracy": 0.8125, "reward/chosen": -1.3828125, "reward/margin": 1.2421875, "reward/rejected": -2.625, "step": 249 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 648.0, "epoch": 0.07498500299940011, "grad_norm": 5.067789077758789, "learning_rate": 9.863002080775954e-06, "logp/chosen": -302.0, "logp/rejected": -368.0, "loss": 0.5274658203125, "reward/accuracy": 0.875, "reward/chosen": -1.765625, "reward/margin": 1.484375, "reward/rejected": -3.25, "step": 250 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 402.0, "epoch": 0.07528494301139772, "grad_norm": 6.608680248260498, "learning_rate": 9.861904589861615e-06, "logp/chosen": -330.0, "logp/rejected": -408.0, "loss": 0.68548583984375, "reward/accuracy": 0.625, "reward/chosen": -1.171875, "reward/margin": 1.1875, "reward/rejected": -2.359375, "step": 251 }, { "approx. KL/chosen": 122.5, "approx. KL/rejected": 210.0, "epoch": 0.07558488302339532, "grad_norm": 5.002890586853027, "learning_rate": 9.860802782017798e-06, "logp/chosen": -388.0, "logp/rejected": -304.0, "loss": 0.54522705078125, "reward/accuracy": 0.75, "reward/chosen": -0.609375, "reward/margin": 1.125, "reward/rejected": -1.734375, "step": 252 }, { "approx. KL/chosen": 251.0, "approx. KL/rejected": 300.0, "epoch": 0.07588482303539291, "grad_norm": 5.772778511047363, "learning_rate": 9.859696658222804e-06, "logp/chosen": -330.0, "logp/rejected": -354.0, "loss": 0.692626953125, "reward/accuracy": 0.5625, "reward/chosen": -1.8125, "reward/margin": 0.2216796875, "reward/rejected": -2.03125, "step": 253 }, { "approx. KL/chosen": 84.5, "approx. KL/rejected": 374.0, "epoch": 0.07618476304739052, "grad_norm": 4.0458831787109375, "learning_rate": 9.858586219458773e-06, "logp/chosen": -209.0, "logp/rejected": -264.0, "loss": 0.3145751953125, "reward/accuracy": 0.9375, "reward/chosen": -0.69921875, "reward/margin": 1.671875, "reward/rejected": -2.375, "step": 254 }, { "approx. KL/chosen": 125.5, "approx. KL/rejected": 374.0, "epoch": 0.07648470305938812, "grad_norm": 4.693269729614258, "learning_rate": 9.857471466711672e-06, "logp/chosen": -338.0, "logp/rejected": -300.0, "loss": 0.51971435546875, "reward/accuracy": 0.625, "reward/chosen": -1.125, "reward/margin": 1.1640625, "reward/rejected": -2.28125, "step": 255 }, { "approx. KL/chosen": 180.0, "approx. KL/rejected": 398.0, "epoch": 0.07678464307138572, "grad_norm": 6.177493095397949, "learning_rate": 9.856352400971302e-06, "logp/chosen": -424.0, "logp/rejected": -262.0, "loss": 0.6185302734375, "reward/accuracy": 0.625, "reward/chosen": -1.6171875, "reward/margin": 0.890625, "reward/rejected": -2.515625, "step": 256 }, { "approx. KL/chosen": 398.0, "approx. KL/rejected": 764.0, "epoch": 0.07708458308338333, "grad_norm": 4.975598335266113, "learning_rate": 9.855229023231293e-06, "logp/chosen": -250.0, "logp/rejected": -402.0, "loss": 0.43389892578125, "reward/accuracy": 0.8125, "reward/chosen": -2.0625, "reward/margin": 1.5078125, "reward/rejected": -3.578125, "step": 257 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 920.0, "epoch": 0.07738452309538092, "grad_norm": 4.198666095733643, "learning_rate": 9.854101334489097e-06, "logp/chosen": -326.0, "logp/rejected": -410.0, "loss": 0.36944580078125, "reward/accuracy": 0.75, "reward/chosen": -1.4609375, "reward/margin": 2.140625, "reward/rejected": -3.609375, "step": 258 }, { "approx. KL/chosen": 96.0, "approx. KL/rejected": 604.0, "epoch": 0.07768446310737852, "grad_norm": 4.187804222106934, "learning_rate": 9.852969335746004e-06, "logp/chosen": -179.0, "logp/rejected": -272.0, "loss": 0.44644927978515625, "reward/accuracy": 0.8125, "reward/chosen": -0.92578125, "reward/margin": 2.0, "reward/rejected": -2.921875, "step": 259 }, { "approx. KL/chosen": 64.0, "approx. KL/rejected": 476.0, "epoch": 0.07798440311937613, "grad_norm": 4.811328887939453, "learning_rate": 9.851833028007122e-06, "logp/chosen": -270.0, "logp/rejected": -444.0, "loss": 0.5299072265625, "reward/accuracy": 0.6875, "reward/chosen": -0.7734375, "reward/margin": 1.6328125, "reward/rejected": -2.40625, "step": 260 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 366.0, "epoch": 0.07828434313137372, "grad_norm": 6.022731304168701, "learning_rate": 9.850692412281393e-06, "logp/chosen": -442.0, "logp/rejected": -314.0, "loss": 0.49546051025390625, "reward/accuracy": 0.8125, "reward/chosen": -0.703125, "reward/margin": 1.3359375, "reward/rejected": -2.046875, "step": 261 }, { "approx. KL/chosen": 76.0, "approx. KL/rejected": 174.0, "epoch": 0.07858428314337132, "grad_norm": 5.651786804199219, "learning_rate": 9.849547489581579e-06, "logp/chosen": -276.0, "logp/rejected": -314.0, "loss": 0.6162109375, "reward/accuracy": 0.6875, "reward/chosen": -0.921875, "reward/margin": 0.439453125, "reward/rejected": -1.359375, "step": 262 }, { "approx. KL/chosen": 95.5, "approx. KL/rejected": 456.0, "epoch": 0.07888422315536893, "grad_norm": 4.94070291519165, "learning_rate": 9.848398260924266e-06, "logp/chosen": -432.0, "logp/rejected": -332.0, "loss": 0.4111328125, "reward/accuracy": 0.875, "reward/chosen": -0.96875, "reward/margin": 1.5703125, "reward/rejected": -2.546875, "step": 263 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 412.0, "epoch": 0.07918416316736653, "grad_norm": 4.342878341674805, "learning_rate": 9.847244727329865e-06, "logp/chosen": -268.0, "logp/rejected": -338.0, "loss": 0.38897705078125, "reward/accuracy": 0.8125, "reward/chosen": -1.125, "reward/margin": 1.4140625, "reward/rejected": -2.546875, "step": 264 }, { "approx. KL/chosen": 90.5, "approx. KL/rejected": 203.0, "epoch": 0.07948410317936412, "grad_norm": 8.231772422790527, "learning_rate": 9.846086889822611e-06, "logp/chosen": -248.0, "logp/rejected": -260.0, "loss": 0.636474609375, "reward/accuracy": 0.75, "reward/chosen": -0.6015625, "reward/margin": 0.6875, "reward/rejected": -1.2890625, "step": 265 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 338.0, "epoch": 0.07978404319136173, "grad_norm": 3.674196481704712, "learning_rate": 9.844924749430556e-06, "logp/chosen": -234.0, "logp/rejected": -248.0, "loss": 0.3885498046875, "reward/accuracy": 0.9375, "reward/chosen": -0.875, "reward/margin": 1.2109375, "reward/rejected": -2.09375, "step": 266 }, { "approx. KL/chosen": 75.0, "approx. KL/rejected": 364.0, "epoch": 0.08008398320335933, "grad_norm": 4.264566898345947, "learning_rate": 9.843758307185576e-06, "logp/chosen": -322.0, "logp/rejected": -326.0, "loss": 0.373687744140625, "reward/accuracy": 0.875, "reward/chosen": -0.9140625, "reward/margin": 1.3984375, "reward/rejected": -2.3125, "step": 267 }, { "approx. KL/chosen": 50.25, "approx. KL/rejected": 290.0, "epoch": 0.08038392321535692, "grad_norm": 4.190629959106445, "learning_rate": 9.842587564123366e-06, "logp/chosen": -278.0, "logp/rejected": -306.0, "loss": 0.3424072265625, "reward/accuracy": 0.75, "reward/chosen": -0.341796875, "reward/margin": 1.546875, "reward/rejected": -1.890625, "step": 268 }, { "approx. KL/chosen": 137.0, "approx. KL/rejected": 572.0, "epoch": 0.08068386322735453, "grad_norm": 3.874502182006836, "learning_rate": 9.841412521283437e-06, "logp/chosen": -322.0, "logp/rejected": -354.0, "loss": 0.3921051025390625, "reward/accuracy": 0.8125, "reward/chosen": -1.3125, "reward/margin": 1.75, "reward/rejected": -3.0625, "step": 269 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 482.0, "epoch": 0.08098380323935213, "grad_norm": 9.155826568603516, "learning_rate": 9.840233179709125e-06, "logp/chosen": -360.0, "logp/rejected": -322.0, "loss": 0.682861328125, "reward/accuracy": 0.6875, "reward/chosen": -1.734375, "reward/margin": 0.859375, "reward/rejected": -2.59375, "step": 270 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 430.0, "epoch": 0.08128374325134972, "grad_norm": 5.4159770011901855, "learning_rate": 9.839049540447575e-06, "logp/chosen": -374.0, "logp/rejected": -408.0, "loss": 0.5054931640625, "reward/accuracy": 0.75, "reward/chosen": -1.1328125, "reward/margin": 1.109375, "reward/rejected": -2.234375, "step": 271 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 418.0, "epoch": 0.08158368326334733, "grad_norm": 4.624964714050293, "learning_rate": 9.837861604549752e-06, "logp/chosen": -282.0, "logp/rejected": -248.0, "loss": 0.591552734375, "reward/accuracy": 0.75, "reward/chosen": -1.8515625, "reward/margin": 0.65234375, "reward/rejected": -2.5, "step": 272 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 740.0, "epoch": 0.08188362327534493, "grad_norm": 7.0822434425354, "learning_rate": 9.836669373070434e-06, "logp/chosen": -408.0, "logp/rejected": -330.0, "loss": 0.56121826171875, "reward/accuracy": 0.75, "reward/chosen": -1.9921875, "reward/margin": 1.453125, "reward/rejected": -3.453125, "step": 273 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 660.0, "epoch": 0.08218356328734253, "grad_norm": 5.072366714477539, "learning_rate": 9.835472847068215e-06, "logp/chosen": -344.0, "logp/rejected": -286.0, "loss": 0.5086669921875, "reward/accuracy": 0.75, "reward/chosen": -1.109375, "reward/margin": 1.703125, "reward/rejected": -2.8125, "step": 274 }, { "approx. KL/chosen": 115.5, "approx. KL/rejected": 436.0, "epoch": 0.08248350329934014, "grad_norm": 3.6691508293151855, "learning_rate": 9.834272027605502e-06, "logp/chosen": -388.0, "logp/rejected": -396.0, "loss": 0.36236572265625, "reward/accuracy": 0.75, "reward/chosen": -1.2890625, "reward/margin": 1.375, "reward/rejected": -2.65625, "step": 275 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 412.0, "epoch": 0.08278344331133773, "grad_norm": 4.848516941070557, "learning_rate": 9.833066915748513e-06, "logp/chosen": -278.0, "logp/rejected": -248.0, "loss": 0.587158203125, "reward/accuracy": 0.8125, "reward/chosen": -1.9609375, "reward/margin": 0.64453125, "reward/rejected": -2.609375, "step": 276 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 752.0, "epoch": 0.08308338332333533, "grad_norm": 3.793009042739868, "learning_rate": 9.831857512567277e-06, "logp/chosen": -388.0, "logp/rejected": -764.0, "loss": 0.407470703125, "reward/accuracy": 0.875, "reward/chosen": -1.375, "reward/margin": 1.8203125, "reward/rejected": -3.1875, "step": 277 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 664.0, "epoch": 0.08338332333533294, "grad_norm": 7.680938243865967, "learning_rate": 9.830643819135637e-06, "logp/chosen": -424.0, "logp/rejected": -408.0, "loss": 0.47100830078125, "reward/accuracy": 0.8125, "reward/chosen": -1.7734375, "reward/margin": 1.484375, "reward/rejected": -3.25, "step": 278 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 624.0, "epoch": 0.08368326334733053, "grad_norm": 5.079525470733643, "learning_rate": 9.829425836531237e-06, "logp/chosen": -382.0, "logp/rejected": -312.0, "loss": 0.4081573486328125, "reward/accuracy": 0.875, "reward/chosen": -1.3828125, "reward/margin": 1.6640625, "reward/rejected": -3.046875, "step": 279 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 584.0, "epoch": 0.08398320335932813, "grad_norm": 8.030095100402832, "learning_rate": 9.82820356583554e-06, "logp/chosen": -360.0, "logp/rejected": -344.0, "loss": 0.48529052734375, "reward/accuracy": 0.8125, "reward/chosen": -1.75, "reward/margin": 1.3828125, "reward/rejected": -3.140625, "step": 280 }, { "approx. KL/chosen": 141.0, "approx. KL/rejected": 812.0, "epoch": 0.08428314337132574, "grad_norm": 3.5617856979370117, "learning_rate": 9.826977008133811e-06, "logp/chosen": -374.0, "logp/rejected": -362.0, "loss": 0.23638916015625, "reward/accuracy": 0.9375, "reward/chosen": -1.265625, "reward/margin": 2.40625, "reward/rejected": -3.65625, "step": 281 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 616.0, "epoch": 0.08458308338332333, "grad_norm": 5.6543354988098145, "learning_rate": 9.825746164515119e-06, "logp/chosen": -302.0, "logp/rejected": -276.0, "loss": 0.367431640625, "reward/accuracy": 0.8125, "reward/chosen": -1.625, "reward/margin": 1.4921875, "reward/rejected": -3.125, "step": 282 }, { "approx. KL/chosen": 304.0, "approx. KL/rejected": 426.0, "epoch": 0.08488302339532093, "grad_norm": 7.513020038604736, "learning_rate": 9.824511036072342e-06, "logp/chosen": -398.0, "logp/rejected": -378.0, "loss": 0.82904052734375, "reward/accuracy": 0.625, "reward/chosen": -1.9921875, "reward/margin": 0.578125, "reward/rejected": -2.5625, "step": 283 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 520.0, "epoch": 0.08518296340731854, "grad_norm": 3.3817312717437744, "learning_rate": 9.823271623902163e-06, "logp/chosen": -414.0, "logp/rejected": -398.0, "loss": 0.4578857421875, "reward/accuracy": 0.875, "reward/chosen": -1.46875, "reward/margin": 1.4296875, "reward/rejected": -2.90625, "step": 284 }, { "approx. KL/chosen": 358.0, "approx. KL/rejected": 482.0, "epoch": 0.08548290341931614, "grad_norm": 5.146857261657715, "learning_rate": 9.822027929105069e-06, "logp/chosen": -408.0, "logp/rejected": -338.0, "loss": 0.621337890625, "reward/accuracy": 0.6875, "reward/chosen": -2.09375, "reward/margin": 0.68359375, "reward/rejected": -2.78125, "step": 285 }, { "approx. KL/chosen": 396.0, "approx. KL/rejected": 860.0, "epoch": 0.08578284343131373, "grad_norm": 7.22175931930542, "learning_rate": 9.820779952785341e-06, "logp/chosen": -316.0, "logp/rejected": -354.0, "loss": 0.518798828125, "reward/accuracy": 0.6875, "reward/chosen": -2.171875, "reward/margin": 1.359375, "reward/rejected": -3.53125, "step": 286 }, { "approx. KL/chosen": 241.0, "approx. KL/rejected": 632.0, "epoch": 0.08608278344331134, "grad_norm": 5.828389644622803, "learning_rate": 9.819527696051078e-06, "logp/chosen": -255.0, "logp/rejected": -264.0, "loss": 0.435638427734375, "reward/accuracy": 0.6875, "reward/chosen": -1.078125, "reward/margin": 1.8515625, "reward/rejected": -2.9375, "step": 287 }, { "approx. KL/chosen": 177.0, "approx. KL/rejected": 456.0, "epoch": 0.08638272345530894, "grad_norm": 7.30534553527832, "learning_rate": 9.818271160014164e-06, "logp/chosen": -288.0, "logp/rejected": -266.0, "loss": 0.6653900146484375, "reward/accuracy": 0.75, "reward/chosen": -1.328125, "reward/margin": 1.234375, "reward/rejected": -2.5625, "step": 288 }, { "approx. KL/chosen": 318.0, "approx. KL/rejected": 422.0, "epoch": 0.08668266346730653, "grad_norm": 7.830941200256348, "learning_rate": 9.81701034579029e-06, "logp/chosen": -458.0, "logp/rejected": -430.0, "loss": 0.908935546875, "reward/accuracy": 0.625, "reward/chosen": -1.8125, "reward/margin": 0.54296875, "reward/rejected": -2.359375, "step": 289 }, { "approx. KL/chosen": 137.0, "approx. KL/rejected": 520.0, "epoch": 0.08698260347930414, "grad_norm": 4.283565998077393, "learning_rate": 9.815745254498945e-06, "logp/chosen": -310.0, "logp/rejected": -328.0, "loss": 0.513336181640625, "reward/accuracy": 0.8125, "reward/chosen": -1.3203125, "reward/margin": 1.3828125, "reward/rejected": -2.703125, "step": 290 }, { "approx. KL/chosen": 206.0, "approx. KL/rejected": 788.0, "epoch": 0.08728254349130174, "grad_norm": 3.5568017959594727, "learning_rate": 9.814475887263415e-06, "logp/chosen": -280.0, "logp/rejected": -352.0, "loss": 0.29351806640625, "reward/accuracy": 0.875, "reward/chosen": -1.53125, "reward/margin": 1.9609375, "reward/rejected": -3.484375, "step": 291 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 388.0, "epoch": 0.08758248350329934, "grad_norm": 5.289478778839111, "learning_rate": 9.813202245210782e-06, "logp/chosen": -384.0, "logp/rejected": -336.0, "loss": 0.631439208984375, "reward/accuracy": 0.5625, "reward/chosen": -1.46875, "reward/margin": 0.82421875, "reward/rejected": -2.296875, "step": 292 }, { "approx. KL/chosen": 219.0, "approx. KL/rejected": 440.0, "epoch": 0.08788242351529695, "grad_norm": 5.681285381317139, "learning_rate": 9.811924329471927e-06, "logp/chosen": -251.0, "logp/rejected": -246.0, "loss": 0.7347412109375, "reward/accuracy": 0.4375, "reward/chosen": -1.75, "reward/margin": 0.7265625, "reward/rejected": -2.46875, "step": 293 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 724.0, "epoch": 0.08818236352729454, "grad_norm": 3.2257537841796875, "learning_rate": 9.810642141181521e-06, "logp/chosen": -424.0, "logp/rejected": -274.0, "loss": 0.26104736328125, "reward/accuracy": 0.875, "reward/chosen": -1.125, "reward/margin": 2.234375, "reward/rejected": -3.359375, "step": 294 }, { "approx. KL/chosen": 102.0, "approx. KL/rejected": 496.0, "epoch": 0.08848230353929214, "grad_norm": 4.856237411499023, "learning_rate": 9.809355681478028e-06, "logp/chosen": -380.0, "logp/rejected": -304.0, "loss": 0.454833984375, "reward/accuracy": 0.75, "reward/chosen": -1.1171875, "reward/margin": 1.5078125, "reward/rejected": -2.625, "step": 295 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 728.0, "epoch": 0.08878224355128975, "grad_norm": 3.593693494796753, "learning_rate": 9.808064951503712e-06, "logp/chosen": -334.0, "logp/rejected": -348.0, "loss": 0.325592041015625, "reward/accuracy": 0.8125, "reward/chosen": -1.109375, "reward/margin": 1.9921875, "reward/rejected": -3.09375, "step": 296 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 672.0, "epoch": 0.08908218356328734, "grad_norm": 5.802212238311768, "learning_rate": 9.806769952404624e-06, "logp/chosen": -414.0, "logp/rejected": -392.0, "loss": 0.37567138671875, "reward/accuracy": 0.8125, "reward/chosen": -1.5703125, "reward/margin": 1.6484375, "reward/rejected": -3.21875, "step": 297 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 516.0, "epoch": 0.08938212357528494, "grad_norm": 4.522717475891113, "learning_rate": 9.805470685330603e-06, "logp/chosen": -324.0, "logp/rejected": -324.0, "loss": 0.45916748046875, "reward/accuracy": 0.6875, "reward/chosen": -1.6796875, "reward/margin": 1.078125, "reward/rejected": -2.765625, "step": 298 }, { "approx. KL/chosen": 94.5, "approx. KL/rejected": 548.0, "epoch": 0.08968206358728255, "grad_norm": 3.248469114303589, "learning_rate": 9.80416715143528e-06, "logp/chosen": -330.0, "logp/rejected": -276.0, "loss": 0.288299560546875, "reward/accuracy": 0.875, "reward/chosen": -1.0625, "reward/margin": 1.8046875, "reward/rejected": -2.859375, "step": 299 }, { "approx. KL/chosen": 72.5, "approx. KL/rejected": 434.0, "epoch": 0.08998200359928014, "grad_norm": 3.7977964878082275, "learning_rate": 9.802859351876074e-06, "logp/chosen": -176.0, "logp/rejected": -193.0, "loss": 0.438507080078125, "reward/accuracy": 0.6875, "reward/chosen": -0.796875, "reward/margin": 1.359375, "reward/rejected": -2.15625, "step": 300 }, { "approx. KL/chosen": 99.0, "approx. KL/rejected": 392.0, "epoch": 0.09028194361127774, "grad_norm": 5.984740257263184, "learning_rate": 9.801547287814195e-06, "logp/chosen": -424.0, "logp/rejected": -376.0, "loss": 0.4031982421875, "reward/accuracy": 0.75, "reward/chosen": -0.76953125, "reward/margin": 1.6328125, "reward/rejected": -2.40625, "step": 301 }, { "approx. KL/chosen": 93.0, "approx. KL/rejected": 498.0, "epoch": 0.09058188362327535, "grad_norm": 2.7887816429138184, "learning_rate": 9.800230960414635e-06, "logp/chosen": -384.0, "logp/rejected": -344.0, "loss": 0.262115478515625, "reward/accuracy": 0.875, "reward/chosen": -0.84375, "reward/margin": 2.015625, "reward/rejected": -2.859375, "step": 302 }, { "approx. KL/chosen": 95.0, "approx. KL/rejected": 456.0, "epoch": 0.09088182363527295, "grad_norm": 3.366628885269165, "learning_rate": 9.798910370846174e-06, "logp/chosen": -338.0, "logp/rejected": -430.0, "loss": 0.26519775390625, "reward/accuracy": 0.875, "reward/chosen": -0.78515625, "reward/margin": 1.796875, "reward/rejected": -2.578125, "step": 303 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 370.0, "epoch": 0.09118176364727054, "grad_norm": 5.472250461578369, "learning_rate": 9.797585520281373e-06, "logp/chosen": -452.0, "logp/rejected": -440.0, "loss": 0.7349853515625, "reward/accuracy": 0.6875, "reward/chosen": -1.4453125, "reward/margin": 0.64453125, "reward/rejected": -2.09375, "step": 304 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 684.0, "epoch": 0.09148170365926815, "grad_norm": 4.5549702644348145, "learning_rate": 9.796256409896583e-06, "logp/chosen": -414.0, "logp/rejected": -446.0, "loss": 0.538970947265625, "reward/accuracy": 0.5625, "reward/chosen": -1.578125, "reward/margin": 1.34375, "reward/rejected": -2.921875, "step": 305 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 314.0, "epoch": 0.09178164367126575, "grad_norm": 5.028624057769775, "learning_rate": 9.79492304087193e-06, "logp/chosen": -334.0, "logp/rejected": -316.0, "loss": 0.6859130859375, "reward/accuracy": 0.6875, "reward/chosen": -1.3828125, "reward/margin": 0.5625, "reward/rejected": -1.9453125, "step": 306 }, { "approx. KL/chosen": 21.25, "approx. KL/rejected": 236.0, "epoch": 0.09208158368326334, "grad_norm": 3.5497679710388184, "learning_rate": 9.793585414391326e-06, "logp/chosen": -270.0, "logp/rejected": -286.0, "loss": 0.33624267578125, "reward/accuracy": 0.8125, "reward/chosen": -0.1962890625, "reward/margin": 1.578125, "reward/rejected": -1.7734375, "step": 307 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 808.0, "epoch": 0.09238152369526095, "grad_norm": 3.2136383056640625, "learning_rate": 9.79224353164246e-06, "logp/chosen": -280.0, "logp/rejected": -227.0, "loss": 0.1966552734375, "reward/accuracy": 1.0, "reward/chosen": -1.453125, "reward/margin": 2.203125, "reward/rejected": -3.65625, "step": 308 }, { "approx. KL/chosen": 105.0, "approx. KL/rejected": 184.0, "epoch": 0.09268146370725855, "grad_norm": 4.458477973937988, "learning_rate": 9.790897393816805e-06, "logp/chosen": -452.0, "logp/rejected": -328.0, "loss": 0.5279541015625, "reward/accuracy": 0.625, "reward/chosen": -0.79296875, "reward/margin": 0.73046875, "reward/rejected": -1.5234375, "step": 309 }, { "approx. KL/chosen": 62.5, "approx. KL/rejected": 206.0, "epoch": 0.09298140371925614, "grad_norm": 4.751542091369629, "learning_rate": 9.789547002109607e-06, "logp/chosen": -366.0, "logp/rejected": -316.0, "loss": 0.530029296875, "reward/accuracy": 0.6875, "reward/chosen": -0.7578125, "reward/margin": 0.85546875, "reward/rejected": -1.609375, "step": 310 }, { "approx. KL/chosen": 42.25, "approx. KL/rejected": 308.0, "epoch": 0.09328134373125375, "grad_norm": 2.963489294052124, "learning_rate": 9.78819235771989e-06, "logp/chosen": -268.0, "logp/rejected": -308.0, "loss": 0.3177490234375, "reward/accuracy": 0.9375, "reward/chosen": -0.65234375, "reward/margin": 1.359375, "reward/rejected": -2.015625, "step": 311 }, { "approx. KL/chosen": 126.0, "approx. KL/rejected": 504.0, "epoch": 0.09358128374325135, "grad_norm": 3.5397682189941406, "learning_rate": 9.786833461850459e-06, "logp/chosen": -246.0, "logp/rejected": -344.0, "loss": 0.35528564453125, "reward/accuracy": 0.75, "reward/chosen": -1.0625, "reward/margin": 1.7578125, "reward/rejected": -2.8125, "step": 312 }, { "approx. KL/chosen": 110.0, "approx. KL/rejected": 804.0, "epoch": 0.09388122375524895, "grad_norm": 3.7637386322021484, "learning_rate": 9.785470315707886e-06, "logp/chosen": -310.0, "logp/rejected": -388.0, "loss": 0.2878570556640625, "reward/accuracy": 0.875, "reward/chosen": -1.171875, "reward/margin": 2.234375, "reward/rejected": -3.40625, "step": 313 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 458.0, "epoch": 0.09418116376724656, "grad_norm": 3.956102132797241, "learning_rate": 9.784102920502522e-06, "logp/chosen": -418.0, "logp/rejected": -484.0, "loss": 0.378173828125, "reward/accuracy": 0.8125, "reward/chosen": -1.28125, "reward/margin": 1.4140625, "reward/rejected": -2.703125, "step": 314 }, { "approx. KL/chosen": 65.5, "approx. KL/rejected": 432.0, "epoch": 0.09448110377924415, "grad_norm": 4.270529747009277, "learning_rate": 9.782731277448493e-06, "logp/chosen": -278.0, "logp/rejected": -292.0, "loss": 0.4494476318359375, "reward/accuracy": 0.75, "reward/chosen": -0.65625, "reward/margin": 1.4296875, "reward/rejected": -2.078125, "step": 315 }, { "approx. KL/chosen": 149.0, "approx. KL/rejected": 344.0, "epoch": 0.09478104379124175, "grad_norm": 6.778989791870117, "learning_rate": 9.781355387763688e-06, "logp/chosen": -272.0, "logp/rejected": -310.0, "loss": 0.5350341796875, "reward/accuracy": 0.625, "reward/chosen": -1.3515625, "reward/margin": 0.80078125, "reward/rejected": -2.15625, "step": 316 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 732.0, "epoch": 0.09508098380323936, "grad_norm": 4.805803298950195, "learning_rate": 9.779975252669777e-06, "logp/chosen": -352.0, "logp/rejected": -366.0, "loss": 0.509765625, "reward/accuracy": 0.6875, "reward/chosen": -1.4765625, "reward/margin": 1.4453125, "reward/rejected": -2.921875, "step": 317 }, { "approx. KL/chosen": 328.0, "approx. KL/rejected": 724.0, "epoch": 0.09538092381523695, "grad_norm": 8.181947708129883, "learning_rate": 9.778590873392192e-06, "logp/chosen": -193.0, "logp/rejected": -292.0, "loss": 0.461181640625, "reward/accuracy": 0.6875, "reward/chosen": -1.734375, "reward/margin": 1.421875, "reward/rejected": -3.15625, "step": 318 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 656.0, "epoch": 0.09568086382723455, "grad_norm": 5.286313533782959, "learning_rate": 9.777202251160133e-06, "logp/chosen": -334.0, "logp/rejected": -350.0, "loss": 0.404937744140625, "reward/accuracy": 0.8125, "reward/chosen": -1.7109375, "reward/margin": 1.5859375, "reward/rejected": -3.296875, "step": 319 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 344.0, "epoch": 0.09598080383923216, "grad_norm": 5.808960914611816, "learning_rate": 9.775809387206577e-06, "logp/chosen": -322.0, "logp/rejected": -306.0, "loss": 0.61212158203125, "reward/accuracy": 0.625, "reward/chosen": -1.4296875, "reward/margin": 0.76953125, "reward/rejected": -2.203125, "step": 320 }, { "approx. KL/chosen": 218.0, "approx. KL/rejected": 368.0, "epoch": 0.09628074385122976, "grad_norm": 6.402250289916992, "learning_rate": 9.774412282768255e-06, "logp/chosen": -452.0, "logp/rejected": -306.0, "loss": 0.811614990234375, "reward/accuracy": 0.625, "reward/chosen": -1.421875, "reward/margin": 0.859375, "reward/rejected": -2.28125, "step": 321 }, { "approx. KL/chosen": 86.0, "approx. KL/rejected": 596.0, "epoch": 0.09658068386322735, "grad_norm": 4.983530044555664, "learning_rate": 9.773010939085673e-06, "logp/chosen": -324.0, "logp/rejected": -416.0, "loss": 0.4454345703125, "reward/accuracy": 0.75, "reward/chosen": -0.703125, "reward/margin": 1.890625, "reward/rejected": -2.59375, "step": 322 }, { "approx. KL/chosen": 49.0, "approx. KL/rejected": 442.0, "epoch": 0.09688062387522496, "grad_norm": 3.5596585273742676, "learning_rate": 9.771605357403094e-06, "logp/chosen": -428.0, "logp/rejected": -298.0, "loss": 0.35302734375, "reward/accuracy": 0.8125, "reward/chosen": -0.6171875, "reward/margin": 1.8828125, "reward/rejected": -2.5, "step": 323 }, { "approx. KL/chosen": 141.0, "approx. KL/rejected": 312.0, "epoch": 0.09718056388722256, "grad_norm": 4.158989429473877, "learning_rate": 9.770195538968546e-06, "logp/chosen": -328.0, "logp/rejected": -350.0, "loss": 0.51220703125, "reward/accuracy": 0.6875, "reward/chosen": -1.1171875, "reward/margin": 0.76953125, "reward/rejected": -1.890625, "step": 324 }, { "approx. KL/chosen": 82.5, "approx. KL/rejected": 402.0, "epoch": 0.09748050389922015, "grad_norm": 3.8510284423828125, "learning_rate": 9.768781485033822e-06, "logp/chosen": -628.0, "logp/rejected": -376.0, "loss": 0.4283638000488281, "reward/accuracy": 0.75, "reward/chosen": -0.31640625, "reward/margin": 1.7109375, "reward/rejected": -2.03125, "step": 325 }, { "approx. KL/chosen": 124.0, "approx. KL/rejected": 456.0, "epoch": 0.09778044391121776, "grad_norm": 5.589832305908203, "learning_rate": 9.767363196854475e-06, "logp/chosen": -330.0, "logp/rejected": -378.0, "loss": 0.513580322265625, "reward/accuracy": 0.6875, "reward/chosen": -0.87890625, "reward/margin": 1.4140625, "reward/rejected": -2.296875, "step": 326 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 672.0, "epoch": 0.09808038392321536, "grad_norm": 9.443709373474121, "learning_rate": 9.765940675689812e-06, "logp/chosen": -408.0, "logp/rejected": -362.0, "loss": 0.68597412109375, "reward/accuracy": 0.625, "reward/chosen": -1.546875, "reward/margin": 1.34375, "reward/rejected": -2.890625, "step": 327 }, { "approx. KL/chosen": 103.5, "approx. KL/rejected": 324.0, "epoch": 0.09838032393521295, "grad_norm": 6.010741710662842, "learning_rate": 9.764513922802906e-06, "logp/chosen": -260.0, "logp/rejected": -288.0, "loss": 0.531982421875, "reward/accuracy": 0.625, "reward/chosen": -0.83203125, "reward/margin": 1.1328125, "reward/rejected": -1.96875, "step": 328 }, { "approx. KL/chosen": 46.75, "approx. KL/rejected": 302.0, "epoch": 0.09868026394721056, "grad_norm": 2.843355894088745, "learning_rate": 9.76308293946058e-06, "logp/chosen": -308.0, "logp/rejected": -242.0, "loss": 0.26544189453125, "reward/accuracy": 0.875, "reward/chosen": -0.072265625, "reward/margin": 1.859375, "reward/rejected": -1.9296875, "step": 329 }, { "approx. KL/chosen": 102.5, "approx. KL/rejected": 245.0, "epoch": 0.09898020395920816, "grad_norm": 4.239112377166748, "learning_rate": 9.761647726933418e-06, "logp/chosen": -272.0, "logp/rejected": -310.0, "loss": 0.47802734375, "reward/accuracy": 0.75, "reward/chosen": -0.55078125, "reward/margin": 1.109375, "reward/rejected": -1.65625, "step": 330 }, { "approx. KL/chosen": 85.5, "approx. KL/rejected": 348.0, "epoch": 0.09928014397120576, "grad_norm": 5.178988933563232, "learning_rate": 9.760208286495762e-06, "logp/chosen": -268.0, "logp/rejected": -388.0, "loss": 0.6600341796875, "reward/accuracy": 0.625, "reward/chosen": -0.71875, "reward/margin": 0.90625, "reward/rejected": -1.625, "step": 331 }, { "approx. KL/chosen": 75.5, "approx. KL/rejected": 249.0, "epoch": 0.09958008398320337, "grad_norm": 4.974863529205322, "learning_rate": 9.758764619425703e-06, "logp/chosen": -278.0, "logp/rejected": -304.0, "loss": 0.5587158203125, "reward/accuracy": 0.5625, "reward/chosen": -0.6875, "reward/margin": 0.88671875, "reward/rejected": -1.578125, "step": 332 }, { "approx. KL/chosen": 61.5, "approx. KL/rejected": 221.0, "epoch": 0.09988002399520096, "grad_norm": 4.635591983795166, "learning_rate": 9.757316727005085e-06, "logp/chosen": -204.0, "logp/rejected": -368.0, "loss": 0.4547119140625, "reward/accuracy": 0.75, "reward/chosen": -0.4765625, "reward/margin": 1.3046875, "reward/rejected": -1.78125, "step": 333 }, { "approx. KL/chosen": 79.0, "approx. KL/rejected": 350.0, "epoch": 0.10017996400719856, "grad_norm": 6.117878437042236, "learning_rate": 9.755864610519502e-06, "logp/chosen": -336.0, "logp/rejected": -304.0, "loss": 0.53118896484375, "reward/accuracy": 0.75, "reward/chosen": -0.671875, "reward/margin": 1.3828125, "reward/rejected": -2.0625, "step": 334 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 524.0, "epoch": 0.10047990401919617, "grad_norm": 5.218805313110352, "learning_rate": 9.754408271258306e-06, "logp/chosen": -400.0, "logp/rejected": -458.0, "loss": 0.418121337890625, "reward/accuracy": 0.75, "reward/chosen": -0.9296875, "reward/margin": 1.5, "reward/rejected": -2.4375, "step": 335 }, { "approx. KL/chosen": 28.875, "approx. KL/rejected": 130.0, "epoch": 0.10077984403119376, "grad_norm": 4.064854621887207, "learning_rate": 9.752947710514594e-06, "logp/chosen": -372.0, "logp/rejected": -296.0, "loss": 0.4957275390625, "reward/accuracy": 0.8125, "reward/chosen": -0.1533203125, "reward/margin": 0.94140625, "reward/rejected": -1.09375, "step": 336 }, { "approx. KL/chosen": 39.0, "approx. KL/rejected": 232.0, "epoch": 0.10107978404319136, "grad_norm": 5.943378448486328, "learning_rate": 9.751482929585209e-06, "logp/chosen": -252.0, "logp/rejected": -251.0, "loss": 0.529937744140625, "reward/accuracy": 0.6875, "reward/chosen": -0.52734375, "reward/margin": 1.0703125, "reward/rejected": -1.6015625, "step": 337 }, { "approx. KL/chosen": 34.5, "approx. KL/rejected": 195.0, "epoch": 0.10137972405518897, "grad_norm": 3.513277053833008, "learning_rate": 9.750013929770743e-06, "logp/chosen": -358.0, "logp/rejected": -266.0, "loss": 0.4595947265625, "reward/accuracy": 0.75, "reward/chosen": -0.4609375, "reward/margin": 1.1015625, "reward/rejected": -1.5625, "step": 338 }, { "approx. KL/chosen": 83.0, "approx. KL/rejected": 406.0, "epoch": 0.10167966406718656, "grad_norm": 6.396583080291748, "learning_rate": 9.748540712375537e-06, "logp/chosen": -256.0, "logp/rejected": -356.0, "loss": 0.6002197265625, "reward/accuracy": 0.6875, "reward/chosen": -0.83984375, "reward/margin": 1.1796875, "reward/rejected": -2.015625, "step": 339 }, { "approx. KL/chosen": 61.75, "approx. KL/rejected": 310.0, "epoch": 0.10197960407918416, "grad_norm": 4.494993686676025, "learning_rate": 9.74706327870767e-06, "logp/chosen": -316.0, "logp/rejected": -322.0, "loss": 0.51190185546875, "reward/accuracy": 0.75, "reward/chosen": -0.43359375, "reward/margin": 1.0703125, "reward/rejected": -1.5078125, "step": 340 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 418.0, "epoch": 0.10227954409118177, "grad_norm": 6.72019100189209, "learning_rate": 9.745581630078975e-06, "logp/chosen": -336.0, "logp/rejected": -308.0, "loss": 0.42153167724609375, "reward/accuracy": 0.75, "reward/chosen": -0.81640625, "reward/margin": 1.640625, "reward/rejected": -2.46875, "step": 341 }, { "approx. KL/chosen": 67.0, "approx. KL/rejected": 150.0, "epoch": 0.10257948410317937, "grad_norm": 5.170680522918701, "learning_rate": 9.744095767805016e-06, "logp/chosen": -350.0, "logp/rejected": -322.0, "loss": 0.614013671875, "reward/accuracy": 0.625, "reward/chosen": -0.61328125, "reward/margin": 0.60546875, "reward/rejected": -1.21875, "step": 342 }, { "approx. KL/chosen": 52.75, "approx. KL/rejected": 174.0, "epoch": 0.10287942411517696, "grad_norm": 4.147933483123779, "learning_rate": 9.742605693205106e-06, "logp/chosen": -378.0, "logp/rejected": -324.0, "loss": 0.4888916015625, "reward/accuracy": 0.75, "reward/chosen": -0.5, "reward/margin": 1.0625, "reward/rejected": -1.5625, "step": 343 }, { "approx. KL/chosen": 33.75, "approx. KL/rejected": 392.0, "epoch": 0.10317936412717456, "grad_norm": 3.462930917739868, "learning_rate": 9.741111407602292e-06, "logp/chosen": -200.0, "logp/rejected": -332.0, "loss": 0.45291900634765625, "reward/accuracy": 0.75, "reward/chosen": -0.5234375, "reward/margin": 1.546875, "reward/rejected": -2.078125, "step": 344 }, { "approx. KL/chosen": 19.0, "approx. KL/rejected": 282.0, "epoch": 0.10347930413917217, "grad_norm": 2.445420742034912, "learning_rate": 9.73961291232337e-06, "logp/chosen": -184.0, "logp/rejected": -228.0, "loss": 0.22955322265625, "reward/accuracy": 1.0, "reward/chosen": -0.2158203125, "reward/margin": 1.796875, "reward/rejected": -2.015625, "step": 345 }, { "approx. KL/chosen": 69.0, "approx. KL/rejected": 193.0, "epoch": 0.10377924415116976, "grad_norm": 3.7764456272125244, "learning_rate": 9.738110208698865e-06, "logp/chosen": -354.0, "logp/rejected": -326.0, "loss": 0.35418701171875, "reward/accuracy": 0.8125, "reward/chosen": -0.026123046875, "reward/margin": 1.3828125, "reward/rejected": -1.40625, "step": 346 }, { "approx. KL/chosen": 52.25, "approx. KL/rejected": 274.0, "epoch": 0.10407918416316736, "grad_norm": 4.597503662109375, "learning_rate": 9.736603298063041e-06, "logp/chosen": -342.0, "logp/rejected": -422.0, "loss": 0.4997711181640625, "reward/accuracy": 0.75, "reward/chosen": -0.451171875, "reward/margin": 1.3671875, "reward/rejected": -1.8203125, "step": 347 }, { "approx. KL/chosen": 125.5, "approx. KL/rejected": 488.0, "epoch": 0.10437912417516497, "grad_norm": 4.116847038269043, "learning_rate": 9.735092181753898e-06, "logp/chosen": -330.0, "logp/rejected": -278.0, "loss": 0.33856201171875, "reward/accuracy": 0.8125, "reward/chosen": -0.625, "reward/margin": 1.8671875, "reward/rejected": -2.484375, "step": 348 }, { "approx. KL/chosen": 43.75, "approx. KL/rejected": 498.0, "epoch": 0.10467906418716257, "grad_norm": 3.406569480895996, "learning_rate": 9.73357686111317e-06, "logp/chosen": -308.0, "logp/rejected": -330.0, "loss": 0.40679931640625, "reward/accuracy": 0.8125, "reward/chosen": -0.359375, "reward/margin": 1.78125, "reward/rejected": -2.140625, "step": 349 }, { "approx. KL/chosen": 112.5, "approx. KL/rejected": 812.0, "epoch": 0.10497900419916016, "grad_norm": 5.004397869110107, "learning_rate": 9.732057337486322e-06, "logp/chosen": -370.0, "logp/rejected": -442.0, "loss": 0.47429656982421875, "reward/accuracy": 0.8125, "reward/chosen": -0.859375, "reward/margin": 2.484375, "reward/rejected": -3.34375, "step": 350 }, { "approx. KL/chosen": 125.0, "approx. KL/rejected": 636.0, "epoch": 0.10527894421115777, "grad_norm": 3.295525550842285, "learning_rate": 9.730533612222556e-06, "logp/chosen": -280.0, "logp/rejected": -398.0, "loss": 0.280548095703125, "reward/accuracy": 0.9375, "reward/chosen": -1.109375, "reward/margin": 1.9296875, "reward/rejected": -3.046875, "step": 351 }, { "approx. KL/chosen": 111.5, "approx. KL/rejected": 664.0, "epoch": 0.10557888422315537, "grad_norm": 2.889561414718628, "learning_rate": 9.729005686674799e-06, "logp/chosen": -249.0, "logp/rejected": -238.0, "loss": 0.26995086669921875, "reward/accuracy": 0.875, "reward/chosen": -0.63671875, "reward/margin": 2.5625, "reward/rejected": -3.203125, "step": 352 }, { "approx. KL/chosen": 53.5, "approx. KL/rejected": 462.0, "epoch": 0.10587882423515296, "grad_norm": 3.195333480834961, "learning_rate": 9.727473562199712e-06, "logp/chosen": -340.0, "logp/rejected": -330.0, "loss": 0.31396484375, "reward/accuracy": 0.875, "reward/chosen": -0.310546875, "reward/margin": 1.8359375, "reward/rejected": -2.140625, "step": 353 }, { "approx. KL/chosen": 68.0, "approx. KL/rejected": 548.0, "epoch": 0.10617876424715057, "grad_norm": 3.946267604827881, "learning_rate": 9.725937240157682e-06, "logp/chosen": -378.0, "logp/rejected": -416.0, "loss": 0.3785400390625, "reward/accuracy": 0.75, "reward/chosen": -0.5703125, "reward/margin": 2.015625, "reward/rejected": -2.59375, "step": 354 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 596.0, "epoch": 0.10647870425914817, "grad_norm": 6.033172607421875, "learning_rate": 9.724396721912824e-06, "logp/chosen": -233.0, "logp/rejected": -330.0, "loss": 0.54400634765625, "reward/accuracy": 0.625, "reward/chosen": -1.1015625, "reward/margin": 1.5078125, "reward/rejected": -2.609375, "step": 355 }, { "approx. KL/chosen": 79.0, "approx. KL/rejected": 298.0, "epoch": 0.10677864427114576, "grad_norm": 4.361678123474121, "learning_rate": 9.722852008832975e-06, "logp/chosen": -342.0, "logp/rejected": -266.0, "loss": 0.463287353515625, "reward/accuracy": 0.8125, "reward/chosen": -0.7109375, "reward/margin": 1.2734375, "reward/rejected": -1.984375, "step": 356 }, { "approx. KL/chosen": 100.0, "approx. KL/rejected": 820.0, "epoch": 0.10707858428314337, "grad_norm": 3.199888229370117, "learning_rate": 9.721303102289704e-06, "logp/chosen": -290.0, "logp/rejected": -370.0, "loss": 0.2502865791320801, "reward/accuracy": 0.8125, "reward/chosen": -0.83203125, "reward/margin": 2.75, "reward/rejected": -3.578125, "step": 357 }, { "approx. KL/chosen": 235.0, "approx. KL/rejected": 660.0, "epoch": 0.10737852429514097, "grad_norm": 4.041927814483643, "learning_rate": 9.719750003658294e-06, "logp/chosen": -370.0, "logp/rejected": -346.0, "loss": 0.4122314453125, "reward/accuracy": 0.875, "reward/chosen": -1.7265625, "reward/margin": 1.390625, "reward/rejected": -3.125, "step": 358 }, { "approx. KL/chosen": 138.0, "approx. KL/rejected": 510.0, "epoch": 0.10767846430713857, "grad_norm": 3.7028682231903076, "learning_rate": 9.718192714317763e-06, "logp/chosen": -450.0, "logp/rejected": -418.0, "loss": 0.3966064453125, "reward/accuracy": 0.75, "reward/chosen": -1.2109375, "reward/margin": 1.5390625, "reward/rejected": -2.75, "step": 359 }, { "approx. KL/chosen": 110.5, "approx. KL/rejected": 508.0, "epoch": 0.10797840431913618, "grad_norm": 4.426498889923096, "learning_rate": 9.716631235650835e-06, "logp/chosen": -394.0, "logp/rejected": -342.0, "loss": 0.5736083984375, "reward/accuracy": 0.6875, "reward/chosen": -0.98046875, "reward/margin": 1.3984375, "reward/rejected": -2.390625, "step": 360 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 704.0, "epoch": 0.10827834433113377, "grad_norm": 3.8229570388793945, "learning_rate": 9.715065569043965e-06, "logp/chosen": -372.0, "logp/rejected": -322.0, "loss": 0.3062744140625, "reward/accuracy": 0.875, "reward/chosen": -1.359375, "reward/margin": 2.0, "reward/rejected": -3.375, "step": 361 }, { "approx. KL/chosen": 168.0, "approx. KL/rejected": 416.0, "epoch": 0.10857828434313137, "grad_norm": 4.483809471130371, "learning_rate": 9.713495715887323e-06, "logp/chosen": -211.0, "logp/rejected": -312.0, "loss": 0.4447021484375, "reward/accuracy": 0.9375, "reward/chosen": -1.375, "reward/margin": 1.2109375, "reward/rejected": -2.59375, "step": 362 }, { "approx. KL/chosen": 350.0, "approx. KL/rejected": 948.0, "epoch": 0.10887822435512898, "grad_norm": 4.448910713195801, "learning_rate": 9.711921677574794e-06, "logp/chosen": -304.0, "logp/rejected": -348.0, "loss": 0.3625030517578125, "reward/accuracy": 0.875, "reward/chosen": -1.9375, "reward/margin": 1.875, "reward/rejected": -3.8125, "step": 363 }, { "approx. KL/chosen": 172.0, "approx. KL/rejected": 764.0, "epoch": 0.10917816436712657, "grad_norm": 3.491405487060547, "learning_rate": 9.710343455503982e-06, "logp/chosen": -247.0, "logp/rejected": -274.0, "loss": 0.3564300537109375, "reward/accuracy": 0.875, "reward/chosen": -1.6328125, "reward/margin": 1.8671875, "reward/rejected": -3.5, "step": 364 }, { "approx. KL/chosen": 392.0, "approx. KL/rejected": 744.0, "epoch": 0.10947810437912417, "grad_norm": 4.489985942840576, "learning_rate": 9.708761051076205e-06, "logp/chosen": -354.0, "logp/rejected": -356.0, "loss": 0.51171875, "reward/accuracy": 0.625, "reward/chosen": -2.265625, "reward/margin": 1.0546875, "reward/rejected": -3.3125, "step": 365 }, { "approx. KL/chosen": 612.0, "approx. KL/rejected": 868.0, "epoch": 0.10977804439112178, "grad_norm": 7.706391334533691, "learning_rate": 9.707174465696495e-06, "logp/chosen": -294.0, "logp/rejected": -328.0, "loss": 0.8543701171875, "reward/accuracy": 0.5625, "reward/chosen": -2.9375, "reward/margin": 0.796875, "reward/rejected": -3.734375, "step": 366 }, { "approx. KL/chosen": 374.0, "approx. KL/rejected": 976.0, "epoch": 0.11007798440311937, "grad_norm": 4.623664855957031, "learning_rate": 9.705583700773594e-06, "logp/chosen": -284.0, "logp/rejected": -362.0, "loss": 0.4500885009765625, "reward/accuracy": 0.75, "reward/chosen": -2.46875, "reward/margin": 1.4765625, "reward/rejected": -3.9375, "step": 367 }, { "approx. KL/chosen": 422.0, "approx. KL/rejected": 1032.0, "epoch": 0.11037792441511697, "grad_norm": 5.357285022735596, "learning_rate": 9.703988757719958e-06, "logp/chosen": -404.0, "logp/rejected": -412.0, "loss": 0.52532958984375, "reward/accuracy": 0.6875, "reward/chosen": -2.421875, "reward/margin": 1.5, "reward/rejected": -3.921875, "step": 368 }, { "approx. KL/chosen": 344.0, "approx. KL/rejected": 948.0, "epoch": 0.11067786442711458, "grad_norm": 3.9655098915100098, "learning_rate": 9.70238963795175e-06, "logp/chosen": -410.0, "logp/rejected": -360.0, "loss": 0.3157958984375, "reward/accuracy": 0.875, "reward/chosen": -2.421875, "reward/margin": 1.625, "reward/rejected": -4.0625, "step": 369 }, { "approx. KL/chosen": 346.0, "approx. KL/rejected": 1004.0, "epoch": 0.11097780443911218, "grad_norm": 5.098918437957764, "learning_rate": 9.700786342888844e-06, "logp/chosen": -298.0, "logp/rejected": -306.0, "loss": 0.47320556640625, "reward/accuracy": 0.75, "reward/chosen": -2.296875, "reward/margin": 1.7734375, "reward/rejected": -4.0625, "step": 370 }, { "approx. KL/chosen": 484.0, "approx. KL/rejected": 972.0, "epoch": 0.11127774445110977, "grad_norm": 4.083571434020996, "learning_rate": 9.69917887395482e-06, "logp/chosen": -416.0, "logp/rejected": -324.0, "loss": 0.4080810546875, "reward/accuracy": 0.8125, "reward/chosen": -2.9375, "reward/margin": 1.2421875, "reward/rejected": -4.1875, "step": 371 }, { "approx. KL/chosen": 576.0, "approx. KL/rejected": 1392.0, "epoch": 0.11157768446310738, "grad_norm": 3.7206525802612305, "learning_rate": 9.697567232576965e-06, "logp/chosen": -296.0, "logp/rejected": -372.0, "loss": 0.34783935546875, "reward/accuracy": 0.875, "reward/chosen": -3.15625, "reward/margin": 1.796875, "reward/rejected": -4.9375, "step": 372 }, { "approx. KL/chosen": 640.0, "approx. KL/rejected": 1144.0, "epoch": 0.11187762447510498, "grad_norm": 5.306097984313965, "learning_rate": 9.69595142018627e-06, "logp/chosen": -354.0, "logp/rejected": -364.0, "loss": 0.594696044921875, "reward/accuracy": 0.75, "reward/chosen": -3.28125, "reward/margin": 1.1796875, "reward/rejected": -4.46875, "step": 373 }, { "approx. KL/chosen": 420.0, "approx. KL/rejected": 1048.0, "epoch": 0.11217756448710257, "grad_norm": 4.192224979400635, "learning_rate": 9.69433143821743e-06, "logp/chosen": -400.0, "logp/rejected": -396.0, "loss": 0.32073974609375, "reward/accuracy": 0.8125, "reward/chosen": -2.53125, "reward/margin": 1.8828125, "reward/rejected": -4.40625, "step": 374 }, { "approx. KL/chosen": 676.0, "approx. KL/rejected": 860.0, "epoch": 0.11247750449910018, "grad_norm": 7.79981803894043, "learning_rate": 9.69270728810884e-06, "logp/chosen": -484.0, "logp/rejected": -474.0, "loss": 0.8824462890625, "reward/accuracy": 0.6875, "reward/chosen": -3.140625, "reward/margin": 0.6171875, "reward/rejected": -3.75, "step": 375 }, { "approx. KL/chosen": 536.0, "approx. KL/rejected": 976.0, "epoch": 0.11277744451109778, "grad_norm": 4.345934867858887, "learning_rate": 9.691078971302599e-06, "logp/chosen": -346.0, "logp/rejected": -402.0, "loss": 0.51031494140625, "reward/accuracy": 0.75, "reward/chosen": -2.8125, "reward/margin": 1.2109375, "reward/rejected": -4.03125, "step": 376 }, { "approx. KL/chosen": 612.0, "approx. KL/rejected": 948.0, "epoch": 0.11307738452309538, "grad_norm": 8.219395637512207, "learning_rate": 9.689446489244506e-06, "logp/chosen": -466.0, "logp/rejected": -410.0, "loss": 0.70477294921875, "reward/accuracy": 0.75, "reward/chosen": -2.9375, "reward/margin": 1.2265625, "reward/rejected": -4.15625, "step": 377 }, { "approx. KL/chosen": 498.0, "approx. KL/rejected": 860.0, "epoch": 0.11337732453509299, "grad_norm": 5.636020183563232, "learning_rate": 9.687809843384053e-06, "logp/chosen": -398.0, "logp/rejected": -396.0, "loss": 0.58935546875, "reward/accuracy": 0.625, "reward/chosen": -2.75, "reward/margin": 1.09375, "reward/rejected": -3.84375, "step": 378 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 1544.0, "epoch": 0.11367726454709058, "grad_norm": 3.8686599731445312, "learning_rate": 9.686169035174435e-06, "logp/chosen": -412.0, "logp/rejected": -552.0, "loss": 0.41552734375, "reward/accuracy": 0.75, "reward/chosen": -2.09375, "reward/margin": 2.0, "reward/rejected": -4.09375, "step": 379 }, { "approx. KL/chosen": 552.0, "approx. KL/rejected": 1264.0, "epoch": 0.11397720455908818, "grad_norm": 5.59911584854126, "learning_rate": 9.68452406607254e-06, "logp/chosen": -352.0, "logp/rejected": -318.0, "loss": 0.4237937927246094, "reward/accuracy": 0.75, "reward/chosen": -2.921875, "reward/margin": 1.84375, "reward/rejected": -4.78125, "step": 380 }, { "approx. KL/chosen": 470.0, "approx. KL/rejected": 1280.0, "epoch": 0.11427714457108579, "grad_norm": 5.327587604522705, "learning_rate": 9.68287493753895e-06, "logp/chosen": -274.0, "logp/rejected": -288.0, "loss": 0.30670166015625, "reward/accuracy": 0.875, "reward/chosen": -2.796875, "reward/margin": 2.0, "reward/rejected": -4.8125, "step": 381 }, { "approx. KL/chosen": 564.0, "approx. KL/rejected": 936.0, "epoch": 0.11457708458308338, "grad_norm": 5.247884273529053, "learning_rate": 9.681221651037947e-06, "logp/chosen": -356.0, "logp/rejected": -414.0, "loss": 0.469482421875, "reward/accuracy": 0.75, "reward/chosen": -2.953125, "reward/margin": 1.1171875, "reward/rejected": -4.0625, "step": 382 }, { "approx. KL/chosen": 508.0, "approx. KL/rejected": 992.0, "epoch": 0.11487702459508098, "grad_norm": 6.880554676055908, "learning_rate": 9.679564208037492e-06, "logp/chosen": -290.0, "logp/rejected": -348.0, "loss": 0.6532745361328125, "reward/accuracy": 0.6875, "reward/chosen": -2.859375, "reward/margin": 1.078125, "reward/rejected": -3.9375, "step": 383 }, { "approx. KL/chosen": 512.0, "approx. KL/rejected": 1328.0, "epoch": 0.11517696460707859, "grad_norm": 4.116121292114258, "learning_rate": 9.677902610009247e-06, "logp/chosen": -272.0, "logp/rejected": -364.0, "loss": 0.35675048828125, "reward/accuracy": 0.6875, "reward/chosen": -3.0, "reward/margin": 1.8046875, "reward/rejected": -4.8125, "step": 384 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 956.0, "epoch": 0.11547690461907618, "grad_norm": 2.6008832454681396, "learning_rate": 9.676236858428558e-06, "logp/chosen": -300.0, "logp/rejected": -370.0, "loss": 0.2370452880859375, "reward/accuracy": 0.9375, "reward/chosen": -2.09375, "reward/margin": 1.9609375, "reward/rejected": -4.0625, "step": 385 }, { "approx. KL/chosen": 1512.0, "approx. KL/rejected": 652.0, "epoch": 0.11577684463107378, "grad_norm": 22.496477127075195, "learning_rate": 9.674566954774465e-06, "logp/chosen": -456.0, "logp/rejected": -444.0, "loss": 1.6357421875, "reward/accuracy": 0.5, "reward/chosen": -3.765625, "reward/margin": -0.474609375, "reward/rejected": -3.296875, "step": 386 }, { "approx. KL/chosen": 454.0, "approx. KL/rejected": 960.0, "epoch": 0.11607678464307139, "grad_norm": 5.5245771408081055, "learning_rate": 9.672892900529688e-06, "logp/chosen": -328.0, "logp/rejected": -276.0, "loss": 0.41290283203125, "reward/accuracy": 0.6875, "reward/chosen": -2.765625, "reward/margin": 1.34375, "reward/rejected": -4.09375, "step": 387 }, { "approx. KL/chosen": 374.0, "approx. KL/rejected": 644.0, "epoch": 0.11637672465506899, "grad_norm": 4.390807628631592, "learning_rate": 9.671214697180635e-06, "logp/chosen": -288.0, "logp/rejected": -237.0, "loss": 0.552825927734375, "reward/accuracy": 0.8125, "reward/chosen": -2.1875, "reward/margin": 1.1328125, "reward/rejected": -3.3125, "step": 388 }, { "approx. KL/chosen": 440.0, "approx. KL/rejected": 668.0, "epoch": 0.11667666466706658, "grad_norm": 6.377657890319824, "learning_rate": 9.6695323462174e-06, "logp/chosen": -410.0, "logp/rejected": -368.0, "loss": 0.9884033203125, "reward/accuracy": 0.5625, "reward/chosen": -2.5625, "reward/margin": 0.51171875, "reward/rejected": -3.0625, "step": 389 }, { "approx. KL/chosen": 664.0, "approx. KL/rejected": 1232.0, "epoch": 0.11697660467906419, "grad_norm": 6.400085926055908, "learning_rate": 9.667845849133754e-06, "logp/chosen": -398.0, "logp/rejected": -302.0, "loss": 0.4915008544921875, "reward/accuracy": 0.875, "reward/chosen": -2.796875, "reward/margin": 1.7734375, "reward/rejected": -4.5625, "step": 390 }, { "approx. KL/chosen": 372.0, "approx. KL/rejected": 896.0, "epoch": 0.11727654469106179, "grad_norm": 5.418231010437012, "learning_rate": 9.666155207427157e-06, "logp/chosen": -462.0, "logp/rejected": -388.0, "loss": 0.474212646484375, "reward/accuracy": 0.8125, "reward/chosen": -2.34375, "reward/margin": 1.5546875, "reward/rejected": -3.90625, "step": 391 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 1064.0, "epoch": 0.11757648470305938, "grad_norm": 3.6946496963500977, "learning_rate": 9.664460422598743e-06, "logp/chosen": -378.0, "logp/rejected": -258.0, "loss": 0.293060302734375, "reward/accuracy": 0.9375, "reward/chosen": -1.8828125, "reward/margin": 2.3125, "reward/rejected": -4.1875, "step": 392 }, { "approx. KL/chosen": 238.0, "approx. KL/rejected": 860.0, "epoch": 0.117876424715057, "grad_norm": 2.3369078636169434, "learning_rate": 9.662761496153328e-06, "logp/chosen": -254.0, "logp/rejected": -328.0, "loss": 0.22265625, "reward/accuracy": 1.0, "reward/chosen": -1.921875, "reward/margin": 1.984375, "reward/rejected": -3.90625, "step": 393 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 660.0, "epoch": 0.11817636472705459, "grad_norm": 6.097661018371582, "learning_rate": 9.661058429599403e-06, "logp/chosen": -442.0, "logp/rejected": -424.0, "loss": 0.77081298828125, "reward/accuracy": 0.6875, "reward/chosen": -1.7421875, "reward/margin": 1.234375, "reward/rejected": -2.984375, "step": 394 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 840.0, "epoch": 0.11847630473905219, "grad_norm": 4.233853816986084, "learning_rate": 9.659351224449139e-06, "logp/chosen": -340.0, "logp/rejected": -388.0, "loss": 0.53338623046875, "reward/accuracy": 0.75, "reward/chosen": -1.375, "reward/margin": 1.8828125, "reward/rejected": -3.265625, "step": 395 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 788.0, "epoch": 0.1187762447510498, "grad_norm": 3.030334711074829, "learning_rate": 9.657639882218376e-06, "logp/chosen": -402.0, "logp/rejected": -306.0, "loss": 0.3382568359375, "reward/accuracy": 0.75, "reward/chosen": -1.328125, "reward/margin": 2.140625, "reward/rejected": -3.46875, "step": 396 }, { "approx. KL/chosen": 410.0, "approx. KL/rejected": 1048.0, "epoch": 0.11907618476304739, "grad_norm": 5.493759632110596, "learning_rate": 9.655924404426634e-06, "logp/chosen": -340.0, "logp/rejected": -336.0, "loss": 0.43058013916015625, "reward/accuracy": 0.8125, "reward/chosen": -1.7578125, "reward/margin": 2.390625, "reward/rejected": -4.15625, "step": 397 }, { "approx. KL/chosen": 354.0, "approx. KL/rejected": 772.0, "epoch": 0.11937612477504499, "grad_norm": 5.27436637878418, "learning_rate": 9.654204792597097e-06, "logp/chosen": -356.0, "logp/rejected": -462.0, "loss": 0.4259033203125, "reward/accuracy": 0.875, "reward/chosen": -2.0, "reward/margin": 1.484375, "reward/rejected": -3.484375, "step": 398 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 612.0, "epoch": 0.1196760647870426, "grad_norm": 4.010319232940674, "learning_rate": 9.652481048256625e-06, "logp/chosen": -358.0, "logp/rejected": -306.0, "loss": 0.52056884765625, "reward/accuracy": 0.8125, "reward/chosen": -2.0, "reward/margin": 1.1796875, "reward/rejected": -3.1875, "step": 399 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 748.0, "epoch": 0.11997600479904019, "grad_norm": 5.693622589111328, "learning_rate": 9.65075317293575e-06, "logp/chosen": -310.0, "logp/rejected": -340.0, "loss": 0.5716552734375, "reward/accuracy": 0.75, "reward/chosen": -1.9453125, "reward/margin": 1.46875, "reward/rejected": -3.40625, "step": 400 }, { "approx. KL/chosen": 79.0, "approx. KL/rejected": 504.0, "epoch": 0.12027594481103779, "grad_norm": 3.2331619262695312, "learning_rate": 9.649021168168664e-06, "logp/chosen": -384.0, "logp/rejected": -338.0, "loss": 0.317474365234375, "reward/accuracy": 0.8125, "reward/chosen": -0.953125, "reward/margin": 1.6875, "reward/rejected": -2.640625, "step": 401 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 440.0, "epoch": 0.1205758848230354, "grad_norm": 6.541049957275391, "learning_rate": 9.647285035493232e-06, "logp/chosen": -362.0, "logp/rejected": -338.0, "loss": 0.4962158203125, "reward/accuracy": 0.75, "reward/chosen": -1.671875, "reward/margin": 1.03125, "reward/rejected": -2.703125, "step": 402 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 320.0, "epoch": 0.120875824835033, "grad_norm": 5.3738813400268555, "learning_rate": 9.645544776450982e-06, "logp/chosen": -342.0, "logp/rejected": -320.0, "loss": 0.613525390625, "reward/accuracy": 0.75, "reward/chosen": -1.28125, "reward/margin": 0.828125, "reward/rejected": -2.109375, "step": 403 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 624.0, "epoch": 0.12117576484703059, "grad_norm": 3.2515289783477783, "learning_rate": 9.643800392587105e-06, "logp/chosen": -217.0, "logp/rejected": -250.0, "loss": 0.3040771484375, "reward/accuracy": 0.8125, "reward/chosen": -1.171875, "reward/margin": 1.8984375, "reward/rejected": -3.078125, "step": 404 }, { "approx. KL/chosen": 121.0, "approx. KL/rejected": 304.0, "epoch": 0.1214757048590282, "grad_norm": 4.0033979415893555, "learning_rate": 9.642051885450458e-06, "logp/chosen": -248.0, "logp/rejected": -221.0, "loss": 0.563720703125, "reward/accuracy": 0.6875, "reward/chosen": -1.125, "reward/margin": 0.9296875, "reward/rejected": -2.0625, "step": 405 }, { "approx. KL/chosen": 125.5, "approx. KL/rejected": 310.0, "epoch": 0.1217756448710258, "grad_norm": 5.649282455444336, "learning_rate": 9.640299256593553e-06, "logp/chosen": -260.0, "logp/rejected": -308.0, "loss": 0.703277587890625, "reward/accuracy": 0.5625, "reward/chosen": -1.0859375, "reward/margin": 0.88671875, "reward/rejected": -1.96875, "step": 406 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 510.0, "epoch": 0.12207558488302339, "grad_norm": 6.136487007141113, "learning_rate": 9.638542507572567e-06, "logp/chosen": -253.0, "logp/rejected": -332.0, "loss": 0.6887359619140625, "reward/accuracy": 0.625, "reward/chosen": -1.046875, "reward/margin": 1.53125, "reward/rejected": -2.578125, "step": 407 }, { "approx. KL/chosen": 249.0, "approx. KL/rejected": 412.0, "epoch": 0.122375524895021, "grad_norm": 8.019283294677734, "learning_rate": 9.636781639947332e-06, "logp/chosen": -242.0, "logp/rejected": -229.0, "loss": 0.5706787109375, "reward/accuracy": 0.75, "reward/chosen": -1.2578125, "reward/margin": 0.94921875, "reward/rejected": -2.203125, "step": 408 }, { "approx. KL/chosen": 51.0, "approx. KL/rejected": 620.0, "epoch": 0.1226754649070186, "grad_norm": 2.776978015899658, "learning_rate": 9.63501665528134e-06, "logp/chosen": -304.0, "logp/rejected": -328.0, "loss": 0.3403739929199219, "reward/accuracy": 0.6875, "reward/chosen": -0.48046875, "reward/margin": 2.25, "reward/rejected": -2.71875, "step": 409 }, { "approx. KL/chosen": 56.0, "approx. KL/rejected": 272.0, "epoch": 0.12297540491901619, "grad_norm": 3.4628705978393555, "learning_rate": 9.633247555141736e-06, "logp/chosen": -282.0, "logp/rejected": -231.0, "loss": 0.400634765625, "reward/accuracy": 0.875, "reward/chosen": -0.75390625, "reward/margin": 1.1953125, "reward/rejected": -1.953125, "step": 410 }, { "approx. KL/chosen": 96.5, "approx. KL/rejected": 544.0, "epoch": 0.1232753449310138, "grad_norm": 4.4205098152160645, "learning_rate": 9.631474341099322e-06, "logp/chosen": -420.0, "logp/rejected": -480.0, "loss": 0.3890533447265625, "reward/accuracy": 0.75, "reward/chosen": -0.78515625, "reward/margin": 1.8828125, "reward/rejected": -2.671875, "step": 411 }, { "approx. KL/chosen": 99.0, "approx. KL/rejected": 540.0, "epoch": 0.1235752849430114, "grad_norm": 4.032832622528076, "learning_rate": 9.629697014728548e-06, "logp/chosen": -436.0, "logp/rejected": -390.0, "loss": 0.317779541015625, "reward/accuracy": 0.875, "reward/chosen": -0.56640625, "reward/margin": 2.03125, "reward/rejected": -2.59375, "step": 412 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 278.0, "epoch": 0.123875224955009, "grad_norm": 5.869244575500488, "learning_rate": 9.62791557760752e-06, "logp/chosen": -464.0, "logp/rejected": -450.0, "loss": 0.5887451171875, "reward/accuracy": 0.75, "reward/chosen": -0.87109375, "reward/margin": 0.94140625, "reward/rejected": -1.8125, "step": 413 }, { "approx. KL/chosen": 109.5, "approx. KL/rejected": 704.0, "epoch": 0.1241751649670066, "grad_norm": 2.5600266456604004, "learning_rate": 9.626130031317994e-06, "logp/chosen": -294.0, "logp/rejected": -310.0, "loss": 0.227752685546875, "reward/accuracy": 0.9375, "reward/chosen": -1.078125, "reward/margin": 2.265625, "reward/rejected": -3.34375, "step": 414 }, { "approx. KL/chosen": 115.0, "approx. KL/rejected": 462.0, "epoch": 0.1244751049790042, "grad_norm": 3.838449716567993, "learning_rate": 9.624340377445369e-06, "logp/chosen": -548.0, "logp/rejected": -536.0, "loss": 0.36602783203125, "reward/accuracy": 0.8125, "reward/chosen": -0.78515625, "reward/margin": 1.7109375, "reward/rejected": -2.5, "step": 415 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 612.0, "epoch": 0.1247750449910018, "grad_norm": 5.419022560119629, "learning_rate": 9.622546617578699e-06, "logp/chosen": -262.0, "logp/rejected": -237.0, "loss": 0.58966064453125, "reward/accuracy": 0.6875, "reward/chosen": -1.5625, "reward/margin": 1.3046875, "reward/rejected": -2.859375, "step": 416 }, { "approx. KL/chosen": 86.5, "approx. KL/rejected": 398.0, "epoch": 0.1250749850029994, "grad_norm": 4.593692302703857, "learning_rate": 9.620748753310677e-06, "logp/chosen": -252.0, "logp/rejected": -288.0, "loss": 0.4112548828125, "reward/accuracy": 0.875, "reward/chosen": -0.75390625, "reward/margin": 1.4375, "reward/rejected": -2.1875, "step": 417 }, { "approx. KL/chosen": 123.0, "approx. KL/rejected": 280.0, "epoch": 0.125374925014997, "grad_norm": 5.620813846588135, "learning_rate": 9.618946786237643e-06, "logp/chosen": -272.0, "logp/rejected": -280.0, "loss": 0.81884765625, "reward/accuracy": 0.625, "reward/chosen": -1.046875, "reward/margin": 0.65234375, "reward/rejected": -1.703125, "step": 418 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 728.0, "epoch": 0.1256748650269946, "grad_norm": 7.041873455047607, "learning_rate": 9.617140717959582e-06, "logp/chosen": -412.0, "logp/rejected": -290.0, "loss": 0.33331298828125, "reward/accuracy": 0.8125, "reward/chosen": -1.6171875, "reward/margin": 1.9375, "reward/rejected": -3.546875, "step": 419 }, { "approx. KL/chosen": 66.0, "approx. KL/rejected": 860.0, "epoch": 0.1259748050389922, "grad_norm": 3.292290687561035, "learning_rate": 9.615330550080118e-06, "logp/chosen": -252.0, "logp/rejected": -464.0, "loss": 0.3002586364746094, "reward/accuracy": 0.6875, "reward/chosen": -0.390625, "reward/margin": 2.859375, "reward/rejected": -3.234375, "step": 420 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 342.0, "epoch": 0.1262747450509898, "grad_norm": 4.062203407287598, "learning_rate": 9.613516284206514e-06, "logp/chosen": -384.0, "logp/rejected": -416.0, "loss": 0.577301025390625, "reward/accuracy": 0.6875, "reward/chosen": -0.9140625, "reward/margin": 1.3125, "reward/rejected": -2.234375, "step": 421 }, { "approx. KL/chosen": 96.0, "approx. KL/rejected": 306.0, "epoch": 0.1265746850629874, "grad_norm": 3.59722900390625, "learning_rate": 9.611697921949674e-06, "logp/chosen": -388.0, "logp/rejected": -350.0, "loss": 0.26025390625, "reward/accuracy": 0.875, "reward/chosen": -0.70703125, "reward/margin": 1.5390625, "reward/rejected": -2.25, "step": 422 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 474.0, "epoch": 0.126874625074985, "grad_norm": 4.32918119430542, "learning_rate": 9.60987546492414e-06, "logp/chosen": -330.0, "logp/rejected": -244.0, "loss": 0.509033203125, "reward/accuracy": 0.75, "reward/chosen": -0.9453125, "reward/margin": 1.3984375, "reward/rejected": -2.34375, "step": 423 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 564.0, "epoch": 0.1271745650869826, "grad_norm": 4.530458450317383, "learning_rate": 9.608048914748085e-06, "logp/chosen": -426.0, "logp/rejected": -278.0, "loss": 0.28375244140625, "reward/accuracy": 0.875, "reward/chosen": -0.99609375, "reward/margin": 1.953125, "reward/rejected": -2.953125, "step": 424 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 556.0, "epoch": 0.1274745050989802, "grad_norm": 4.437320232391357, "learning_rate": 9.606218273043324e-06, "logp/chosen": -298.0, "logp/rejected": -420.0, "loss": 0.57843017578125, "reward/accuracy": 0.6875, "reward/chosen": -1.390625, "reward/margin": 1.203125, "reward/rejected": -2.59375, "step": 425 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 564.0, "epoch": 0.1277744451109778, "grad_norm": 4.37675142288208, "learning_rate": 9.604383541435296e-06, "logp/chosen": -308.0, "logp/rejected": -324.0, "loss": 0.6024169921875, "reward/accuracy": 0.75, "reward/chosen": -1.5078125, "reward/margin": 1.1484375, "reward/rejected": -2.65625, "step": 426 }, { "approx. KL/chosen": 127.5, "approx. KL/rejected": 796.0, "epoch": 0.1280743851229754, "grad_norm": 3.885266065597534, "learning_rate": 9.60254472155308e-06, "logp/chosen": -404.0, "logp/rejected": -340.0, "loss": 0.46776580810546875, "reward/accuracy": 0.8125, "reward/chosen": -1.2421875, "reward/margin": 2.21875, "reward/rejected": -3.46875, "step": 427 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 584.0, "epoch": 0.12837432513497302, "grad_norm": 3.727501630783081, "learning_rate": 9.600701815029377e-06, "logp/chosen": -249.0, "logp/rejected": -262.0, "loss": 0.42901611328125, "reward/accuracy": 0.75, "reward/chosen": -1.203125, "reward/margin": 1.796875, "reward/rejected": -3.0, "step": 428 }, { "approx. KL/chosen": 96.0, "approx. KL/rejected": 544.0, "epoch": 0.1286742651469706, "grad_norm": 3.720954656600952, "learning_rate": 9.598854823500523e-06, "logp/chosen": -350.0, "logp/rejected": -296.0, "loss": 0.4139404296875, "reward/accuracy": 0.75, "reward/chosen": -1.0078125, "reward/margin": 1.6171875, "reward/rejected": -2.625, "step": 429 }, { "approx. KL/chosen": 115.0, "approx. KL/rejected": 352.0, "epoch": 0.1289742051589682, "grad_norm": 3.930058479309082, "learning_rate": 9.59700374860648e-06, "logp/chosen": -254.0, "logp/rejected": -390.0, "loss": 0.47265625, "reward/accuracy": 0.6875, "reward/chosen": -1.0625, "reward/margin": 0.87890625, "reward/rejected": -1.9453125, "step": 430 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 520.0, "epoch": 0.1292741451709658, "grad_norm": 6.251007080078125, "learning_rate": 9.59514859199083e-06, "logp/chosen": -282.0, "logp/rejected": -252.0, "loss": 0.5042724609375, "reward/accuracy": 0.6875, "reward/chosen": -1.484375, "reward/margin": 1.4375, "reward/rejected": -2.921875, "step": 431 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 552.0, "epoch": 0.1295740851829634, "grad_norm": 4.495062828063965, "learning_rate": 9.593289355300787e-06, "logp/chosen": -304.0, "logp/rejected": -294.0, "loss": 0.3262786865234375, "reward/accuracy": 0.8125, "reward/chosen": -1.0859375, "reward/margin": 1.96875, "reward/rejected": -3.0625, "step": 432 }, { "approx. KL/chosen": 193.0, "approx. KL/rejected": 716.0, "epoch": 0.129874025194961, "grad_norm": 4.074666976928711, "learning_rate": 9.591426040187182e-06, "logp/chosen": -294.0, "logp/rejected": -280.0, "loss": 0.349365234375, "reward/accuracy": 0.75, "reward/chosen": -1.625, "reward/margin": 1.7890625, "reward/rejected": -3.40625, "step": 433 }, { "approx. KL/chosen": 247.0, "approx. KL/rejected": 948.0, "epoch": 0.13017396520695862, "grad_norm": 3.2670891284942627, "learning_rate": 9.589558648304471e-06, "logp/chosen": -416.0, "logp/rejected": -620.0, "loss": 0.2246856689453125, "reward/accuracy": 0.875, "reward/chosen": -1.6328125, "reward/margin": 2.28125, "reward/rejected": -3.90625, "step": 434 }, { "approx. KL/chosen": 75.5, "approx. KL/rejected": 258.0, "epoch": 0.13047390521895622, "grad_norm": 3.5128262042999268, "learning_rate": 9.58768718131073e-06, "logp/chosen": -348.0, "logp/rejected": -350.0, "loss": 0.3995361328125, "reward/accuracy": 0.8125, "reward/chosen": -0.9375, "reward/margin": 1.0625, "reward/rejected": -2.0, "step": 435 }, { "approx. KL/chosen": 97.0, "approx. KL/rejected": 374.0, "epoch": 0.1307738452309538, "grad_norm": 3.9468131065368652, "learning_rate": 9.585811640867648e-06, "logp/chosen": -300.0, "logp/rejected": -264.0, "loss": 0.5372695922851562, "reward/accuracy": 0.6875, "reward/chosen": -0.96875, "reward/margin": 0.953125, "reward/rejected": -1.921875, "step": 436 }, { "approx. KL/chosen": 80.5, "approx. KL/rejected": 456.0, "epoch": 0.1310737852429514, "grad_norm": 4.033977031707764, "learning_rate": 9.583932028640536e-06, "logp/chosen": -464.0, "logp/rejected": -386.0, "loss": 0.4579315185546875, "reward/accuracy": 0.75, "reward/chosen": -1.046875, "reward/margin": 1.546875, "reward/rejected": -2.59375, "step": 437 }, { "approx. KL/chosen": 87.0, "approx. KL/rejected": 340.0, "epoch": 0.131373725254949, "grad_norm": 2.7745442390441895, "learning_rate": 9.58204834629832e-06, "logp/chosen": -322.0, "logp/rejected": -276.0, "loss": 0.3245849609375, "reward/accuracy": 0.9375, "reward/chosen": -0.7109375, "reward/margin": 1.46875, "reward/rejected": -2.1875, "step": 438 }, { "approx. KL/chosen": 84.0, "approx. KL/rejected": 312.0, "epoch": 0.1316736652669466, "grad_norm": 5.633321762084961, "learning_rate": 9.580160595513536e-06, "logp/chosen": -348.0, "logp/rejected": -360.0, "loss": 0.4149169921875, "reward/accuracy": 0.8125, "reward/chosen": -0.7734375, "reward/margin": 1.21875, "reward/rejected": -1.9921875, "step": 439 }, { "approx. KL/chosen": 113.5, "approx. KL/rejected": 438.0, "epoch": 0.13197360527894422, "grad_norm": 4.167738914489746, "learning_rate": 9.578268777962339e-06, "logp/chosen": -296.0, "logp/rejected": -278.0, "loss": 0.32183837890625, "reward/accuracy": 0.8125, "reward/chosen": -1.015625, "reward/margin": 1.5703125, "reward/rejected": -2.59375, "step": 440 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 624.0, "epoch": 0.13227354529094182, "grad_norm": 6.280569076538086, "learning_rate": 9.576372895324486e-06, "logp/chosen": -454.0, "logp/rejected": -428.0, "loss": 0.3580322265625, "reward/accuracy": 0.75, "reward/chosen": -1.296875, "reward/margin": 1.8828125, "reward/rejected": -3.1875, "step": 441 }, { "approx. KL/chosen": 83.5, "approx. KL/rejected": 324.0, "epoch": 0.13257348530293941, "grad_norm": 3.3439621925354004, "learning_rate": 9.574472949283354e-06, "logp/chosen": -338.0, "logp/rejected": -358.0, "loss": 0.3157958984375, "reward/accuracy": 0.9375, "reward/chosen": -0.73828125, "reward/margin": 1.515625, "reward/rejected": -2.25, "step": 442 }, { "approx. KL/chosen": 71.0, "approx. KL/rejected": 496.0, "epoch": 0.132873425314937, "grad_norm": 3.937971830368042, "learning_rate": 9.57256894152592e-06, "logp/chosen": -340.0, "logp/rejected": -366.0, "loss": 0.3741798400878906, "reward/accuracy": 0.8125, "reward/chosen": -0.67578125, "reward/margin": 1.8828125, "reward/rejected": -2.546875, "step": 443 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 392.0, "epoch": 0.1331733653269346, "grad_norm": 4.85635232925415, "learning_rate": 9.570660873742768e-06, "logp/chosen": -428.0, "logp/rejected": -364.0, "loss": 0.5775146484375, "reward/accuracy": 0.5625, "reward/chosen": -0.98828125, "reward/margin": 0.92578125, "reward/rejected": -1.9140625, "step": 444 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 736.0, "epoch": 0.1334733053389322, "grad_norm": 2.729360580444336, "learning_rate": 9.568748747628091e-06, "logp/chosen": -306.0, "logp/rejected": -249.0, "loss": 0.231353759765625, "reward/accuracy": 0.9375, "reward/chosen": -1.1796875, "reward/margin": 1.96875, "reward/rejected": -3.140625, "step": 445 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 386.0, "epoch": 0.13377324535092983, "grad_norm": 4.0201735496521, "learning_rate": 9.566832564879685e-06, "logp/chosen": -296.0, "logp/rejected": -252.0, "loss": 0.53662109375, "reward/accuracy": 0.6875, "reward/chosen": -1.359375, "reward/margin": 0.8359375, "reward/rejected": -2.203125, "step": 446 }, { "approx. KL/chosen": 149.0, "approx. KL/rejected": 684.0, "epoch": 0.13407318536292742, "grad_norm": 5.073997497558594, "learning_rate": 9.564912327198941e-06, "logp/chosen": -332.0, "logp/rejected": -384.0, "loss": 0.505615234375, "reward/accuracy": 0.625, "reward/chosen": -1.3046875, "reward/margin": 1.515625, "reward/rejected": -2.8125, "step": 447 }, { "approx. KL/chosen": 120.5, "approx. KL/rejected": 644.0, "epoch": 0.13437312537492502, "grad_norm": 2.9543368816375732, "learning_rate": 9.56298803629086e-06, "logp/chosen": -288.0, "logp/rejected": -233.0, "loss": 0.270355224609375, "reward/accuracy": 0.9375, "reward/chosen": -1.046875, "reward/margin": 2.125, "reward/rejected": -3.171875, "step": 448 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 588.0, "epoch": 0.1346730653869226, "grad_norm": 4.767649173736572, "learning_rate": 9.561059693864035e-06, "logp/chosen": -352.0, "logp/rejected": -372.0, "loss": 0.472900390625, "reward/accuracy": 0.8125, "reward/chosen": -1.5546875, "reward/margin": 1.4921875, "reward/rejected": -3.046875, "step": 449 }, { "approx. KL/chosen": 104.0, "approx. KL/rejected": 720.0, "epoch": 0.1349730053989202, "grad_norm": 4.4138665199279785, "learning_rate": 9.559127301630662e-06, "logp/chosen": -318.0, "logp/rejected": -292.0, "loss": 0.252960205078125, "reward/accuracy": 0.9375, "reward/chosen": -1.078125, "reward/margin": 2.359375, "reward/rejected": -3.4375, "step": 450 }, { "approx. KL/chosen": 154.0, "approx. KL/rejected": 446.0, "epoch": 0.1352729454109178, "grad_norm": 5.956427097320557, "learning_rate": 9.557190861306525e-06, "logp/chosen": -360.0, "logp/rejected": -354.0, "loss": 0.5506591796875, "reward/accuracy": 0.625, "reward/chosen": -1.1953125, "reward/margin": 1.2578125, "reward/rejected": -2.453125, "step": 451 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 402.0, "epoch": 0.13557288542291543, "grad_norm": 4.967025279998779, "learning_rate": 9.555250374611012e-06, "logp/chosen": -310.0, "logp/rejected": -386.0, "loss": 0.6087646484375, "reward/accuracy": 0.6875, "reward/chosen": -1.6953125, "reward/margin": 0.62890625, "reward/rejected": -2.328125, "step": 452 }, { "approx. KL/chosen": 150.0, "approx. KL/rejected": 668.0, "epoch": 0.13587282543491302, "grad_norm": 3.4001219272613525, "learning_rate": 9.553305843267094e-06, "logp/chosen": -312.0, "logp/rejected": -262.0, "loss": 0.39801025390625, "reward/accuracy": 0.8125, "reward/chosen": -1.125, "reward/margin": 1.7734375, "reward/rejected": -2.890625, "step": 453 }, { "approx. KL/chosen": 322.0, "approx. KL/rejected": 724.0, "epoch": 0.13617276544691062, "grad_norm": 6.36029577255249, "learning_rate": 9.551357269001342e-06, "logp/chosen": -298.0, "logp/rejected": -396.0, "loss": 0.4957275390625, "reward/accuracy": 0.75, "reward/chosen": -1.6640625, "reward/margin": 1.578125, "reward/rejected": -3.234375, "step": 454 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 1600.0, "epoch": 0.13647270545890822, "grad_norm": 3.3342583179473877, "learning_rate": 9.549404653543911e-06, "logp/chosen": -388.0, "logp/rejected": -478.0, "loss": 0.2506256103515625, "reward/accuracy": 0.8125, "reward/chosen": -1.359375, "reward/margin": 3.390625, "reward/rejected": -4.75, "step": 455 }, { "approx. KL/chosen": 110.5, "approx. KL/rejected": 852.0, "epoch": 0.1367726454709058, "grad_norm": 2.507901668548584, "learning_rate": 9.547447998628548e-06, "logp/chosen": -286.0, "logp/rejected": -296.0, "loss": 0.222991943359375, "reward/accuracy": 0.9375, "reward/chosen": -1.1640625, "reward/margin": 2.390625, "reward/rejected": -3.546875, "step": 456 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 584.0, "epoch": 0.1370725854829034, "grad_norm": 3.3755922317504883, "learning_rate": 9.545487305992582e-06, "logp/chosen": -352.0, "logp/rejected": -288.0, "loss": 0.3307342529296875, "reward/accuracy": 0.8125, "reward/chosen": -1.0859375, "reward/margin": 1.9765625, "reward/rejected": -3.0625, "step": 457 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 448.0, "epoch": 0.13737252549490103, "grad_norm": 5.067196846008301, "learning_rate": 9.543522577376932e-06, "logp/chosen": -282.0, "logp/rejected": -304.0, "loss": 0.62060546875, "reward/accuracy": 0.5625, "reward/chosen": -1.4765625, "reward/margin": 0.890625, "reward/rejected": -2.375, "step": 458 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 458.0, "epoch": 0.13767246550689863, "grad_norm": 4.94052267074585, "learning_rate": 9.541553814526096e-06, "logp/chosen": -270.0, "logp/rejected": -206.0, "loss": 0.513427734375, "reward/accuracy": 0.6875, "reward/chosen": -1.296875, "reward/margin": 1.28125, "reward/rejected": -2.578125, "step": 459 }, { "approx. KL/chosen": 205.0, "approx. KL/rejected": 800.0, "epoch": 0.13797240551889622, "grad_norm": 3.2578320503234863, "learning_rate": 9.539581019188158e-06, "logp/chosen": -416.0, "logp/rejected": -446.0, "loss": 0.3137092590332031, "reward/accuracy": 0.75, "reward/chosen": -1.2109375, "reward/margin": 1.953125, "reward/rejected": -3.171875, "step": 460 }, { "approx. KL/chosen": 180.0, "approx. KL/rejected": 672.0, "epoch": 0.13827234553089382, "grad_norm": 4.060821533203125, "learning_rate": 9.537604193114783e-06, "logp/chosen": -241.0, "logp/rejected": -296.0, "loss": 0.381591796875, "reward/accuracy": 0.8125, "reward/chosen": -1.328125, "reward/margin": 1.7265625, "reward/rejected": -3.0625, "step": 461 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 672.0, "epoch": 0.13857228554289142, "grad_norm": 3.16912579536438, "learning_rate": 9.53562333806121e-06, "logp/chosen": -438.0, "logp/rejected": -388.0, "loss": 0.308807373046875, "reward/accuracy": 0.8125, "reward/chosen": -1.2734375, "reward/margin": 1.8046875, "reward/rejected": -3.078125, "step": 462 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 852.0, "epoch": 0.138872225554889, "grad_norm": 3.283353805541992, "learning_rate": 9.53363845578626e-06, "logp/chosen": -252.0, "logp/rejected": -354.0, "loss": 0.30785179138183594, "reward/accuracy": 0.875, "reward/chosen": -0.875, "reward/margin": 2.296875, "reward/rejected": -3.171875, "step": 463 }, { "approx. KL/chosen": 148.0, "approx. KL/rejected": 544.0, "epoch": 0.13917216556688664, "grad_norm": 4.104714870452881, "learning_rate": 9.531649548052329e-06, "logp/chosen": -253.0, "logp/rejected": -268.0, "loss": 0.3969879150390625, "reward/accuracy": 0.8125, "reward/chosen": -1.3828125, "reward/margin": 1.546875, "reward/rejected": -2.921875, "step": 464 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 900.0, "epoch": 0.13947210557888423, "grad_norm": 2.3242149353027344, "learning_rate": 9.529656616625384e-06, "logp/chosen": -290.0, "logp/rejected": -320.0, "loss": 0.252349853515625, "reward/accuracy": 0.875, "reward/chosen": -1.4609375, "reward/margin": 2.375, "reward/rejected": -3.828125, "step": 465 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 1328.0, "epoch": 0.13977204559088183, "grad_norm": 2.940847158432007, "learning_rate": 9.527659663274967e-06, "logp/chosen": -404.0, "logp/rejected": -480.0, "loss": 0.2017059326171875, "reward/accuracy": 0.875, "reward/chosen": -1.640625, "reward/margin": 3.046875, "reward/rejected": -4.6875, "step": 466 }, { "approx. KL/chosen": 386.0, "approx. KL/rejected": 900.0, "epoch": 0.14007198560287942, "grad_norm": 4.548080921173096, "learning_rate": 9.525658689774194e-06, "logp/chosen": -362.0, "logp/rejected": -410.0, "loss": 0.52203369140625, "reward/accuracy": 0.75, "reward/chosen": -2.15625, "reward/margin": 1.640625, "reward/rejected": -3.796875, "step": 467 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 928.0, "epoch": 0.14037192561487702, "grad_norm": 4.631075382232666, "learning_rate": 9.523653697899742e-06, "logp/chosen": -240.0, "logp/rejected": -364.0, "loss": 0.43212890625, "reward/accuracy": 0.75, "reward/chosen": -2.015625, "reward/margin": 1.5703125, "reward/rejected": -3.578125, "step": 468 }, { "approx. KL/chosen": 832.0, "approx. KL/rejected": 996.0, "epoch": 0.14067186562687461, "grad_norm": 11.728839874267578, "learning_rate": 9.521644689431868e-06, "logp/chosen": -362.0, "logp/rejected": -227.0, "loss": 1.176513671875, "reward/accuracy": 0.5625, "reward/chosen": -3.21875, "reward/margin": 0.64453125, "reward/rejected": -3.859375, "step": 469 }, { "approx. KL/chosen": 504.0, "approx. KL/rejected": 1304.0, "epoch": 0.14097180563887224, "grad_norm": 5.450295448303223, "learning_rate": 9.519631666154385e-06, "logp/chosen": -426.0, "logp/rejected": -416.0, "loss": 0.431365966796875, "reward/accuracy": 0.8125, "reward/chosen": -2.3125, "reward/margin": 2.28125, "reward/rejected": -4.59375, "step": 470 }, { "approx. KL/chosen": 486.0, "approx. KL/rejected": 808.0, "epoch": 0.14127174565086983, "grad_norm": 6.246323108673096, "learning_rate": 9.517614629854677e-06, "logp/chosen": -264.0, "logp/rejected": -324.0, "loss": 0.6392822265625, "reward/accuracy": 0.6875, "reward/chosen": -2.03125, "reward/margin": 1.4765625, "reward/rejected": -3.5, "step": 471 }, { "approx. KL/chosen": 428.0, "approx. KL/rejected": 744.0, "epoch": 0.14157168566286743, "grad_norm": 7.223180294036865, "learning_rate": 9.515593582323688e-06, "logp/chosen": -358.0, "logp/rejected": -388.0, "loss": 0.939453125, "reward/accuracy": 0.625, "reward/chosen": -2.5625, "reward/margin": 0.5625, "reward/rejected": -3.125, "step": 472 }, { "approx. KL/chosen": 310.0, "approx. KL/rejected": 1104.0, "epoch": 0.14187162567486503, "grad_norm": 3.1647815704345703, "learning_rate": 9.513568525355923e-06, "logp/chosen": -438.0, "logp/rejected": -344.0, "loss": 0.17822265625, "reward/accuracy": 0.9375, "reward/chosen": -2.046875, "reward/margin": 2.421875, "reward/rejected": -4.46875, "step": 473 }, { "approx. KL/chosen": 436.0, "approx. KL/rejected": 1680.0, "epoch": 0.14217156568686262, "grad_norm": 3.576754093170166, "learning_rate": 9.51153946074945e-06, "logp/chosen": -310.0, "logp/rejected": -360.0, "loss": 0.460784912109375, "reward/accuracy": 0.9375, "reward/chosen": -2.40625, "reward/margin": 2.484375, "reward/rejected": -4.90625, "step": 474 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 952.0, "epoch": 0.14247150569886022, "grad_norm": 2.5422263145446777, "learning_rate": 9.509506390305896e-06, "logp/chosen": -340.0, "logp/rejected": -346.0, "loss": 0.23955535888671875, "reward/accuracy": 0.875, "reward/chosen": -1.4609375, "reward/margin": 2.328125, "reward/rejected": -3.78125, "step": 475 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 1088.0, "epoch": 0.14277144571085784, "grad_norm": 6.063265800476074, "learning_rate": 9.50746931583044e-06, "logp/chosen": -402.0, "logp/rejected": -488.0, "loss": 0.5971145629882812, "reward/accuracy": 0.75, "reward/chosen": -1.921875, "reward/margin": 2.03125, "reward/rejected": -3.953125, "step": 476 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 940.0, "epoch": 0.14307138572285544, "grad_norm": 5.015918731689453, "learning_rate": 9.505428239131818e-06, "logp/chosen": -354.0, "logp/rejected": -308.0, "loss": 0.4708251953125, "reward/accuracy": 0.6875, "reward/chosen": -1.8359375, "reward/margin": 1.734375, "reward/rejected": -3.578125, "step": 477 }, { "approx. KL/chosen": 496.0, "approx. KL/rejected": 1136.0, "epoch": 0.14337132573485303, "grad_norm": 11.028759002685547, "learning_rate": 9.503383162022324e-06, "logp/chosen": -444.0, "logp/rejected": -384.0, "loss": 0.5498046875, "reward/accuracy": 0.6875, "reward/chosen": -2.1875, "reward/margin": 1.640625, "reward/rejected": -3.828125, "step": 478 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 756.0, "epoch": 0.14367126574685063, "grad_norm": 3.0728938579559326, "learning_rate": 9.5013340863178e-06, "logp/chosen": -386.0, "logp/rejected": -324.0, "loss": 0.2313079833984375, "reward/accuracy": 0.875, "reward/chosen": -1.28125, "reward/margin": 2.09375, "reward/rejected": -3.375, "step": 479 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 504.0, "epoch": 0.14397120575884823, "grad_norm": 3.8657257556915283, "learning_rate": 9.499281013837637e-06, "logp/chosen": -312.0, "logp/rejected": -318.0, "loss": 0.417572021484375, "reward/accuracy": 0.75, "reward/chosen": -1.4765625, "reward/margin": 1.296875, "reward/rejected": -2.78125, "step": 480 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 1032.0, "epoch": 0.14427114577084582, "grad_norm": 3.7147936820983887, "learning_rate": 9.497223946404778e-06, "logp/chosen": -278.0, "logp/rejected": -384.0, "loss": 0.253173828125, "reward/accuracy": 0.9375, "reward/chosen": -1.25, "reward/margin": 2.625, "reward/rejected": -3.875, "step": 481 }, { "approx. KL/chosen": 346.0, "approx. KL/rejected": 776.0, "epoch": 0.14457108578284344, "grad_norm": 5.241866111755371, "learning_rate": 9.495162885845713e-06, "logp/chosen": -488.0, "logp/rejected": -360.0, "loss": 0.4995269775390625, "reward/accuracy": 0.8125, "reward/chosen": -1.6796875, "reward/margin": 1.765625, "reward/rejected": -3.453125, "step": 482 }, { "approx. KL/chosen": 116.5, "approx. KL/rejected": 732.0, "epoch": 0.14487102579484104, "grad_norm": 3.1131203174591064, "learning_rate": 9.493097833990474e-06, "logp/chosen": -204.0, "logp/rejected": -268.0, "loss": 0.3458251953125, "reward/accuracy": 0.8125, "reward/chosen": -0.51953125, "reward/margin": 2.234375, "reward/rejected": -2.75, "step": 483 }, { "approx. KL/chosen": 146.0, "approx. KL/rejected": 244.0, "epoch": 0.14517096580683864, "grad_norm": 4.471687316894531, "learning_rate": 9.49102879267264e-06, "logp/chosen": -334.0, "logp/rejected": -292.0, "loss": 0.4781494140625, "reward/accuracy": 0.75, "reward/chosen": -0.85546875, "reward/margin": 1.0546875, "reward/rejected": -1.90625, "step": 484 }, { "approx. KL/chosen": 154.0, "approx. KL/rejected": 536.0, "epoch": 0.14547090581883623, "grad_norm": 6.827117443084717, "learning_rate": 9.488955763729334e-06, "logp/chosen": -284.0, "logp/rejected": -344.0, "loss": 0.5205078125, "reward/accuracy": 0.75, "reward/chosen": -1.34375, "reward/margin": 1.53125, "reward/rejected": -2.875, "step": 485 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 318.0, "epoch": 0.14577084583083383, "grad_norm": 4.3942389488220215, "learning_rate": 9.486878749001215e-06, "logp/chosen": -288.0, "logp/rejected": -244.0, "loss": 0.58734130859375, "reward/accuracy": 0.8125, "reward/chosen": -1.1796875, "reward/margin": 0.9375, "reward/rejected": -2.125, "step": 486 }, { "approx. KL/chosen": 62.25, "approx. KL/rejected": 640.0, "epoch": 0.14607078584283142, "grad_norm": 4.1936445236206055, "learning_rate": 9.484797750332486e-06, "logp/chosen": -334.0, "logp/rejected": -310.0, "loss": 0.26287841796875, "reward/accuracy": 0.8125, "reward/chosen": -0.478515625, "reward/margin": 2.515625, "reward/rejected": -3.0, "step": 487 }, { "approx. KL/chosen": 75.0, "approx. KL/rejected": 386.0, "epoch": 0.14637072585482905, "grad_norm": 4.920354843139648, "learning_rate": 9.48271276957088e-06, "logp/chosen": -414.0, "logp/rejected": -468.0, "loss": 0.45806884765625, "reward/accuracy": 0.8125, "reward/chosen": -0.68359375, "reward/margin": 1.40625, "reward/rejected": -2.09375, "step": 488 }, { "approx. KL/chosen": 82.0, "approx. KL/rejected": 704.0, "epoch": 0.14667066586682664, "grad_norm": 4.8474225997924805, "learning_rate": 9.480623808567677e-06, "logp/chosen": -296.0, "logp/rejected": -308.0, "loss": 0.457763671875, "reward/accuracy": 0.75, "reward/chosen": -0.87890625, "reward/margin": 1.8828125, "reward/rejected": -2.765625, "step": 489 }, { "approx. KL/chosen": 92.0, "approx. KL/rejected": 608.0, "epoch": 0.14697060587882424, "grad_norm": 3.1914970874786377, "learning_rate": 9.478530869177678e-06, "logp/chosen": -314.0, "logp/rejected": -238.0, "loss": 0.25958251953125, "reward/accuracy": 0.9375, "reward/chosen": -0.52734375, "reward/margin": 2.265625, "reward/rejected": -2.796875, "step": 490 }, { "approx. KL/chosen": 67.5, "approx. KL/rejected": 233.0, "epoch": 0.14727054589082184, "grad_norm": 5.581817626953125, "learning_rate": 9.476433953259227e-06, "logp/chosen": -390.0, "logp/rejected": -372.0, "loss": 0.591064453125, "reward/accuracy": 0.5625, "reward/chosen": -0.57421875, "reward/margin": 0.75390625, "reward/rejected": -1.328125, "step": 491 }, { "approx. KL/chosen": 83.0, "approx. KL/rejected": 536.0, "epoch": 0.14757048590281943, "grad_norm": 5.125807762145996, "learning_rate": 9.474333062674194e-06, "logp/chosen": -320.0, "logp/rejected": -368.0, "loss": 0.5504512786865234, "reward/accuracy": 0.75, "reward/chosen": -0.388671875, "reward/margin": 1.5859375, "reward/rejected": -1.9765625, "step": 492 }, { "approx. KL/chosen": 18.0, "approx. KL/rejected": 222.0, "epoch": 0.14787042591481703, "grad_norm": 3.8695671558380127, "learning_rate": 9.472228199287978e-06, "logp/chosen": -510.0, "logp/rejected": -366.0, "loss": 0.412841796875, "reward/accuracy": 0.75, "reward/chosen": -0.1962890625, "reward/margin": 1.2109375, "reward/rejected": -1.40625, "step": 493 }, { "approx. KL/chosen": 57.5, "approx. KL/rejected": 432.0, "epoch": 0.14817036592681462, "grad_norm": 4.0683441162109375, "learning_rate": 9.470119364969507e-06, "logp/chosen": -296.0, "logp/rejected": -350.0, "loss": 0.4688720703125, "reward/accuracy": 0.8125, "reward/chosen": -0.494140625, "reward/margin": 1.515625, "reward/rejected": -2.015625, "step": 494 }, { "approx. KL/chosen": 99.5, "approx. KL/rejected": 167.0, "epoch": 0.14847030593881225, "grad_norm": 5.195794105529785, "learning_rate": 9.468006561591234e-06, "logp/chosen": -292.0, "logp/rejected": -268.0, "loss": 0.61248779296875, "reward/accuracy": 0.75, "reward/chosen": -0.4453125, "reward/margin": 0.79296875, "reward/rejected": -1.234375, "step": 495 }, { "approx. KL/chosen": 38.25, "approx. KL/rejected": 492.0, "epoch": 0.14877024595080984, "grad_norm": 3.3542630672454834, "learning_rate": 9.465889791029136e-06, "logp/chosen": -255.0, "logp/rejected": -276.0, "loss": 0.2523193359375, "reward/accuracy": 0.875, "reward/chosen": -0.33984375, "reward/margin": 2.28125, "reward/rejected": -2.609375, "step": 496 }, { "approx. KL/chosen": 167.0, "approx. KL/rejected": 276.0, "epoch": 0.14907018596280744, "grad_norm": 6.242645263671875, "learning_rate": 9.463769055162715e-06, "logp/chosen": -332.0, "logp/rejected": -274.0, "loss": 0.623046875, "reward/accuracy": 0.8125, "reward/chosen": -1.1640625, "reward/margin": 0.62890625, "reward/rejected": -1.7890625, "step": 497 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 302.0, "epoch": 0.14937012597480503, "grad_norm": 5.032497882843018, "learning_rate": 9.46164435587499e-06, "logp/chosen": -334.0, "logp/rejected": -336.0, "loss": 0.7022705078125, "reward/accuracy": 0.625, "reward/chosen": -1.140625, "reward/margin": 0.640625, "reward/rejected": -1.78125, "step": 498 }, { "approx. KL/chosen": 38.5, "approx. KL/rejected": 652.0, "epoch": 0.14967006598680263, "grad_norm": 3.522458791732788, "learning_rate": 9.459515695052501e-06, "logp/chosen": -280.0, "logp/rejected": -324.0, "loss": 0.3021392822265625, "reward/accuracy": 0.8125, "reward/chosen": -0.44140625, "reward/margin": 2.546875, "reward/rejected": -2.984375, "step": 499 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 458.0, "epoch": 0.14997000599880023, "grad_norm": 5.688084125518799, "learning_rate": 9.457383074585307e-06, "logp/chosen": -336.0, "logp/rejected": -276.0, "loss": 0.6234130859375, "reward/accuracy": 0.8125, "reward/chosen": -1.40625, "reward/margin": 1.078125, "reward/rejected": -2.484375, "step": 500 }, { "approx. KL/chosen": 112.0, "approx. KL/rejected": 502.0, "epoch": 0.15026994601079785, "grad_norm": 3.633244752883911, "learning_rate": 9.455246496366979e-06, "logp/chosen": -340.0, "logp/rejected": -352.0, "loss": 0.4485321044921875, "reward/accuracy": 0.8125, "reward/chosen": -0.74609375, "reward/margin": 1.5546875, "reward/rejected": -2.296875, "step": 501 }, { "approx. KL/chosen": 88.5, "approx. KL/rejected": 696.0, "epoch": 0.15056988602279545, "grad_norm": 3.9552478790283203, "learning_rate": 9.453105962294604e-06, "logp/chosen": -284.0, "logp/rejected": -304.0, "loss": 0.25958251953125, "reward/accuracy": 0.875, "reward/chosen": -0.66015625, "reward/margin": 2.75, "reward/rejected": -3.40625, "step": 502 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 472.0, "epoch": 0.15086982603479304, "grad_norm": 4.308509349822998, "learning_rate": 9.450961474268783e-06, "logp/chosen": -396.0, "logp/rejected": -364.0, "loss": 0.5233154296875, "reward/accuracy": 0.625, "reward/chosen": -1.3359375, "reward/margin": 0.875, "reward/rejected": -2.21875, "step": 503 }, { "approx. KL/chosen": 243.0, "approx. KL/rejected": 366.0, "epoch": 0.15116976604679064, "grad_norm": 6.801818370819092, "learning_rate": 9.448813034193626e-06, "logp/chosen": -404.0, "logp/rejected": -374.0, "loss": 0.5986328125, "reward/accuracy": 0.6875, "reward/chosen": -1.3203125, "reward/margin": 0.63671875, "reward/rejected": -1.9609375, "step": 504 }, { "approx. KL/chosen": 61.25, "approx. KL/rejected": 362.0, "epoch": 0.15146970605878823, "grad_norm": 7.106175899505615, "learning_rate": 9.446660643976755e-06, "logp/chosen": -370.0, "logp/rejected": -344.0, "loss": 0.4818115234375, "reward/accuracy": 0.75, "reward/chosen": -0.6953125, "reward/margin": 1.15625, "reward/rejected": -1.8515625, "step": 505 }, { "approx. KL/chosen": 89.5, "approx. KL/rejected": 512.0, "epoch": 0.15176964607078583, "grad_norm": 4.737998962402344, "learning_rate": 9.444504305529294e-06, "logp/chosen": -274.0, "logp/rejected": -370.0, "loss": 0.34234619140625, "reward/accuracy": 0.875, "reward/chosen": -0.609375, "reward/margin": 1.9140625, "reward/rejected": -2.515625, "step": 506 }, { "approx. KL/chosen": 294.0, "approx. KL/rejected": 368.0, "epoch": 0.15206958608278345, "grad_norm": 7.4077301025390625, "learning_rate": 9.442344020765876e-06, "logp/chosen": -230.0, "logp/rejected": -256.0, "loss": 0.831298828125, "reward/accuracy": 0.625, "reward/chosen": -1.1875, "reward/margin": 1.03125, "reward/rejected": -2.21875, "step": 507 }, { "approx. KL/chosen": 48.0, "approx. KL/rejected": 408.0, "epoch": 0.15236952609478105, "grad_norm": 2.758010149002075, "learning_rate": 9.440179791604638e-06, "logp/chosen": -272.0, "logp/rejected": -378.0, "loss": 0.32891845703125, "reward/accuracy": 0.75, "reward/chosen": -0.5, "reward/margin": 1.7890625, "reward/rejected": -2.28125, "step": 508 }, { "approx. KL/chosen": 101.5, "approx. KL/rejected": 384.0, "epoch": 0.15266946610677865, "grad_norm": 4.989963531494141, "learning_rate": 9.438011619967218e-06, "logp/chosen": -308.0, "logp/rejected": -364.0, "loss": 0.5245361328125, "reward/accuracy": 0.6875, "reward/chosen": -0.5, "reward/margin": 1.4375, "reward/rejected": -1.9375, "step": 509 }, { "approx. KL/chosen": 222.0, "approx. KL/rejected": 410.0, "epoch": 0.15296940611877624, "grad_norm": 6.9831366539001465, "learning_rate": 9.435839507778758e-06, "logp/chosen": -406.0, "logp/rejected": -480.0, "loss": 0.80743408203125, "reward/accuracy": 0.6875, "reward/chosen": -1.28125, "reward/margin": 0.9140625, "reward/rejected": -2.203125, "step": 510 }, { "approx. KL/chosen": 102.0, "approx. KL/rejected": 221.0, "epoch": 0.15326934613077384, "grad_norm": 4.859940528869629, "learning_rate": 9.433663456967893e-06, "logp/chosen": -300.0, "logp/rejected": -268.0, "loss": 0.495849609375, "reward/accuracy": 0.6875, "reward/chosen": -0.75, "reward/margin": 0.86328125, "reward/rejected": -1.6171875, "step": 511 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 466.0, "epoch": 0.15356928614277143, "grad_norm": 7.350013732910156, "learning_rate": 9.43148346946676e-06, "logp/chosen": -378.0, "logp/rejected": -308.0, "loss": 0.5643310546875, "reward/accuracy": 0.6875, "reward/chosen": -0.7734375, "reward/margin": 1.2421875, "reward/rejected": -2.015625, "step": 512 }, { "approx. KL/chosen": 187.0, "approx. KL/rejected": 752.0, "epoch": 0.15386922615476906, "grad_norm": 4.6429901123046875, "learning_rate": 9.42929954721099e-06, "logp/chosen": -336.0, "logp/rejected": -380.0, "loss": 0.3002471923828125, "reward/accuracy": 0.75, "reward/chosen": -0.92578125, "reward/margin": 2.4375, "reward/rejected": -3.375, "step": 513 }, { "approx. KL/chosen": 74.5, "approx. KL/rejected": 284.0, "epoch": 0.15416916616676665, "grad_norm": 3.859644651412964, "learning_rate": 9.427111692139708e-06, "logp/chosen": -286.0, "logp/rejected": -242.0, "loss": 0.47235107421875, "reward/accuracy": 0.6875, "reward/chosen": -0.66796875, "reward/margin": 1.1796875, "reward/rejected": -1.8515625, "step": 514 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 516.0, "epoch": 0.15446910617876425, "grad_norm": 6.925060749053955, "learning_rate": 9.42491990619553e-06, "logp/chosen": -386.0, "logp/rejected": -348.0, "loss": 0.60040283203125, "reward/accuracy": 0.625, "reward/chosen": -1.15625, "reward/margin": 1.5078125, "reward/rejected": -2.671875, "step": 515 }, { "approx. KL/chosen": 64.0, "approx. KL/rejected": 532.0, "epoch": 0.15476904619076184, "grad_norm": 3.3489809036254883, "learning_rate": 9.42272419132456e-06, "logp/chosen": -336.0, "logp/rejected": -396.0, "loss": 0.395477294921875, "reward/accuracy": 0.75, "reward/chosen": -0.63671875, "reward/margin": 1.828125, "reward/rejected": -2.453125, "step": 516 }, { "approx. KL/chosen": 62.25, "approx. KL/rejected": 266.0, "epoch": 0.15506898620275944, "grad_norm": 6.868459701538086, "learning_rate": 9.420524549476398e-06, "logp/chosen": -298.0, "logp/rejected": -270.0, "loss": 0.62261962890625, "reward/accuracy": 0.6875, "reward/chosen": -0.62109375, "reward/margin": 0.9375, "reward/rejected": -1.5625, "step": 517 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 540.0, "epoch": 0.15536892621475704, "grad_norm": 5.08246374130249, "learning_rate": 9.418320982604124e-06, "logp/chosen": -374.0, "logp/rejected": -326.0, "loss": 0.605377197265625, "reward/accuracy": 0.75, "reward/chosen": -1.328125, "reward/margin": 1.3671875, "reward/rejected": -2.703125, "step": 518 }, { "approx. KL/chosen": 64.5, "approx. KL/rejected": 448.0, "epoch": 0.15566886622675466, "grad_norm": 3.6167144775390625, "learning_rate": 9.416113492664303e-06, "logp/chosen": -238.0, "logp/rejected": -334.0, "loss": 0.36187744140625, "reward/accuracy": 0.75, "reward/chosen": -0.44921875, "reward/margin": 1.875, "reward/rejected": -2.328125, "step": 519 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 596.0, "epoch": 0.15596880623875226, "grad_norm": 4.0301032066345215, "learning_rate": 9.41390208161699e-06, "logp/chosen": -314.0, "logp/rejected": -372.0, "loss": 0.3737030029296875, "reward/accuracy": 0.75, "reward/chosen": -1.28125, "reward/margin": 1.8984375, "reward/rejected": -3.1875, "step": 520 }, { "approx. KL/chosen": 81.0, "approx. KL/rejected": 350.0, "epoch": 0.15626874625074985, "grad_norm": 3.634796142578125, "learning_rate": 9.411686751425711e-06, "logp/chosen": -241.0, "logp/rejected": -330.0, "loss": 0.38470458984375, "reward/accuracy": 0.8125, "reward/chosen": -0.5390625, "reward/margin": 1.4765625, "reward/rejected": -2.015625, "step": 521 }, { "approx. KL/chosen": 48.25, "approx. KL/rejected": 466.0, "epoch": 0.15656868626274745, "grad_norm": 4.7135162353515625, "learning_rate": 9.409467504057484e-06, "logp/chosen": -352.0, "logp/rejected": -340.0, "loss": 0.4438495635986328, "reward/accuracy": 0.8125, "reward/chosen": -0.345703125, "reward/margin": 1.7265625, "reward/rejected": -2.078125, "step": 522 }, { "approx. KL/chosen": 85.0, "approx. KL/rejected": 412.0, "epoch": 0.15686862627474504, "grad_norm": 4.792730808258057, "learning_rate": 9.407244341482795e-06, "logp/chosen": -366.0, "logp/rejected": -276.0, "loss": 0.5303955078125, "reward/accuracy": 0.75, "reward/chosen": -0.8828125, "reward/margin": 1.3125, "reward/rejected": -2.1875, "step": 523 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 616.0, "epoch": 0.15716856628674264, "grad_norm": 3.83632755279541, "learning_rate": 9.405017265675613e-06, "logp/chosen": -242.0, "logp/rejected": -316.0, "loss": 0.33636474609375, "reward/accuracy": 0.875, "reward/chosen": -1.203125, "reward/margin": 1.8125, "reward/rejected": -3.015625, "step": 524 }, { "approx. KL/chosen": 65.5, "approx. KL/rejected": 460.0, "epoch": 0.15746850629874026, "grad_norm": 4.152481555938721, "learning_rate": 9.402786278613378e-06, "logp/chosen": -356.0, "logp/rejected": -382.0, "loss": 0.453125, "reward/accuracy": 0.6875, "reward/chosen": -0.71875, "reward/margin": 1.734375, "reward/rejected": -2.453125, "step": 525 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 684.0, "epoch": 0.15776844631073786, "grad_norm": 5.536340236663818, "learning_rate": 9.400551382277003e-06, "logp/chosen": -324.0, "logp/rejected": -326.0, "loss": 0.7086181640625, "reward/accuracy": 0.6875, "reward/chosen": -1.4296875, "reward/margin": 1.46875, "reward/rejected": -2.90625, "step": 526 }, { "approx. KL/chosen": 67.5, "approx. KL/rejected": 360.0, "epoch": 0.15806838632273545, "grad_norm": 5.170590877532959, "learning_rate": 9.398312578650872e-06, "logp/chosen": -390.0, "logp/rejected": -364.0, "loss": 0.453948974609375, "reward/accuracy": 0.75, "reward/chosen": -0.439453125, "reward/margin": 1.703125, "reward/rejected": -2.140625, "step": 527 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 648.0, "epoch": 0.15836832633473305, "grad_norm": 5.231219291687012, "learning_rate": 9.396069869722842e-06, "logp/chosen": -298.0, "logp/rejected": -364.0, "loss": 0.52484130859375, "reward/accuracy": 0.625, "reward/chosen": -1.625, "reward/margin": 1.5390625, "reward/rejected": -3.15625, "step": 528 }, { "approx. KL/chosen": 167.0, "approx. KL/rejected": 576.0, "epoch": 0.15866826634673065, "grad_norm": 3.8628275394439697, "learning_rate": 9.393823257484232e-06, "logp/chosen": -330.0, "logp/rejected": -416.0, "loss": 0.340576171875, "reward/accuracy": 0.8125, "reward/chosen": -1.2578125, "reward/margin": 1.7421875, "reward/rejected": -3.0, "step": 529 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 636.0, "epoch": 0.15896820635872824, "grad_norm": 5.985877990722656, "learning_rate": 9.391572743929832e-06, "logp/chosen": -340.0, "logp/rejected": -286.0, "loss": 0.6292724609375, "reward/accuracy": 0.625, "reward/chosen": -1.6015625, "reward/margin": 1.40625, "reward/rejected": -3.015625, "step": 530 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 696.0, "epoch": 0.15926814637072587, "grad_norm": 6.233911991119385, "learning_rate": 9.389318331057894e-06, "logp/chosen": -306.0, "logp/rejected": -378.0, "loss": 0.6260986328125, "reward/accuracy": 0.5625, "reward/chosen": -1.1875, "reward/margin": 1.4140625, "reward/rejected": -2.609375, "step": 531 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 516.0, "epoch": 0.15956808638272346, "grad_norm": 7.467121601104736, "learning_rate": 9.387060020870127e-06, "logp/chosen": -308.0, "logp/rejected": -212.0, "loss": 0.762939453125, "reward/accuracy": 0.75, "reward/chosen": -1.8671875, "reward/margin": 0.8671875, "reward/rejected": -2.734375, "step": 532 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 712.0, "epoch": 0.15986802639472106, "grad_norm": 2.491490602493286, "learning_rate": 9.384797815371708e-06, "logp/chosen": -376.0, "logp/rejected": -254.0, "loss": 0.2418212890625, "reward/accuracy": 0.9375, "reward/chosen": -1.28125, "reward/margin": 2.03125, "reward/rejected": -3.3125, "step": 533 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 568.0, "epoch": 0.16016796640671865, "grad_norm": 4.524711608886719, "learning_rate": 9.382531716571273e-06, "logp/chosen": -382.0, "logp/rejected": -398.0, "loss": 0.44146728515625, "reward/accuracy": 0.875, "reward/chosen": -1.5703125, "reward/margin": 1.5078125, "reward/rejected": -3.078125, "step": 534 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 640.0, "epoch": 0.16046790641871625, "grad_norm": 3.925401210784912, "learning_rate": 9.380261726480906e-06, "logp/chosen": -412.0, "logp/rejected": -400.0, "loss": 0.415496826171875, "reward/accuracy": 0.8125, "reward/chosen": -1.3359375, "reward/margin": 1.6875, "reward/rejected": -3.015625, "step": 535 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 438.0, "epoch": 0.16076784643071385, "grad_norm": 6.029313087463379, "learning_rate": 9.377987847116156e-06, "logp/chosen": -326.0, "logp/rejected": -348.0, "loss": 0.84039306640625, "reward/accuracy": 0.5625, "reward/chosen": -1.8671875, "reward/margin": 0.57421875, "reward/rejected": -2.4375, "step": 536 }, { "approx. KL/chosen": 138.0, "approx. KL/rejected": 848.0, "epoch": 0.16106778644271147, "grad_norm": 3.6902353763580322, "learning_rate": 9.375710080496023e-06, "logp/chosen": -302.0, "logp/rejected": -436.0, "loss": 0.2574615478515625, "reward/accuracy": 0.875, "reward/chosen": -1.328125, "reward/margin": 2.375, "reward/rejected": -3.703125, "step": 537 }, { "approx. KL/chosen": 105.5, "approx. KL/rejected": 660.0, "epoch": 0.16136772645470907, "grad_norm": 3.676837921142578, "learning_rate": 9.37342842864295e-06, "logp/chosen": -328.0, "logp/rejected": -362.0, "loss": 0.40692138671875, "reward/accuracy": 0.8125, "reward/chosen": -0.859375, "reward/margin": 2.21875, "reward/rejected": -3.09375, "step": 538 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 342.0, "epoch": 0.16166766646670666, "grad_norm": 5.130422115325928, "learning_rate": 9.371142893582843e-06, "logp/chosen": -364.0, "logp/rejected": -312.0, "loss": 0.63232421875, "reward/accuracy": 0.625, "reward/chosen": -1.0703125, "reward/margin": 1.125, "reward/rejected": -2.203125, "step": 539 }, { "approx. KL/chosen": 310.0, "approx. KL/rejected": 500.0, "epoch": 0.16196760647870426, "grad_norm": 8.984579086303711, "learning_rate": 9.368853477345047e-06, "logp/chosen": -414.0, "logp/rejected": -408.0, "loss": 0.9063720703125, "reward/accuracy": 0.5625, "reward/chosen": -1.75, "reward/margin": 0.7578125, "reward/rejected": -2.5, "step": 540 }, { "approx. KL/chosen": 237.0, "approx. KL/rejected": 508.0, "epoch": 0.16226754649070185, "grad_norm": 4.6926469802856445, "learning_rate": 9.366560181962354e-06, "logp/chosen": -436.0, "logp/rejected": -476.0, "loss": 0.4818572998046875, "reward/accuracy": 0.75, "reward/chosen": -1.53125, "reward/margin": 1.28125, "reward/rejected": -2.8125, "step": 541 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 744.0, "epoch": 0.16256748650269945, "grad_norm": 3.764634847640991, "learning_rate": 9.364263009471006e-06, "logp/chosen": -402.0, "logp/rejected": -376.0, "loss": 0.391448974609375, "reward/accuracy": 0.75, "reward/chosen": -1.03125, "reward/margin": 1.7890625, "reward/rejected": -2.828125, "step": 542 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 600.0, "epoch": 0.16286742651469707, "grad_norm": 3.440852403640747, "learning_rate": 9.361961961910679e-06, "logp/chosen": -398.0, "logp/rejected": -330.0, "loss": 0.364349365234375, "reward/accuracy": 0.9375, "reward/chosen": -1.484375, "reward/margin": 1.671875, "reward/rejected": -3.171875, "step": 543 }, { "approx. KL/chosen": 114.5, "approx. KL/rejected": 536.0, "epoch": 0.16316736652669467, "grad_norm": 3.2621867656707764, "learning_rate": 9.359657041324496e-06, "logp/chosen": -352.0, "logp/rejected": -358.0, "loss": 0.41943359375, "reward/accuracy": 0.75, "reward/chosen": -1.2734375, "reward/margin": 1.5859375, "reward/rejected": -2.859375, "step": 544 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 472.0, "epoch": 0.16346730653869226, "grad_norm": 4.026131629943848, "learning_rate": 9.357348249759017e-06, "logp/chosen": -348.0, "logp/rejected": -368.0, "loss": 0.484375, "reward/accuracy": 0.75, "reward/chosen": -1.5625, "reward/margin": 1.09375, "reward/rejected": -2.65625, "step": 545 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 796.0, "epoch": 0.16376724655068986, "grad_norm": 3.6269097328186035, "learning_rate": 9.35503558926424e-06, "logp/chosen": -382.0, "logp/rejected": -452.0, "loss": 0.3956298828125, "reward/accuracy": 0.75, "reward/chosen": -1.4765625, "reward/margin": 2.015625, "reward/rejected": -3.484375, "step": 546 }, { "approx. KL/chosen": 167.0, "approx. KL/rejected": 362.0, "epoch": 0.16406718656268746, "grad_norm": 5.228461265563965, "learning_rate": 9.352719061893597e-06, "logp/chosen": -330.0, "logp/rejected": -336.0, "loss": 0.591064453125, "reward/accuracy": 0.6875, "reward/chosen": -1.2578125, "reward/margin": 0.92578125, "reward/rejected": -2.1875, "step": 547 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 498.0, "epoch": 0.16436712657468505, "grad_norm": 4.524836540222168, "learning_rate": 9.350398669703953e-06, "logp/chosen": -290.0, "logp/rejected": -402.0, "loss": 0.45556640625, "reward/accuracy": 0.8125, "reward/chosen": -1.515625, "reward/margin": 1.1328125, "reward/rejected": -2.640625, "step": 548 }, { "approx. KL/chosen": 116.0, "approx. KL/rejected": 724.0, "epoch": 0.16466706658668268, "grad_norm": 5.822017669677734, "learning_rate": 9.348074414755605e-06, "logp/chosen": -352.0, "logp/rejected": -358.0, "loss": 0.4456787109375, "reward/accuracy": 0.8125, "reward/chosen": -0.99609375, "reward/margin": 1.9140625, "reward/rejected": -2.90625, "step": 549 }, { "approx. KL/chosen": 145.0, "approx. KL/rejected": 528.0, "epoch": 0.16496700659868027, "grad_norm": 3.4647271633148193, "learning_rate": 9.345746299112282e-06, "logp/chosen": -292.0, "logp/rejected": -298.0, "loss": 0.428131103515625, "reward/accuracy": 0.8125, "reward/chosen": -1.359375, "reward/margin": 1.4453125, "reward/rejected": -2.796875, "step": 550 }, { "approx. KL/chosen": 304.0, "approx. KL/rejected": 446.0, "epoch": 0.16526694661067787, "grad_norm": 4.392799377441406, "learning_rate": 9.343414324841136e-06, "logp/chosen": -406.0, "logp/rejected": -308.0, "loss": 0.550048828125, "reward/accuracy": 0.6875, "reward/chosen": -1.890625, "reward/margin": 0.75, "reward/rejected": -2.640625, "step": 551 }, { "approx. KL/chosen": 448.0, "approx. KL/rejected": 664.0, "epoch": 0.16556688662267546, "grad_norm": 5.272342681884766, "learning_rate": 9.341078494012752e-06, "logp/chosen": -438.0, "logp/rejected": -374.0, "loss": 0.6900634765625, "reward/accuracy": 0.5, "reward/chosen": -2.609375, "reward/margin": 0.61328125, "reward/rejected": -3.234375, "step": 552 }, { "approx. KL/chosen": 219.0, "approx. KL/rejected": 644.0, "epoch": 0.16586682663467306, "grad_norm": 3.1124401092529297, "learning_rate": 9.338738808701135e-06, "logp/chosen": -268.0, "logp/rejected": -274.0, "loss": 0.41900634765625, "reward/accuracy": 0.875, "reward/chosen": -1.703125, "reward/margin": 1.3828125, "reward/rejected": -3.078125, "step": 553 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 644.0, "epoch": 0.16616676664667065, "grad_norm": 4.41973876953125, "learning_rate": 9.33639527098371e-06, "logp/chosen": -334.0, "logp/rejected": -298.0, "loss": 0.331390380859375, "reward/accuracy": 0.875, "reward/chosen": -1.234375, "reward/margin": 2.0625, "reward/rejected": -3.28125, "step": 554 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 580.0, "epoch": 0.16646670665866828, "grad_norm": 3.5428237915039062, "learning_rate": 9.334047882941328e-06, "logp/chosen": -276.0, "logp/rejected": -330.0, "loss": 0.43463134765625, "reward/accuracy": 0.8125, "reward/chosen": -1.6875, "reward/margin": 1.359375, "reward/rejected": -3.046875, "step": 555 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 636.0, "epoch": 0.16676664667066587, "grad_norm": 4.819982051849365, "learning_rate": 9.331696646658255e-06, "logp/chosen": -237.0, "logp/rejected": -280.0, "loss": 0.430267333984375, "reward/accuracy": 0.875, "reward/chosen": -1.4296875, "reward/margin": 1.71875, "reward/rejected": -3.140625, "step": 556 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 616.0, "epoch": 0.16706658668266347, "grad_norm": 3.4376683235168457, "learning_rate": 9.329341564222176e-06, "logp/chosen": -392.0, "logp/rejected": -414.0, "loss": 0.38323974609375, "reward/accuracy": 0.6875, "reward/chosen": -1.375, "reward/margin": 1.515625, "reward/rejected": -2.890625, "step": 557 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 624.0, "epoch": 0.16736652669466107, "grad_norm": 5.048758029937744, "learning_rate": 9.32698263772419e-06, "logp/chosen": -324.0, "logp/rejected": -376.0, "loss": 0.5861968994140625, "reward/accuracy": 0.75, "reward/chosen": -1.75, "reward/margin": 1.34375, "reward/rejected": -3.09375, "step": 558 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 764.0, "epoch": 0.16766646670665866, "grad_norm": 5.584498882293701, "learning_rate": 9.324619869258811e-06, "logp/chosen": -360.0, "logp/rejected": -348.0, "loss": 0.68017578125, "reward/accuracy": 0.6875, "reward/chosen": -2.03125, "reward/margin": 1.3046875, "reward/rejected": -3.328125, "step": 559 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 560.0, "epoch": 0.16796640671865626, "grad_norm": 4.683656692504883, "learning_rate": 9.322253260923961e-06, "logp/chosen": -360.0, "logp/rejected": -360.0, "loss": 0.549041748046875, "reward/accuracy": 0.6875, "reward/chosen": -1.8125, "reward/margin": 1.1875, "reward/rejected": -3.0, "step": 560 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 708.0, "epoch": 0.16826634673065388, "grad_norm": 3.8533291816711426, "learning_rate": 9.319882814820971e-06, "logp/chosen": -245.0, "logp/rejected": -338.0, "loss": 0.283538818359375, "reward/accuracy": 0.875, "reward/chosen": -1.6640625, "reward/margin": 1.6640625, "reward/rejected": -3.328125, "step": 561 }, { "approx. KL/chosen": 245.0, "approx. KL/rejected": 672.0, "epoch": 0.16856628674265148, "grad_norm": 3.0688235759735107, "learning_rate": 9.317508533054587e-06, "logp/chosen": -372.0, "logp/rejected": -396.0, "loss": 0.2756195068359375, "reward/accuracy": 0.9375, "reward/chosen": -1.453125, "reward/margin": 1.84375, "reward/rejected": -3.296875, "step": 562 }, { "approx. KL/chosen": 294.0, "approx. KL/rejected": 580.0, "epoch": 0.16886622675464907, "grad_norm": 4.588286399841309, "learning_rate": 9.31513041773295e-06, "logp/chosen": -276.0, "logp/rejected": -342.0, "loss": 0.52947998046875, "reward/accuracy": 0.75, "reward/chosen": -2.046875, "reward/margin": 0.9921875, "reward/rejected": -3.03125, "step": 563 }, { "approx. KL/chosen": 188.0, "approx. KL/rejected": 512.0, "epoch": 0.16916616676664667, "grad_norm": 3.4932756423950195, "learning_rate": 9.312748470967615e-06, "logp/chosen": -366.0, "logp/rejected": -296.0, "loss": 0.384918212890625, "reward/accuracy": 0.875, "reward/chosen": -1.65625, "reward/margin": 1.1875, "reward/rejected": -2.84375, "step": 564 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 512.0, "epoch": 0.16946610677864427, "grad_norm": 4.148908615112305, "learning_rate": 9.31036269487353e-06, "logp/chosen": -332.0, "logp/rejected": -262.0, "loss": 0.49615478515625, "reward/accuracy": 0.75, "reward/chosen": -1.3828125, "reward/margin": 1.484375, "reward/rejected": -2.859375, "step": 565 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 356.0, "epoch": 0.16976604679064186, "grad_norm": 4.082188606262207, "learning_rate": 9.30797309156905e-06, "logp/chosen": -318.0, "logp/rejected": -354.0, "loss": 0.52001953125, "reward/accuracy": 0.75, "reward/chosen": -1.5234375, "reward/margin": 0.77734375, "reward/rejected": -2.296875, "step": 566 }, { "approx. KL/chosen": 352.0, "approx. KL/rejected": 484.0, "epoch": 0.17006598680263948, "grad_norm": 4.5078959465026855, "learning_rate": 9.305579663175924e-06, "logp/chosen": -380.0, "logp/rejected": -316.0, "loss": 0.70098876953125, "reward/accuracy": 0.625, "reward/chosen": -2.28125, "reward/margin": 0.4453125, "reward/rejected": -2.734375, "step": 567 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 712.0, "epoch": 0.17036592681463708, "grad_norm": 3.8909077644348145, "learning_rate": 9.303182411819301e-06, "logp/chosen": -492.0, "logp/rejected": -356.0, "loss": 0.36785888671875, "reward/accuracy": 0.8125, "reward/chosen": -1.7734375, "reward/margin": 1.7421875, "reward/rejected": -3.515625, "step": 568 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 572.0, "epoch": 0.17066586682663468, "grad_norm": 4.3983845710754395, "learning_rate": 9.30078133962772e-06, "logp/chosen": -356.0, "logp/rejected": -312.0, "loss": 0.580047607421875, "reward/accuracy": 0.5625, "reward/chosen": -1.7265625, "reward/margin": 1.0390625, "reward/rejected": -2.765625, "step": 569 }, { "approx. KL/chosen": 248.0, "approx. KL/rejected": 540.0, "epoch": 0.17096580683863227, "grad_norm": 4.454798698425293, "learning_rate": 9.298376448733119e-06, "logp/chosen": -424.0, "logp/rejected": -474.0, "loss": 0.474334716796875, "reward/accuracy": 0.8125, "reward/chosen": -1.9453125, "reward/margin": 1.0234375, "reward/rejected": -2.96875, "step": 570 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 732.0, "epoch": 0.17126574685062987, "grad_norm": 2.842893123626709, "learning_rate": 9.295967741270818e-06, "logp/chosen": -450.0, "logp/rejected": -392.0, "loss": 0.26824951171875, "reward/accuracy": 0.875, "reward/chosen": -1.8046875, "reward/margin": 1.6796875, "reward/rejected": -3.484375, "step": 571 }, { "approx. KL/chosen": 229.0, "approx. KL/rejected": 572.0, "epoch": 0.17156568686262746, "grad_norm": 4.024355888366699, "learning_rate": 9.293555219379531e-06, "logp/chosen": -402.0, "logp/rejected": -426.0, "loss": 0.53076171875, "reward/accuracy": 0.8125, "reward/chosen": -1.7890625, "reward/margin": 1.1640625, "reward/rejected": -2.953125, "step": 572 }, { "approx. KL/chosen": 226.0, "approx. KL/rejected": 532.0, "epoch": 0.1718656268746251, "grad_norm": 5.404664516448975, "learning_rate": 9.291138885201364e-06, "logp/chosen": -202.0, "logp/rejected": -272.0, "loss": 0.668212890625, "reward/accuracy": 0.5625, "reward/chosen": -1.921875, "reward/margin": 0.76953125, "reward/rejected": -2.6875, "step": 573 }, { "approx. KL/chosen": 318.0, "approx. KL/rejected": 620.0, "epoch": 0.17216556688662268, "grad_norm": 5.677847385406494, "learning_rate": 9.288718740881793e-06, "logp/chosen": -310.0, "logp/rejected": -396.0, "loss": 0.596954345703125, "reward/accuracy": 0.75, "reward/chosen": -2.15625, "reward/margin": 1.0703125, "reward/rejected": -3.21875, "step": 574 }, { "approx. KL/chosen": 174.0, "approx. KL/rejected": 498.0, "epoch": 0.17246550689862028, "grad_norm": 3.5930991172790527, "learning_rate": 9.286294788569691e-06, "logp/chosen": -344.0, "logp/rejected": -336.0, "loss": 0.36773681640625, "reward/accuracy": 0.875, "reward/chosen": -1.7265625, "reward/margin": 1.2421875, "reward/rejected": -2.96875, "step": 575 }, { "approx. KL/chosen": 94.0, "approx. KL/rejected": 532.0, "epoch": 0.17276544691061788, "grad_norm": 2.5149035453796387, "learning_rate": 9.283867030417308e-06, "logp/chosen": -382.0, "logp/rejected": -346.0, "loss": 0.236480712890625, "reward/accuracy": 1.0, "reward/chosen": -1.1953125, "reward/margin": 1.8125, "reward/rejected": -3.015625, "step": 576 }, { "approx. KL/chosen": 330.0, "approx. KL/rejected": 668.0, "epoch": 0.17306538692261547, "grad_norm": 8.009276390075684, "learning_rate": 9.281435468580273e-06, "logp/chosen": -278.0, "logp/rejected": -286.0, "loss": 0.56494140625, "reward/accuracy": 0.8125, "reward/chosen": -1.8984375, "reward/margin": 1.5234375, "reward/rejected": -3.421875, "step": 577 }, { "approx. KL/chosen": 252.0, "approx. KL/rejected": 596.0, "epoch": 0.17336532693461307, "grad_norm": 4.371428489685059, "learning_rate": 9.279000105217591e-06, "logp/chosen": -346.0, "logp/rejected": -308.0, "loss": 0.4604644775390625, "reward/accuracy": 0.75, "reward/chosen": -1.453125, "reward/margin": 1.5, "reward/rejected": -2.953125, "step": 578 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 528.0, "epoch": 0.1736652669466107, "grad_norm": 3.084519624710083, "learning_rate": 9.276560942491642e-06, "logp/chosen": -280.0, "logp/rejected": -384.0, "loss": 0.30712890625, "reward/accuracy": 0.875, "reward/chosen": -1.40625, "reward/margin": 1.578125, "reward/rejected": -2.984375, "step": 579 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 528.0, "epoch": 0.1739652069586083, "grad_norm": 6.688216209411621, "learning_rate": 9.27411798256818e-06, "logp/chosen": -286.0, "logp/rejected": -306.0, "loss": 0.7624969482421875, "reward/accuracy": 0.375, "reward/chosen": -2.265625, "reward/margin": 0.470703125, "reward/rejected": -2.734375, "step": 580 }, { "approx. KL/chosen": 233.0, "approx. KL/rejected": 564.0, "epoch": 0.17426514697060588, "grad_norm": 4.273782730102539, "learning_rate": 9.271671227616337e-06, "logp/chosen": -314.0, "logp/rejected": -324.0, "loss": 0.5272216796875, "reward/accuracy": 0.8125, "reward/chosen": -1.6484375, "reward/margin": 1.3203125, "reward/rejected": -2.96875, "step": 581 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 414.0, "epoch": 0.17456508698260348, "grad_norm": 5.047565460205078, "learning_rate": 9.269220679808603e-06, "logp/chosen": -336.0, "logp/rejected": -316.0, "loss": 0.6297607421875, "reward/accuracy": 0.6875, "reward/chosen": -2.03125, "reward/margin": 0.58203125, "reward/rejected": -2.609375, "step": 582 }, { "approx. KL/chosen": 184.0, "approx. KL/rejected": 516.0, "epoch": 0.17486502699460107, "grad_norm": 4.940764904022217, "learning_rate": 9.266766341320846e-06, "logp/chosen": -304.0, "logp/rejected": -338.0, "loss": 0.59173583984375, "reward/accuracy": 0.5625, "reward/chosen": -1.6953125, "reward/margin": 0.7890625, "reward/rejected": -2.484375, "step": 583 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 458.0, "epoch": 0.17516496700659867, "grad_norm": 5.5273613929748535, "learning_rate": 9.264308214332294e-06, "logp/chosen": -306.0, "logp/rejected": -360.0, "loss": 0.5740966796875, "reward/accuracy": 0.6875, "reward/chosen": -1.9609375, "reward/margin": 0.8671875, "reward/rejected": -2.828125, "step": 584 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 458.0, "epoch": 0.1754649070185963, "grad_norm": 4.33233642578125, "learning_rate": 9.261846301025539e-06, "logp/chosen": -284.0, "logp/rejected": -280.0, "loss": 0.5086669921875, "reward/accuracy": 0.8125, "reward/chosen": -1.8515625, "reward/margin": 1.0, "reward/rejected": -2.859375, "step": 585 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 580.0, "epoch": 0.1757648470305939, "grad_norm": 3.7290868759155273, "learning_rate": 9.259380603586537e-06, "logp/chosen": -308.0, "logp/rejected": -384.0, "loss": 0.358642578125, "reward/accuracy": 0.8125, "reward/chosen": -1.6328125, "reward/margin": 1.546875, "reward/rejected": -3.1875, "step": 586 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 506.0, "epoch": 0.1760647870425915, "grad_norm": 4.024445533752441, "learning_rate": 9.256911124204606e-06, "logp/chosen": -270.0, "logp/rejected": -262.0, "loss": 0.474945068359375, "reward/accuracy": 0.8125, "reward/chosen": -0.91796875, "reward/margin": 1.8203125, "reward/rejected": -2.734375, "step": 587 }, { "approx. KL/chosen": 119.0, "approx. KL/rejected": 584.0, "epoch": 0.17636472705458908, "grad_norm": 3.3963828086853027, "learning_rate": 9.254437865072415e-06, "logp/chosen": -416.0, "logp/rejected": -326.0, "loss": 0.31890869140625, "reward/accuracy": 0.9375, "reward/chosen": -0.9921875, "reward/margin": 1.78125, "reward/rejected": -2.78125, "step": 588 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 360.0, "epoch": 0.17666466706658668, "grad_norm": 6.73430061340332, "learning_rate": 9.251960828385997e-06, "logp/chosen": -308.0, "logp/rejected": -332.0, "loss": 0.70025634765625, "reward/accuracy": 0.6875, "reward/chosen": -1.5546875, "reward/margin": 0.7890625, "reward/rejected": -2.34375, "step": 589 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 540.0, "epoch": 0.17696460707858427, "grad_norm": 5.725857734680176, "learning_rate": 9.249480016344733e-06, "logp/chosen": -350.0, "logp/rejected": -426.0, "loss": 0.3928375244140625, "reward/accuracy": 0.75, "reward/chosen": -1.3046875, "reward/margin": 1.4453125, "reward/rejected": -2.75, "step": 590 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 844.0, "epoch": 0.17726454709058187, "grad_norm": 5.835639953613281, "learning_rate": 9.24699543115136e-06, "logp/chosen": -388.0, "logp/rejected": -308.0, "loss": 0.5653839111328125, "reward/accuracy": 0.75, "reward/chosen": -1.3828125, "reward/margin": 2.265625, "reward/rejected": -3.640625, "step": 591 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 592.0, "epoch": 0.1775644871025795, "grad_norm": 3.95389986038208, "learning_rate": 9.244507075011964e-06, "logp/chosen": -278.0, "logp/rejected": -362.0, "loss": 0.4439697265625, "reward/accuracy": 0.8125, "reward/chosen": -1.2421875, "reward/margin": 1.828125, "reward/rejected": -3.0625, "step": 592 }, { "approx. KL/chosen": 168.0, "approx. KL/rejected": 644.0, "epoch": 0.1778644271145771, "grad_norm": 5.475035667419434, "learning_rate": 9.242014950135977e-06, "logp/chosen": -376.0, "logp/rejected": -406.0, "loss": 0.58917236328125, "reward/accuracy": 0.625, "reward/chosen": -1.296875, "reward/margin": 1.46875, "reward/rejected": -2.765625, "step": 593 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 478.0, "epoch": 0.17816436712657469, "grad_norm": 7.839498043060303, "learning_rate": 9.239519058736182e-06, "logp/chosen": -256.0, "logp/rejected": -308.0, "loss": 0.792724609375, "reward/accuracy": 0.625, "reward/chosen": -1.4453125, "reward/margin": 0.703125, "reward/rejected": -2.15625, "step": 594 }, { "approx. KL/chosen": 111.5, "approx. KL/rejected": 458.0, "epoch": 0.17846430713857228, "grad_norm": 3.148022413253784, "learning_rate": 9.237019403028703e-06, "logp/chosen": -255.0, "logp/rejected": -300.0, "loss": 0.36767578125, "reward/accuracy": 0.875, "reward/chosen": -0.9921875, "reward/margin": 1.4765625, "reward/rejected": -2.46875, "step": 595 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 624.0, "epoch": 0.17876424715056988, "grad_norm": 5.494253635406494, "learning_rate": 9.234515985233007e-06, "logp/chosen": -288.0, "logp/rejected": -272.0, "loss": 0.60723876953125, "reward/accuracy": 0.6875, "reward/chosen": -1.609375, "reward/margin": 1.0625, "reward/rejected": -2.671875, "step": 596 }, { "approx. KL/chosen": 184.0, "approx. KL/rejected": 344.0, "epoch": 0.17906418716256747, "grad_norm": 4.604544639587402, "learning_rate": 9.232008807571902e-06, "logp/chosen": -366.0, "logp/rejected": -426.0, "loss": 0.6070556640625, "reward/accuracy": 0.6875, "reward/chosen": -1.1875, "reward/margin": 0.98828125, "reward/rejected": -2.171875, "step": 597 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 482.0, "epoch": 0.1793641271745651, "grad_norm": 5.895886421203613, "learning_rate": 9.229497872271533e-06, "logp/chosen": -340.0, "logp/rejected": -332.0, "loss": 0.5198974609375, "reward/accuracy": 0.75, "reward/chosen": -1.6640625, "reward/margin": 1.1328125, "reward/rejected": -2.796875, "step": 598 }, { "approx. KL/chosen": 324.0, "approx. KL/rejected": 568.0, "epoch": 0.1796640671865627, "grad_norm": 7.6960954666137695, "learning_rate": 9.226983181561382e-06, "logp/chosen": -516.0, "logp/rejected": -354.0, "loss": 0.79412841796875, "reward/accuracy": 0.5, "reward/chosen": -1.765625, "reward/margin": 0.6953125, "reward/rejected": -2.46875, "step": 599 }, { "approx. KL/chosen": 129.0, "approx. KL/rejected": 480.0, "epoch": 0.1799640071985603, "grad_norm": 4.321328639984131, "learning_rate": 9.224464737674268e-06, "logp/chosen": -402.0, "logp/rejected": -372.0, "loss": 0.5457763671875, "reward/accuracy": 0.6875, "reward/chosen": -1.140625, "reward/margin": 1.34375, "reward/rejected": -2.484375, "step": 600 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 500.0, "epoch": 0.18026394721055788, "grad_norm": 4.520024299621582, "learning_rate": 9.221942542846337e-06, "logp/chosen": -462.0, "logp/rejected": -324.0, "loss": 0.5391845703125, "reward/accuracy": 0.875, "reward/chosen": -1.4375, "reward/margin": 1.1484375, "reward/rejected": -2.578125, "step": 601 }, { "approx. KL/chosen": 252.0, "approx. KL/rejected": 568.0, "epoch": 0.18056388722255548, "grad_norm": 4.173646926879883, "learning_rate": 9.219416599317073e-06, "logp/chosen": -274.0, "logp/rejected": -336.0, "loss": 0.51654052734375, "reward/accuracy": 0.625, "reward/chosen": -1.625, "reward/margin": 1.1484375, "reward/rejected": -2.765625, "step": 602 }, { "approx. KL/chosen": 105.0, "approx. KL/rejected": 224.0, "epoch": 0.18086382723455308, "grad_norm": 4.822140216827393, "learning_rate": 9.21688690932928e-06, "logp/chosen": -426.0, "logp/rejected": -318.0, "loss": 0.703857421875, "reward/accuracy": 0.5, "reward/chosen": -1.203125, "reward/margin": 0.4453125, "reward/rejected": -1.6484375, "step": 603 }, { "approx. KL/chosen": 49.25, "approx. KL/rejected": 246.0, "epoch": 0.1811637672465507, "grad_norm": 3.87782621383667, "learning_rate": 9.214353475129096e-06, "logp/chosen": -368.0, "logp/rejected": -280.0, "loss": 0.51654052734375, "reward/accuracy": 0.625, "reward/chosen": -0.69921875, "reward/margin": 0.953125, "reward/rejected": -1.6484375, "step": 604 }, { "approx. KL/chosen": 127.0, "approx. KL/rejected": 322.0, "epoch": 0.1814637072585483, "grad_norm": 5.580539703369141, "learning_rate": 9.211816298965978e-06, "logp/chosen": -402.0, "logp/rejected": -278.0, "loss": 0.554931640625, "reward/accuracy": 0.75, "reward/chosen": -1.2265625, "reward/margin": 0.8984375, "reward/rejected": -2.125, "step": 605 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 484.0, "epoch": 0.1817636472705459, "grad_norm": 3.9214682579040527, "learning_rate": 9.209275383092712e-06, "logp/chosen": -280.0, "logp/rejected": -350.0, "loss": 0.45330810546875, "reward/accuracy": 0.75, "reward/chosen": -1.2890625, "reward/margin": 1.15625, "reward/rejected": -2.453125, "step": 606 }, { "approx. KL/chosen": 209.0, "approx. KL/rejected": 282.0, "epoch": 0.1820635872825435, "grad_norm": 6.652394771575928, "learning_rate": 9.206730729765396e-06, "logp/chosen": -466.0, "logp/rejected": -354.0, "loss": 0.672607421875, "reward/accuracy": 0.4375, "reward/chosen": -1.765625, "reward/margin": 0.376953125, "reward/rejected": -2.140625, "step": 607 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 430.0, "epoch": 0.18236352729454108, "grad_norm": 3.775057315826416, "learning_rate": 9.204182341243453e-06, "logp/chosen": -412.0, "logp/rejected": -326.0, "loss": 0.4151611328125, "reward/accuracy": 0.8125, "reward/chosen": -1.140625, "reward/margin": 1.265625, "reward/rejected": -2.40625, "step": 608 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 494.0, "epoch": 0.18266346730653868, "grad_norm": 3.3033998012542725, "learning_rate": 9.201630219789619e-06, "logp/chosen": -446.0, "logp/rejected": -408.0, "loss": 0.33233642578125, "reward/accuracy": 0.8125, "reward/chosen": -1.109375, "reward/margin": 1.5625, "reward/rejected": -2.671875, "step": 609 }, { "approx. KL/chosen": 102.0, "approx. KL/rejected": 400.0, "epoch": 0.1829634073185363, "grad_norm": 3.1289095878601074, "learning_rate": 9.199074367669947e-06, "logp/chosen": -249.0, "logp/rejected": -276.0, "loss": 0.41790771484375, "reward/accuracy": 0.6875, "reward/chosen": -1.2265625, "reward/margin": 1.328125, "reward/rejected": -2.5625, "step": 610 }, { "approx. KL/chosen": 150.0, "approx. KL/rejected": 392.0, "epoch": 0.1832633473305339, "grad_norm": 4.0170578956604, "learning_rate": 9.196514787153804e-06, "logp/chosen": -366.0, "logp/rejected": -215.0, "loss": 0.5145263671875, "reward/accuracy": 0.6875, "reward/chosen": -1.3359375, "reward/margin": 1.1328125, "reward/rejected": -2.46875, "step": 611 }, { "approx. KL/chosen": 109.5, "approx. KL/rejected": 380.0, "epoch": 0.1835632873425315, "grad_norm": 3.8750345706939697, "learning_rate": 9.193951480513859e-06, "logp/chosen": -232.0, "logp/rejected": -292.0, "loss": 0.3431396484375, "reward/accuracy": 0.8125, "reward/chosen": -1.140625, "reward/margin": 1.4140625, "reward/rejected": -2.5625, "step": 612 }, { "approx. KL/chosen": 137.0, "approx. KL/rejected": 672.0, "epoch": 0.1838632273545291, "grad_norm": 3.564692974090576, "learning_rate": 9.1913844500261e-06, "logp/chosen": -306.0, "logp/rejected": -250.0, "loss": 0.36456298828125, "reward/accuracy": 0.75, "reward/chosen": -1.5234375, "reward/margin": 1.6328125, "reward/rejected": -3.15625, "step": 613 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 506.0, "epoch": 0.1841631673665267, "grad_norm": 4.89831018447876, "learning_rate": 9.188813697969814e-06, "logp/chosen": -350.0, "logp/rejected": -354.0, "loss": 0.5517578125, "reward/accuracy": 0.75, "reward/chosen": -1.5546875, "reward/margin": 1.2421875, "reward/rejected": -2.796875, "step": 614 }, { "approx. KL/chosen": 167.0, "approx. KL/rejected": 516.0, "epoch": 0.18446310737852428, "grad_norm": 3.682974338531494, "learning_rate": 9.186239226627597e-06, "logp/chosen": -284.0, "logp/rejected": -270.0, "loss": 0.4613037109375, "reward/accuracy": 0.75, "reward/chosen": -1.375, "reward/margin": 1.359375, "reward/rejected": -2.734375, "step": 615 }, { "approx. KL/chosen": 174.0, "approx. KL/rejected": 472.0, "epoch": 0.1847630473905219, "grad_norm": 5.615275859832764, "learning_rate": 9.183661038285347e-06, "logp/chosen": -422.0, "logp/rejected": -380.0, "loss": 0.52618408203125, "reward/accuracy": 0.625, "reward/chosen": -1.3046875, "reward/margin": 1.0546875, "reward/rejected": -2.359375, "step": 616 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 728.0, "epoch": 0.1850629874025195, "grad_norm": 3.064850091934204, "learning_rate": 9.181079135232258e-06, "logp/chosen": -372.0, "logp/rejected": -338.0, "loss": 0.3582763671875, "reward/accuracy": 0.8125, "reward/chosen": -1.3671875, "reward/margin": 1.828125, "reward/rejected": -3.1875, "step": 617 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 588.0, "epoch": 0.1853629274145171, "grad_norm": 4.8408050537109375, "learning_rate": 9.178493519760827e-06, "logp/chosen": -460.0, "logp/rejected": -488.0, "loss": 0.517120361328125, "reward/accuracy": 0.625, "reward/chosen": -1.7734375, "reward/margin": 1.2421875, "reward/rejected": -3.015625, "step": 618 }, { "approx. KL/chosen": 89.5, "approx. KL/rejected": 704.0, "epoch": 0.1856628674265147, "grad_norm": 1.8344824314117432, "learning_rate": 9.175904194166845e-06, "logp/chosen": -410.0, "logp/rejected": -356.0, "loss": 0.1514892578125, "reward/accuracy": 1.0, "reward/chosen": -0.71484375, "reward/margin": 2.546875, "reward/rejected": -3.265625, "step": 619 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 374.0, "epoch": 0.1859628074385123, "grad_norm": 6.561540603637695, "learning_rate": 9.173311160749399e-06, "logp/chosen": -229.0, "logp/rejected": -290.0, "loss": 0.83935546875, "reward/accuracy": 0.6875, "reward/chosen": -2.03125, "reward/margin": 0.296875, "reward/rejected": -2.328125, "step": 620 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 552.0, "epoch": 0.18626274745050989, "grad_norm": 5.918765544891357, "learning_rate": 9.170714421810864e-06, "logp/chosen": -296.0, "logp/rejected": -306.0, "loss": 0.5809326171875, "reward/accuracy": 0.75, "reward/chosen": -1.59375, "reward/margin": 1.125, "reward/rejected": -2.71875, "step": 621 }, { "approx. KL/chosen": 222.0, "approx. KL/rejected": 672.0, "epoch": 0.1865626874625075, "grad_norm": 3.928384780883789, "learning_rate": 9.168113979656913e-06, "logp/chosen": -372.0, "logp/rejected": -262.0, "loss": 0.37518310546875, "reward/accuracy": 0.8125, "reward/chosen": -1.5546875, "reward/margin": 1.5546875, "reward/rejected": -3.109375, "step": 622 }, { "approx. KL/chosen": 406.0, "approx. KL/rejected": 596.0, "epoch": 0.1868626274745051, "grad_norm": 8.577937126159668, "learning_rate": 9.165509836596495e-06, "logp/chosen": -434.0, "logp/rejected": -296.0, "loss": 0.5472412109375, "reward/accuracy": 0.8125, "reward/chosen": -2.171875, "reward/margin": 1.0390625, "reward/rejected": -3.21875, "step": 623 }, { "approx. KL/chosen": 378.0, "approx. KL/rejected": 588.0, "epoch": 0.1871625674865027, "grad_norm": 4.3325018882751465, "learning_rate": 9.162901994941862e-06, "logp/chosen": -294.0, "logp/rejected": -262.0, "loss": 0.54937744140625, "reward/accuracy": 0.6875, "reward/chosen": -2.078125, "reward/margin": 1.15625, "reward/rejected": -3.234375, "step": 624 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 652.0, "epoch": 0.1874625074985003, "grad_norm": 3.8314735889434814, "learning_rate": 9.160290457008531e-06, "logp/chosen": -290.0, "logp/rejected": -310.0, "loss": 0.538818359375, "reward/accuracy": 0.75, "reward/chosen": -1.53125, "reward/margin": 1.3671875, "reward/rejected": -2.90625, "step": 625 }, { "approx. KL/chosen": 235.0, "approx. KL/rejected": 960.0, "epoch": 0.1877624475104979, "grad_norm": 6.174846649169922, "learning_rate": 9.157675225115318e-06, "logp/chosen": -340.0, "logp/rejected": -251.0, "loss": 0.3250732421875, "reward/accuracy": 0.875, "reward/chosen": -1.7890625, "reward/margin": 2.328125, "reward/rejected": -4.125, "step": 626 }, { "approx. KL/chosen": 374.0, "approx. KL/rejected": 536.0, "epoch": 0.1880623875224955, "grad_norm": 4.612677574157715, "learning_rate": 9.155056301584307e-06, "logp/chosen": -398.0, "logp/rejected": -302.0, "loss": 0.71551513671875, "reward/accuracy": 0.625, "reward/chosen": -2.03125, "reward/margin": 0.73828125, "reward/rejected": -2.78125, "step": 627 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 568.0, "epoch": 0.1883623275344931, "grad_norm": 4.308254718780518, "learning_rate": 9.152433688740864e-06, "logp/chosen": -336.0, "logp/rejected": -334.0, "loss": 0.612060546875, "reward/accuracy": 0.8125, "reward/chosen": -1.921875, "reward/margin": 1.234375, "reward/rejected": -3.15625, "step": 628 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 612.0, "epoch": 0.1886622675464907, "grad_norm": 6.152891635894775, "learning_rate": 9.149807388913634e-06, "logp/chosen": -312.0, "logp/rejected": -416.0, "loss": 0.5567626953125, "reward/accuracy": 0.75, "reward/chosen": -1.921875, "reward/margin": 1.1796875, "reward/rejected": -3.109375, "step": 629 }, { "approx. KL/chosen": 364.0, "approx. KL/rejected": 512.0, "epoch": 0.1889622075584883, "grad_norm": 5.908814430236816, "learning_rate": 9.147177404434529e-06, "logp/chosen": -366.0, "logp/rejected": -342.0, "loss": 0.745208740234375, "reward/accuracy": 0.5, "reward/chosen": -2.265625, "reward/margin": 0.64453125, "reward/rejected": -2.90625, "step": 630 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 748.0, "epoch": 0.1892621475704859, "grad_norm": 5.32388973236084, "learning_rate": 9.14454373763874e-06, "logp/chosen": -320.0, "logp/rejected": -342.0, "loss": 0.4345703125, "reward/accuracy": 0.75, "reward/chosen": -1.921875, "reward/margin": 1.25, "reward/rejected": -3.171875, "step": 631 }, { "approx. KL/chosen": 72.5, "approx. KL/rejected": 328.0, "epoch": 0.1895620875824835, "grad_norm": 4.362879276275635, "learning_rate": 9.14190639086472e-06, "logp/chosen": -532.0, "logp/rejected": -430.0, "loss": 0.4249267578125, "reward/accuracy": 0.75, "reward/chosen": -0.92578125, "reward/margin": 1.21875, "reward/rejected": -2.140625, "step": 632 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 916.0, "epoch": 0.1898620275944811, "grad_norm": 5.355600357055664, "learning_rate": 9.139265366454195e-06, "logp/chosen": -280.0, "logp/rejected": -334.0, "loss": 0.525634765625, "reward/accuracy": 0.875, "reward/chosen": -1.71875, "reward/margin": 1.46875, "reward/rejected": -3.1875, "step": 633 }, { "approx. KL/chosen": 146.0, "approx. KL/rejected": 436.0, "epoch": 0.19016196760647872, "grad_norm": 3.685229778289795, "learning_rate": 9.136620666752155e-06, "logp/chosen": -436.0, "logp/rejected": -316.0, "loss": 0.44976806640625, "reward/accuracy": 0.75, "reward/chosen": -1.40625, "reward/margin": 1.09375, "reward/rejected": -2.5, "step": 634 }, { "approx. KL/chosen": 464.0, "approx. KL/rejected": 608.0, "epoch": 0.1904619076184763, "grad_norm": 4.913893222808838, "learning_rate": 9.133972294106852e-06, "logp/chosen": -464.0, "logp/rejected": -316.0, "loss": 0.63067626953125, "reward/accuracy": 0.6875, "reward/chosen": -2.28125, "reward/margin": 0.71875, "reward/rejected": -3.0, "step": 635 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 572.0, "epoch": 0.1907618476304739, "grad_norm": 4.908456325531006, "learning_rate": 9.131320250869802e-06, "logp/chosen": -394.0, "logp/rejected": -444.0, "loss": 0.4659423828125, "reward/accuracy": 0.8125, "reward/chosen": -1.5859375, "reward/margin": 1.34375, "reward/rejected": -2.921875, "step": 636 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 524.0, "epoch": 0.1910617876424715, "grad_norm": 4.304422378540039, "learning_rate": 9.128664539395778e-06, "logp/chosen": -440.0, "logp/rejected": -322.0, "loss": 0.476318359375, "reward/accuracy": 0.8125, "reward/chosen": -1.375, "reward/margin": 1.5625, "reward/rejected": -2.9375, "step": 637 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 516.0, "epoch": 0.1913617276544691, "grad_norm": 4.965632915496826, "learning_rate": 9.126005162042808e-06, "logp/chosen": -284.0, "logp/rejected": -233.0, "loss": 0.4041748046875, "reward/accuracy": 0.75, "reward/chosen": -1.578125, "reward/margin": 1.296875, "reward/rejected": -2.875, "step": 638 }, { "approx. KL/chosen": 147.0, "approx. KL/rejected": 468.0, "epoch": 0.1916616676664667, "grad_norm": 3.9134271144866943, "learning_rate": 9.123342121172179e-06, "logp/chosen": -394.0, "logp/rejected": -348.0, "loss": 0.4114990234375, "reward/accuracy": 0.75, "reward/chosen": -1.3515625, "reward/margin": 1.265625, "reward/rejected": -2.625, "step": 639 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 592.0, "epoch": 0.19196160767846432, "grad_norm": 4.391886234283447, "learning_rate": 9.12067541914843e-06, "logp/chosen": -230.0, "logp/rejected": -296.0, "loss": 0.49260711669921875, "reward/accuracy": 0.625, "reward/chosen": -1.3828125, "reward/margin": 1.578125, "reward/rejected": -2.96875, "step": 640 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 366.0, "epoch": 0.19226154769046191, "grad_norm": 3.579380989074707, "learning_rate": 9.11800505833935e-06, "logp/chosen": -386.0, "logp/rejected": -296.0, "loss": 0.4722900390625, "reward/accuracy": 0.75, "reward/chosen": -1.1953125, "reward/margin": 1.125, "reward/rejected": -2.3125, "step": 641 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 480.0, "epoch": 0.1925614877024595, "grad_norm": 3.7941219806671143, "learning_rate": 9.115331041115977e-06, "logp/chosen": -242.0, "logp/rejected": -286.0, "loss": 0.47119140625, "reward/accuracy": 0.6875, "reward/chosen": -1.40625, "reward/margin": 1.2265625, "reward/rejected": -2.640625, "step": 642 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 532.0, "epoch": 0.1928614277144571, "grad_norm": 4.004888534545898, "learning_rate": 9.112653369852593e-06, "logp/chosen": -346.0, "logp/rejected": -312.0, "loss": 0.56201171875, "reward/accuracy": 0.75, "reward/chosen": -1.671875, "reward/margin": 1.0078125, "reward/rejected": -2.671875, "step": 643 }, { "approx. KL/chosen": 83.5, "approx. KL/rejected": 346.0, "epoch": 0.1931613677264547, "grad_norm": 2.843838930130005, "learning_rate": 9.10997204692673e-06, "logp/chosen": -338.0, "logp/rejected": -219.0, "loss": 0.331787109375, "reward/accuracy": 0.875, "reward/chosen": -0.8984375, "reward/margin": 1.3203125, "reward/rejected": -2.21875, "step": 644 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 332.0, "epoch": 0.1934613077384523, "grad_norm": 4.91260290145874, "learning_rate": 9.107287074719158e-06, "logp/chosen": -414.0, "logp/rejected": -334.0, "loss": 0.51861572265625, "reward/accuracy": 0.6875, "reward/chosen": -1.0078125, "reward/margin": 1.03125, "reward/rejected": -2.03125, "step": 645 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 458.0, "epoch": 0.19376124775044992, "grad_norm": 3.5834596157073975, "learning_rate": 9.104598455613886e-06, "logp/chosen": -406.0, "logp/rejected": -338.0, "loss": 0.4898681640625, "reward/accuracy": 0.75, "reward/chosen": -1.5, "reward/margin": 1.171875, "reward/rejected": -2.671875, "step": 646 }, { "approx. KL/chosen": 111.0, "approx. KL/rejected": 436.0, "epoch": 0.19406118776244752, "grad_norm": 5.223979473114014, "learning_rate": 9.101906191998166e-06, "logp/chosen": -490.0, "logp/rejected": -452.0, "loss": 0.3411712646484375, "reward/accuracy": 0.75, "reward/chosen": -0.69921875, "reward/margin": 1.6328125, "reward/rejected": -2.328125, "step": 647 }, { "approx. KL/chosen": 118.0, "approx. KL/rejected": 260.0, "epoch": 0.1943611277744451, "grad_norm": 4.751780986785889, "learning_rate": 9.099210286262484e-06, "logp/chosen": -338.0, "logp/rejected": -370.0, "loss": 0.52392578125, "reward/accuracy": 0.6875, "reward/chosen": -1.0859375, "reward/margin": 0.78125, "reward/rejected": -1.8671875, "step": 648 }, { "approx. KL/chosen": 92.5, "approx. KL/rejected": 260.0, "epoch": 0.1946610677864427, "grad_norm": 4.052358627319336, "learning_rate": 9.096510740800556e-06, "logp/chosen": -248.0, "logp/rejected": -288.0, "loss": 0.50244140625, "reward/accuracy": 0.6875, "reward/chosen": -1.125, "reward/margin": 0.75390625, "reward/rejected": -1.875, "step": 649 }, { "approx. KL/chosen": 99.0, "approx. KL/rejected": 398.0, "epoch": 0.1949610077984403, "grad_norm": 4.509274005889893, "learning_rate": 9.093807558009336e-06, "logp/chosen": -400.0, "logp/rejected": -394.0, "loss": 0.481201171875, "reward/accuracy": 0.6875, "reward/chosen": -1.0390625, "reward/margin": 1.2109375, "reward/rejected": -2.25, "step": 650 }, { "approx. KL/chosen": 338.0, "approx. KL/rejected": 640.0, "epoch": 0.1952609478104379, "grad_norm": 9.941773414611816, "learning_rate": 9.091100740289005e-06, "logp/chosen": -226.0, "logp/rejected": -278.0, "loss": 1.032684326171875, "reward/accuracy": 0.5625, "reward/chosen": -1.71875, "reward/margin": 1.21875, "reward/rejected": -2.9375, "step": 651 }, { "approx. KL/chosen": 124.5, "approx. KL/rejected": 502.0, "epoch": 0.19556088782243553, "grad_norm": 3.13883376121521, "learning_rate": 9.088390290042969e-06, "logp/chosen": -258.0, "logp/rejected": -320.0, "loss": 0.36627197265625, "reward/accuracy": 0.875, "reward/chosen": -1.171875, "reward/margin": 1.4375, "reward/rejected": -2.609375, "step": 652 }, { "approx. KL/chosen": 107.5, "approx. KL/rejected": 426.0, "epoch": 0.19586082783443312, "grad_norm": 3.881477117538452, "learning_rate": 9.085676209677862e-06, "logp/chosen": -268.0, "logp/rejected": -282.0, "loss": 0.421142578125, "reward/accuracy": 0.8125, "reward/chosen": -1.125, "reward/margin": 1.4375, "reward/rejected": -2.5625, "step": 653 }, { "approx. KL/chosen": 77.5, "approx. KL/rejected": 189.0, "epoch": 0.19616076784643072, "grad_norm": 5.287189960479736, "learning_rate": 9.082958501603543e-06, "logp/chosen": -352.0, "logp/rejected": -340.0, "loss": 0.60491943359375, "reward/accuracy": 0.6875, "reward/chosen": -0.6640625, "reward/margin": 0.69140625, "reward/rejected": -1.359375, "step": 654 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 512.0, "epoch": 0.1964607078584283, "grad_norm": 3.1997010707855225, "learning_rate": 9.080237168233086e-06, "logp/chosen": -280.0, "logp/rejected": -278.0, "loss": 0.34759521484375, "reward/accuracy": 0.9375, "reward/chosen": -1.3359375, "reward/margin": 1.484375, "reward/rejected": -2.8125, "step": 655 }, { "approx. KL/chosen": 63.0, "approx. KL/rejected": 332.0, "epoch": 0.1967606478704259, "grad_norm": 2.6680822372436523, "learning_rate": 9.07751221198279e-06, "logp/chosen": -374.0, "logp/rejected": -346.0, "loss": 0.25048828125, "reward/accuracy": 0.875, "reward/chosen": -0.2421875, "reward/margin": 1.8046875, "reward/rejected": -2.046875, "step": 656 }, { "approx. KL/chosen": 75.0, "approx. KL/rejected": 360.0, "epoch": 0.1970605878824235, "grad_norm": 4.550297737121582, "learning_rate": 9.074783635272171e-06, "logp/chosen": -412.0, "logp/rejected": -366.0, "loss": 0.3656005859375, "reward/accuracy": 0.75, "reward/chosen": -0.5859375, "reward/margin": 1.6953125, "reward/rejected": -2.28125, "step": 657 }, { "approx. KL/chosen": 31.125, "approx. KL/rejected": 420.0, "epoch": 0.19736052789442113, "grad_norm": 2.970229387283325, "learning_rate": 9.072051440523952e-06, "logp/chosen": -320.0, "logp/rejected": -468.0, "loss": 0.3258056640625, "reward/accuracy": 0.8125, "reward/chosen": -0.458984375, "reward/margin": 1.703125, "reward/rejected": -2.171875, "step": 658 }, { "approx. KL/chosen": 248.0, "approx. KL/rejected": 568.0, "epoch": 0.19766046790641872, "grad_norm": 4.239219665527344, "learning_rate": 9.069315630164077e-06, "logp/chosen": -330.0, "logp/rejected": -282.0, "loss": 0.415771484375, "reward/accuracy": 0.75, "reward/chosen": -1.4140625, "reward/margin": 1.5234375, "reward/rejected": -2.9375, "step": 659 }, { "approx. KL/chosen": 126.0, "approx. KL/rejected": 362.0, "epoch": 0.19796040791841632, "grad_norm": 4.408296585083008, "learning_rate": 9.066576206621695e-06, "logp/chosen": -244.0, "logp/rejected": -206.0, "loss": 0.641448974609375, "reward/accuracy": 0.8125, "reward/chosen": -0.9453125, "reward/margin": 1.234375, "reward/rejected": -2.171875, "step": 660 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 418.0, "epoch": 0.19826034793041392, "grad_norm": 4.265423774719238, "learning_rate": 9.063833172329168e-06, "logp/chosen": -227.0, "logp/rejected": -288.0, "loss": 0.53302001953125, "reward/accuracy": 0.75, "reward/chosen": -1.125, "reward/margin": 1.1015625, "reward/rejected": -2.234375, "step": 661 }, { "approx. KL/chosen": 113.5, "approx. KL/rejected": 344.0, "epoch": 0.1985602879424115, "grad_norm": 4.425296306610107, "learning_rate": 9.061086529722061e-06, "logp/chosen": -352.0, "logp/rejected": -266.0, "loss": 0.5293083190917969, "reward/accuracy": 0.8125, "reward/chosen": -0.76171875, "reward/margin": 1.375, "reward/rejected": -2.140625, "step": 662 }, { "approx. KL/chosen": 100.0, "approx. KL/rejected": 508.0, "epoch": 0.1988602279544091, "grad_norm": 3.4379453659057617, "learning_rate": 9.05833628123914e-06, "logp/chosen": -330.0, "logp/rejected": -302.0, "loss": 0.331207275390625, "reward/accuracy": 0.875, "reward/chosen": -0.59375, "reward/margin": 1.9921875, "reward/rejected": -2.59375, "step": 663 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 434.0, "epoch": 0.19916016796640673, "grad_norm": 5.257157802581787, "learning_rate": 9.055582429322376e-06, "logp/chosen": -280.0, "logp/rejected": -334.0, "loss": 0.54278564453125, "reward/accuracy": 0.625, "reward/chosen": -1.4140625, "reward/margin": 1.203125, "reward/rejected": -2.625, "step": 664 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 416.0, "epoch": 0.19946010797840433, "grad_norm": 4.69926118850708, "learning_rate": 9.052824976416942e-06, "logp/chosen": -292.0, "logp/rejected": -378.0, "loss": 0.595458984375, "reward/accuracy": 0.75, "reward/chosen": -1.1171875, "reward/margin": 1.078125, "reward/rejected": -2.1875, "step": 665 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 436.0, "epoch": 0.19976004799040192, "grad_norm": 5.078623294830322, "learning_rate": 9.050063924971204e-06, "logp/chosen": -390.0, "logp/rejected": -310.0, "loss": 0.5559844970703125, "reward/accuracy": 0.6875, "reward/chosen": -1.0234375, "reward/margin": 1.1640625, "reward/rejected": -2.1875, "step": 666 }, { "approx. KL/chosen": 168.0, "approx. KL/rejected": 486.0, "epoch": 0.20005998800239952, "grad_norm": 4.649980545043945, "learning_rate": 9.047299277436726e-06, "logp/chosen": -316.0, "logp/rejected": -400.0, "loss": 0.6328125, "reward/accuracy": 0.75, "reward/chosen": -1.234375, "reward/margin": 1.125, "reward/rejected": -2.359375, "step": 667 }, { "approx. KL/chosen": 97.0, "approx. KL/rejected": 632.0, "epoch": 0.20035992801439712, "grad_norm": 3.835418224334717, "learning_rate": 9.044531036268263e-06, "logp/chosen": -446.0, "logp/rejected": -432.0, "loss": 0.42389965057373047, "reward/accuracy": 0.75, "reward/chosen": -0.78125, "reward/margin": 1.984375, "reward/rejected": -2.765625, "step": 668 }, { "approx. KL/chosen": 76.0, "approx. KL/rejected": 444.0, "epoch": 0.2006598680263947, "grad_norm": 4.3461408615112305, "learning_rate": 9.04175920392376e-06, "logp/chosen": -414.0, "logp/rejected": -350.0, "loss": 0.36273193359375, "reward/accuracy": 0.8125, "reward/chosen": -0.65234375, "reward/margin": 1.7421875, "reward/rejected": -2.390625, "step": 669 }, { "approx. KL/chosen": 45.75, "approx. KL/rejected": 360.0, "epoch": 0.20095980803839233, "grad_norm": 2.790010929107666, "learning_rate": 9.038983782864355e-06, "logp/chosen": -334.0, "logp/rejected": -416.0, "loss": 0.31719970703125, "reward/accuracy": 0.8125, "reward/chosen": -0.474609375, "reward/margin": 1.765625, "reward/rejected": -2.234375, "step": 670 }, { "approx. KL/chosen": 112.5, "approx. KL/rejected": 660.0, "epoch": 0.20125974805038993, "grad_norm": 3.8281965255737305, "learning_rate": 9.036204775554367e-06, "logp/chosen": -274.0, "logp/rejected": -330.0, "loss": 0.40313720703125, "reward/accuracy": 0.75, "reward/chosen": -0.953125, "reward/margin": 2.203125, "reward/rejected": -3.15625, "step": 671 }, { "approx. KL/chosen": 68.5, "approx. KL/rejected": 548.0, "epoch": 0.20155968806238753, "grad_norm": 2.7030625343322754, "learning_rate": 9.033422184461303e-06, "logp/chosen": -368.0, "logp/rejected": -300.0, "loss": 0.2135772705078125, "reward/accuracy": 0.875, "reward/chosen": -0.053466796875, "reward/margin": 2.75, "reward/rejected": -2.796875, "step": 672 }, { "approx. KL/chosen": 76.5, "approx. KL/rejected": 219.0, "epoch": 0.20185962807438512, "grad_norm": 4.471836566925049, "learning_rate": 9.030636012055854e-06, "logp/chosen": -356.0, "logp/rejected": -466.0, "loss": 0.568603515625, "reward/accuracy": 0.6875, "reward/chosen": -0.8984375, "reward/margin": 0.71484375, "reward/rejected": -1.6171875, "step": 673 }, { "approx. KL/chosen": 124.0, "approx. KL/rejected": 342.0, "epoch": 0.20215956808638272, "grad_norm": 4.599592685699463, "learning_rate": 9.027846260811883e-06, "logp/chosen": -360.0, "logp/rejected": -328.0, "loss": 0.5086669921875, "reward/accuracy": 0.6875, "reward/chosen": -0.8125, "reward/margin": 1.0703125, "reward/rejected": -1.8828125, "step": 674 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 932.0, "epoch": 0.20245950809838031, "grad_norm": 5.672457218170166, "learning_rate": 9.025052933206438e-06, "logp/chosen": -245.0, "logp/rejected": -306.0, "loss": 0.625732421875, "reward/accuracy": 0.8125, "reward/chosen": -1.65625, "reward/margin": 1.8515625, "reward/rejected": -3.5, "step": 675 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 628.0, "epoch": 0.20275944811037794, "grad_norm": 4.063258171081543, "learning_rate": 9.02225603171974e-06, "logp/chosen": -422.0, "logp/rejected": -364.0, "loss": 0.4705810546875, "reward/accuracy": 0.75, "reward/chosen": -1.25, "reward/margin": 1.7421875, "reward/rejected": -3.0, "step": 676 }, { "approx. KL/chosen": 149.0, "approx. KL/rejected": 516.0, "epoch": 0.20305938812237553, "grad_norm": 3.645437479019165, "learning_rate": 9.019455558835185e-06, "logp/chosen": -296.0, "logp/rejected": -332.0, "loss": 0.4244384765625, "reward/accuracy": 0.8125, "reward/chosen": -1.3046875, "reward/margin": 1.21875, "reward/rejected": -2.515625, "step": 677 }, { "approx. KL/chosen": 72.5, "approx. KL/rejected": 310.0, "epoch": 0.20335932813437313, "grad_norm": 3.63203763961792, "learning_rate": 9.016651517039337e-06, "logp/chosen": -338.0, "logp/rejected": -243.0, "loss": 0.464599609375, "reward/accuracy": 0.75, "reward/chosen": -0.609375, "reward/margin": 1.328125, "reward/rejected": -1.9375, "step": 678 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 640.0, "epoch": 0.20365926814637073, "grad_norm": 5.749486923217773, "learning_rate": 9.013843908821928e-06, "logp/chosen": -296.0, "logp/rejected": -444.0, "loss": 0.59228515625, "reward/accuracy": 0.625, "reward/chosen": -1.703125, "reward/margin": 1.1484375, "reward/rejected": -2.859375, "step": 679 }, { "approx. KL/chosen": 154.0, "approx. KL/rejected": 404.0, "epoch": 0.20395920815836832, "grad_norm": 4.9689483642578125, "learning_rate": 9.011032736675862e-06, "logp/chosen": -332.0, "logp/rejected": -302.0, "loss": 0.6739501953125, "reward/accuracy": 0.5625, "reward/chosen": -1.03125, "reward/margin": 0.93359375, "reward/rejected": -1.96875, "step": 680 }, { "approx. KL/chosen": 124.5, "approx. KL/rejected": 502.0, "epoch": 0.20425914817036592, "grad_norm": 4.774108409881592, "learning_rate": 9.008218003097204e-06, "logp/chosen": -236.0, "logp/rejected": -258.0, "loss": 0.324127197265625, "reward/accuracy": 0.875, "reward/chosen": -0.8125, "reward/margin": 1.953125, "reward/rejected": -2.765625, "step": 681 }, { "approx. KL/chosen": 62.0, "approx. KL/rejected": 239.0, "epoch": 0.20455908818236354, "grad_norm": 3.610598087310791, "learning_rate": 9.005399710585181e-06, "logp/chosen": -406.0, "logp/rejected": -320.0, "loss": 0.46185302734375, "reward/accuracy": 0.6875, "reward/chosen": -0.7109375, "reward/margin": 0.93359375, "reward/rejected": -1.640625, "step": 682 }, { "approx. KL/chosen": 123.5, "approx. KL/rejected": 512.0, "epoch": 0.20485902819436114, "grad_norm": 3.924010753631592, "learning_rate": 9.002577861642181e-06, "logp/chosen": -354.0, "logp/rejected": -280.0, "loss": 0.32413482666015625, "reward/accuracy": 0.8125, "reward/chosen": -0.5390625, "reward/margin": 2.125, "reward/rejected": -2.671875, "step": 683 }, { "approx. KL/chosen": 84.0, "approx. KL/rejected": 496.0, "epoch": 0.20515896820635873, "grad_norm": 5.041423320770264, "learning_rate": 8.99975245877375e-06, "logp/chosen": -386.0, "logp/rejected": -476.0, "loss": 0.369415283203125, "reward/accuracy": 0.75, "reward/chosen": -0.703125, "reward/margin": 1.75, "reward/rejected": -2.453125, "step": 684 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 680.0, "epoch": 0.20545890821835633, "grad_norm": 5.508944988250732, "learning_rate": 8.996923504488588e-06, "logp/chosen": -442.0, "logp/rejected": -386.0, "loss": 0.43474388122558594, "reward/accuracy": 0.6875, "reward/chosen": -0.796875, "reward/margin": 2.25, "reward/rejected": -3.046875, "step": 685 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 604.0, "epoch": 0.20575884823035392, "grad_norm": 5.382984638214111, "learning_rate": 8.994091001298548e-06, "logp/chosen": -360.0, "logp/rejected": -300.0, "loss": 0.753265380859375, "reward/accuracy": 0.75, "reward/chosen": -1.0703125, "reward/margin": 1.859375, "reward/rejected": -2.9375, "step": 686 }, { "approx. KL/chosen": 246.0, "approx. KL/rejected": 304.0, "epoch": 0.20605878824235152, "grad_norm": 5.5943756103515625, "learning_rate": 8.991254951718638e-06, "logp/chosen": -334.0, "logp/rejected": -282.0, "loss": 0.697784423828125, "reward/accuracy": 0.8125, "reward/chosen": -1.203125, "reward/margin": 0.83203125, "reward/rejected": -2.03125, "step": 687 }, { "approx. KL/chosen": 130.0, "approx. KL/rejected": 500.0, "epoch": 0.20635872825434912, "grad_norm": 3.5575437545776367, "learning_rate": 8.988415358267011e-06, "logp/chosen": -284.0, "logp/rejected": -320.0, "loss": 0.4591064453125, "reward/accuracy": 0.6875, "reward/chosen": -1.25, "reward/margin": 1.3203125, "reward/rejected": -2.578125, "step": 688 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 584.0, "epoch": 0.20665866826634674, "grad_norm": 5.158921718597412, "learning_rate": 8.98557222346497e-06, "logp/chosen": -426.0, "logp/rejected": -540.0, "loss": 0.59906005859375, "reward/accuracy": 0.6875, "reward/chosen": -1.0625, "reward/margin": 1.1953125, "reward/rejected": -2.265625, "step": 689 }, { "approx. KL/chosen": 185.0, "approx. KL/rejected": 736.0, "epoch": 0.20695860827834434, "grad_norm": 4.690436363220215, "learning_rate": 8.982725549836956e-06, "logp/chosen": -494.0, "logp/rejected": -410.0, "loss": 0.458587646484375, "reward/accuracy": 0.8125, "reward/chosen": -1.3046875, "reward/margin": 1.921875, "reward/rejected": -3.234375, "step": 690 }, { "approx. KL/chosen": 115.0, "approx. KL/rejected": 448.0, "epoch": 0.20725854829034193, "grad_norm": 3.6113646030426025, "learning_rate": 8.979875339910563e-06, "logp/chosen": -362.0, "logp/rejected": -464.0, "loss": 0.4150390625, "reward/accuracy": 0.8125, "reward/chosen": -1.078125, "reward/margin": 1.3671875, "reward/rejected": -2.4375, "step": 691 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 868.0, "epoch": 0.20755848830233953, "grad_norm": 3.545523166656494, "learning_rate": 8.977021596216512e-06, "logp/chosen": -247.0, "logp/rejected": -362.0, "loss": 0.31158447265625, "reward/accuracy": 0.875, "reward/chosen": -0.91796875, "reward/margin": 2.84375, "reward/rejected": -3.765625, "step": 692 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 480.0, "epoch": 0.20785842831433712, "grad_norm": 3.9003422260284424, "learning_rate": 8.974164321288671e-06, "logp/chosen": -296.0, "logp/rejected": -312.0, "loss": 0.46826171875, "reward/accuracy": 0.8125, "reward/chosen": -1.640625, "reward/margin": 1.234375, "reward/rejected": -2.875, "step": 693 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 788.0, "epoch": 0.20815836832633472, "grad_norm": 3.55063533782959, "learning_rate": 8.97130351766404e-06, "logp/chosen": -209.0, "logp/rejected": -300.0, "loss": 0.3314971923828125, "reward/accuracy": 0.8125, "reward/chosen": -1.625, "reward/margin": 1.859375, "reward/rejected": -3.484375, "step": 694 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 992.0, "epoch": 0.20845830833833234, "grad_norm": 4.161904811859131, "learning_rate": 8.968439187882754e-06, "logp/chosen": -328.0, "logp/rejected": -390.0, "loss": 0.358642578125, "reward/accuracy": 0.6875, "reward/chosen": -1.859375, "reward/margin": 2.09375, "reward/rejected": -3.953125, "step": 695 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 420.0, "epoch": 0.20875824835032994, "grad_norm": 5.80014181137085, "learning_rate": 8.965571334488076e-06, "logp/chosen": -392.0, "logp/rejected": -402.0, "loss": 0.7890625, "reward/accuracy": 0.5625, "reward/chosen": -1.8515625, "reward/margin": 0.470703125, "reward/rejected": -2.328125, "step": 696 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 580.0, "epoch": 0.20905818836232753, "grad_norm": 4.552361488342285, "learning_rate": 8.962699960026399e-06, "logp/chosen": -480.0, "logp/rejected": -352.0, "loss": 0.470458984375, "reward/accuracy": 0.8125, "reward/chosen": -1.421875, "reward/margin": 1.4609375, "reward/rejected": -2.875, "step": 697 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 596.0, "epoch": 0.20935812837432513, "grad_norm": 3.4799513816833496, "learning_rate": 8.959825067047244e-06, "logp/chosen": -284.0, "logp/rejected": -390.0, "loss": 0.359375, "reward/accuracy": 0.875, "reward/chosen": -1.71875, "reward/margin": 1.4375, "reward/rejected": -3.15625, "step": 698 }, { "approx. KL/chosen": 448.0, "approx. KL/rejected": 804.0, "epoch": 0.20965806838632273, "grad_norm": 6.186742782592773, "learning_rate": 8.956946658103254e-06, "logp/chosen": -372.0, "logp/rejected": -320.0, "loss": 0.621490478515625, "reward/accuracy": 0.75, "reward/chosen": -2.375, "reward/margin": 1.171875, "reward/rejected": -3.546875, "step": 699 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 532.0, "epoch": 0.20995800839832032, "grad_norm": 4.801718235015869, "learning_rate": 8.954064735750193e-06, "logp/chosen": -350.0, "logp/rejected": -354.0, "loss": 0.74884033203125, "reward/accuracy": 0.6875, "reward/chosen": -1.5625, "reward/margin": 0.90625, "reward/rejected": -2.46875, "step": 700 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 620.0, "epoch": 0.21025794841031795, "grad_norm": 2.784597635269165, "learning_rate": 8.95117930254695e-06, "logp/chosen": -286.0, "logp/rejected": -332.0, "loss": 0.33837890625, "reward/accuracy": 0.875, "reward/chosen": -1.4765625, "reward/margin": 1.578125, "reward/rejected": -3.0625, "step": 701 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 664.0, "epoch": 0.21055788842231554, "grad_norm": 3.606605291366577, "learning_rate": 8.948290361055525e-06, "logp/chosen": -264.0, "logp/rejected": -268.0, "loss": 0.44403076171875, "reward/accuracy": 0.6875, "reward/chosen": -1.6875, "reward/margin": 1.390625, "reward/rejected": -3.078125, "step": 702 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 322.0, "epoch": 0.21085782843431314, "grad_norm": 4.198213577270508, "learning_rate": 8.945397913841034e-06, "logp/chosen": -254.0, "logp/rejected": -244.0, "loss": 0.6351318359375, "reward/accuracy": 0.625, "reward/chosen": -1.46875, "reward/margin": 0.70703125, "reward/rejected": -2.171875, "step": 703 }, { "approx. KL/chosen": 127.5, "approx. KL/rejected": 664.0, "epoch": 0.21115776844631073, "grad_norm": 3.3537752628326416, "learning_rate": 8.942501963471711e-06, "logp/chosen": -362.0, "logp/rejected": -398.0, "loss": 0.286956787109375, "reward/accuracy": 0.8125, "reward/chosen": -1.1640625, "reward/margin": 1.9609375, "reward/rejected": -3.125, "step": 704 }, { "approx. KL/chosen": 248.0, "approx. KL/rejected": 620.0, "epoch": 0.21145770845830833, "grad_norm": 7.291372299194336, "learning_rate": 8.939602512518895e-06, "logp/chosen": -388.0, "logp/rejected": -390.0, "loss": 0.63427734375, "reward/accuracy": 0.6875, "reward/chosen": -1.7421875, "reward/margin": 1.25, "reward/rejected": -3.0, "step": 705 }, { "approx. KL/chosen": 212.0, "approx. KL/rejected": 840.0, "epoch": 0.21175764847030593, "grad_norm": 2.725101947784424, "learning_rate": 8.936699563557031e-06, "logp/chosen": -372.0, "logp/rejected": -264.0, "loss": 0.2657623291015625, "reward/accuracy": 0.8125, "reward/chosen": -1.6328125, "reward/margin": 2.0625, "reward/rejected": -3.703125, "step": 706 }, { "approx. KL/chosen": 246.0, "approx. KL/rejected": 564.0, "epoch": 0.21205758848230355, "grad_norm": 5.372864246368408, "learning_rate": 8.933793119163683e-06, "logp/chosen": -300.0, "logp/rejected": -330.0, "loss": 0.65283203125, "reward/accuracy": 0.5625, "reward/chosen": -1.8046875, "reward/margin": 1.0625, "reward/rejected": -2.859375, "step": 707 }, { "approx. KL/chosen": 126.0, "approx. KL/rejected": 560.0, "epoch": 0.21235752849430115, "grad_norm": 3.4618144035339355, "learning_rate": 8.9308831819195e-06, "logp/chosen": -338.0, "logp/rejected": -294.0, "loss": 0.3386077880859375, "reward/accuracy": 0.875, "reward/chosen": -1.0859375, "reward/margin": 1.859375, "reward/rejected": -2.9375, "step": 708 }, { "approx. KL/chosen": 106.5, "approx. KL/rejected": 700.0, "epoch": 0.21265746850629874, "grad_norm": 3.1080527305603027, "learning_rate": 8.92796975440825e-06, "logp/chosen": -200.0, "logp/rejected": -330.0, "loss": 0.324951171875, "reward/accuracy": 0.75, "reward/chosen": -1.0078125, "reward/margin": 1.890625, "reward/rejected": -2.90625, "step": 709 }, { "approx. KL/chosen": 404.0, "approx. KL/rejected": 616.0, "epoch": 0.21295740851829634, "grad_norm": 5.18543815612793, "learning_rate": 8.925052839216785e-06, "logp/chosen": -360.0, "logp/rejected": -394.0, "loss": 0.5860137939453125, "reward/accuracy": 0.5625, "reward/chosen": -2.21875, "reward/margin": 1.03125, "reward/rejected": -3.25, "step": 710 }, { "approx. KL/chosen": 378.0, "approx. KL/rejected": 776.0, "epoch": 0.21325734853029393, "grad_norm": 5.658353805541992, "learning_rate": 8.922132438935063e-06, "logp/chosen": -322.0, "logp/rejected": -430.0, "loss": 0.52783203125, "reward/accuracy": 0.6875, "reward/chosen": -1.8984375, "reward/margin": 1.4375, "reward/rejected": -3.328125, "step": 711 }, { "approx. KL/chosen": 92.0, "approx. KL/rejected": 452.0, "epoch": 0.21355728854229153, "grad_norm": 3.3923888206481934, "learning_rate": 8.919208556156137e-06, "logp/chosen": -278.0, "logp/rejected": -256.0, "loss": 0.28662109375, "reward/accuracy": 0.9375, "reward/chosen": -1.0625, "reward/margin": 1.625, "reward/rejected": -2.6875, "step": 712 }, { "approx. KL/chosen": 86.0, "approx. KL/rejected": 544.0, "epoch": 0.21385722855428915, "grad_norm": 3.367854118347168, "learning_rate": 8.916281193476146e-06, "logp/chosen": -262.0, "logp/rejected": -258.0, "loss": 0.35479736328125, "reward/accuracy": 0.8125, "reward/chosen": -1.015625, "reward/margin": 1.859375, "reward/rejected": -2.875, "step": 713 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 840.0, "epoch": 0.21415716856628675, "grad_norm": 4.25062370300293, "learning_rate": 8.913350353494323e-06, "logp/chosen": -262.0, "logp/rejected": -434.0, "loss": 0.560546875, "reward/accuracy": 0.75, "reward/chosen": -1.3515625, "reward/margin": 1.6015625, "reward/rejected": -2.953125, "step": 714 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 656.0, "epoch": 0.21445710857828434, "grad_norm": 3.9650418758392334, "learning_rate": 8.910416038812988e-06, "logp/chosen": -264.0, "logp/rejected": -262.0, "loss": 0.438018798828125, "reward/accuracy": 0.75, "reward/chosen": -1.6640625, "reward/margin": 1.46875, "reward/rejected": -3.125, "step": 715 }, { "approx. KL/chosen": 254.0, "approx. KL/rejected": 412.0, "epoch": 0.21475704859028194, "grad_norm": 4.785401821136475, "learning_rate": 8.907478252037544e-06, "logp/chosen": -342.0, "logp/rejected": -268.0, "loss": 0.725341796875, "reward/accuracy": 0.6875, "reward/chosen": -1.71875, "reward/margin": 0.68359375, "reward/rejected": -2.390625, "step": 716 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 364.0, "epoch": 0.21505698860227954, "grad_norm": 5.228271484375, "learning_rate": 8.90453699577648e-06, "logp/chosen": -350.0, "logp/rejected": -472.0, "loss": 0.524658203125, "reward/accuracy": 0.6875, "reward/chosen": -1.171875, "reward/margin": 0.8984375, "reward/rejected": -2.078125, "step": 717 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 692.0, "epoch": 0.21535692861427713, "grad_norm": 4.2244553565979, "learning_rate": 8.901592272641367e-06, "logp/chosen": -296.0, "logp/rejected": -380.0, "loss": 0.53106689453125, "reward/accuracy": 0.6875, "reward/chosen": -1.359375, "reward/margin": 1.6328125, "reward/rejected": -2.984375, "step": 718 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 438.0, "epoch": 0.21565686862627476, "grad_norm": 5.179501056671143, "learning_rate": 8.898644085246846e-06, "logp/chosen": -448.0, "logp/rejected": -334.0, "loss": 0.656005859375, "reward/accuracy": 0.75, "reward/chosen": -1.3046875, "reward/margin": 1.1484375, "reward/rejected": -2.453125, "step": 719 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 580.0, "epoch": 0.21595680863827235, "grad_norm": 6.031996250152588, "learning_rate": 8.895692436210643e-06, "logp/chosen": -382.0, "logp/rejected": -356.0, "loss": 0.608184814453125, "reward/accuracy": 0.75, "reward/chosen": -1.3984375, "reward/margin": 1.5546875, "reward/rejected": -2.953125, "step": 720 }, { "approx. KL/chosen": 310.0, "approx. KL/rejected": 792.0, "epoch": 0.21625674865026995, "grad_norm": 4.327134132385254, "learning_rate": 8.892737328153554e-06, "logp/chosen": -352.0, "logp/rejected": -288.0, "loss": 0.30291748046875, "reward/accuracy": 0.875, "reward/chosen": -1.546875, "reward/margin": 1.8515625, "reward/rejected": -3.390625, "step": 721 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 648.0, "epoch": 0.21655668866226754, "grad_norm": 5.646986961364746, "learning_rate": 8.889778763699446e-06, "logp/chosen": -296.0, "logp/rejected": -358.0, "loss": 0.5166015625, "reward/accuracy": 0.5625, "reward/chosen": -1.7421875, "reward/margin": 1.4140625, "reward/rejected": -3.15625, "step": 722 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 544.0, "epoch": 0.21685662867426514, "grad_norm": 4.592745304107666, "learning_rate": 8.886816745475255e-06, "logp/chosen": -298.0, "logp/rejected": -330.0, "loss": 0.625244140625, "reward/accuracy": 0.625, "reward/chosen": -1.9375, "reward/margin": 0.9296875, "reward/rejected": -2.875, "step": 723 }, { "approx. KL/chosen": 188.0, "approx. KL/rejected": 596.0, "epoch": 0.21715656868626274, "grad_norm": 4.171831130981445, "learning_rate": 8.883851276110985e-06, "logp/chosen": -243.0, "logp/rejected": -310.0, "loss": 0.5823974609375, "reward/accuracy": 0.75, "reward/chosen": -1.234375, "reward/margin": 1.421875, "reward/rejected": -2.65625, "step": 724 }, { "approx. KL/chosen": 438.0, "approx. KL/rejected": 728.0, "epoch": 0.21745650869826036, "grad_norm": 5.250747203826904, "learning_rate": 8.880882358239702e-06, "logp/chosen": -446.0, "logp/rejected": -412.0, "loss": 0.39617919921875, "reward/accuracy": 0.8125, "reward/chosen": -2.0625, "reward/margin": 1.421875, "reward/rejected": -3.484375, "step": 725 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 362.0, "epoch": 0.21775644871025795, "grad_norm": 7.286559104919434, "learning_rate": 8.877909994497536e-06, "logp/chosen": -302.0, "logp/rejected": -336.0, "loss": 0.79339599609375, "reward/accuracy": 0.5625, "reward/chosen": -1.75, "reward/margin": 0.578125, "reward/rejected": -2.328125, "step": 726 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 436.0, "epoch": 0.21805638872225555, "grad_norm": 4.166751384735107, "learning_rate": 8.874934187523676e-06, "logp/chosen": -364.0, "logp/rejected": -318.0, "loss": 0.4884033203125, "reward/accuracy": 0.8125, "reward/chosen": -1.4609375, "reward/margin": 1.3359375, "reward/rejected": -2.796875, "step": 727 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 868.0, "epoch": 0.21835632873425315, "grad_norm": 2.846764087677002, "learning_rate": 8.87195493996037e-06, "logp/chosen": -384.0, "logp/rejected": -386.0, "loss": 0.26678466796875, "reward/accuracy": 0.875, "reward/chosen": -1.8046875, "reward/margin": 1.703125, "reward/rejected": -3.515625, "step": 728 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 386.0, "epoch": 0.21865626874625074, "grad_norm": 7.3955230712890625, "learning_rate": 8.868972254452915e-06, "logp/chosen": -262.0, "logp/rejected": -330.0, "loss": 0.596435546875, "reward/accuracy": 0.75, "reward/chosen": -1.34375, "reward/margin": 1.09375, "reward/rejected": -2.4375, "step": 729 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 296.0, "epoch": 0.21895620875824834, "grad_norm": 6.5238823890686035, "learning_rate": 8.865986133649667e-06, "logp/chosen": -380.0, "logp/rejected": -308.0, "loss": 0.76641845703125, "reward/accuracy": 0.625, "reward/chosen": -1.4375, "reward/margin": 0.6328125, "reward/rejected": -2.078125, "step": 730 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 440.0, "epoch": 0.21925614877024596, "grad_norm": 3.34847354888916, "learning_rate": 8.86299658020203e-06, "logp/chosen": -338.0, "logp/rejected": -350.0, "loss": 0.3978271484375, "reward/accuracy": 0.8125, "reward/chosen": -1.125, "reward/margin": 1.515625, "reward/rejected": -2.65625, "step": 731 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 480.0, "epoch": 0.21955608878224356, "grad_norm": 4.56366491317749, "learning_rate": 8.860003596764455e-06, "logp/chosen": -256.0, "logp/rejected": -282.0, "loss": 0.4767303466796875, "reward/accuracy": 0.625, "reward/chosen": -1.296875, "reward/margin": 1.3515625, "reward/rejected": -2.65625, "step": 732 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 748.0, "epoch": 0.21985602879424115, "grad_norm": 5.267603874206543, "learning_rate": 8.857007185994441e-06, "logp/chosen": -462.0, "logp/rejected": -332.0, "loss": 0.30438232421875, "reward/accuracy": 0.875, "reward/chosen": -1.78125, "reward/margin": 1.8125, "reward/rejected": -3.59375, "step": 733 }, { "approx. KL/chosen": 209.0, "approx. KL/rejected": 596.0, "epoch": 0.22015596880623875, "grad_norm": 2.815423011779785, "learning_rate": 8.854007350552526e-06, "logp/chosen": -318.0, "logp/rejected": -282.0, "loss": 0.3257904052734375, "reward/accuracy": 0.875, "reward/chosen": -1.2109375, "reward/margin": 1.859375, "reward/rejected": -3.078125, "step": 734 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 700.0, "epoch": 0.22045590881823635, "grad_norm": 2.90560245513916, "learning_rate": 8.851004093102295e-06, "logp/chosen": -233.0, "logp/rejected": -292.0, "loss": 0.3115081787109375, "reward/accuracy": 0.8125, "reward/chosen": -1.5234375, "reward/margin": 1.890625, "reward/rejected": -3.421875, "step": 735 }, { "approx. KL/chosen": 80.0, "approx. KL/rejected": 496.0, "epoch": 0.22075584883023394, "grad_norm": 3.4031898975372314, "learning_rate": 8.847997416310365e-06, "logp/chosen": -274.0, "logp/rejected": -276.0, "loss": 0.39068603515625, "reward/accuracy": 0.75, "reward/chosen": -0.92578125, "reward/margin": 1.484375, "reward/rejected": -2.40625, "step": 736 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 616.0, "epoch": 0.22105578884223157, "grad_norm": 3.771022319793701, "learning_rate": 8.844987322846393e-06, "logp/chosen": -490.0, "logp/rejected": -470.0, "loss": 0.31756591796875, "reward/accuracy": 0.875, "reward/chosen": -1.34375, "reward/margin": 1.6953125, "reward/rejected": -3.03125, "step": 737 }, { "approx. KL/chosen": 101.5, "approx. KL/rejected": 664.0, "epoch": 0.22135572885422916, "grad_norm": 2.9492406845092773, "learning_rate": 8.841973815383065e-06, "logp/chosen": -306.0, "logp/rejected": -322.0, "loss": 0.235809326171875, "reward/accuracy": 0.9375, "reward/chosen": -1.03125, "reward/margin": 2.1875, "reward/rejected": -3.21875, "step": 738 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 346.0, "epoch": 0.22165566886622676, "grad_norm": 4.97387170791626, "learning_rate": 8.838956896596105e-06, "logp/chosen": -300.0, "logp/rejected": -272.0, "loss": 0.4658203125, "reward/accuracy": 0.8125, "reward/chosen": -1.1796875, "reward/margin": 1.1953125, "reward/rejected": -2.375, "step": 739 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 692.0, "epoch": 0.22195560887822435, "grad_norm": 5.21980094909668, "learning_rate": 8.835936569164263e-06, "logp/chosen": -300.0, "logp/rejected": -438.0, "loss": 0.5080108642578125, "reward/accuracy": 0.6875, "reward/chosen": -1.46875, "reward/margin": 1.609375, "reward/rejected": -3.078125, "step": 740 }, { "approx. KL/chosen": 133.0, "approx. KL/rejected": 438.0, "epoch": 0.22225554889022195, "grad_norm": 4.256836414337158, "learning_rate": 8.832912835769314e-06, "logp/chosen": -424.0, "logp/rejected": -330.0, "loss": 0.449432373046875, "reward/accuracy": 0.75, "reward/chosen": -1.2734375, "reward/margin": 1.2890625, "reward/rejected": -2.5625, "step": 741 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 418.0, "epoch": 0.22255548890221954, "grad_norm": 4.295534133911133, "learning_rate": 8.82988569909606e-06, "logp/chosen": -388.0, "logp/rejected": -382.0, "loss": 0.49774169921875, "reward/accuracy": 0.875, "reward/chosen": -1.21875, "reward/margin": 1.3203125, "reward/rejected": -2.546875, "step": 742 }, { "approx. KL/chosen": 253.0, "approx. KL/rejected": 440.0, "epoch": 0.22285542891421717, "grad_norm": 6.320316314697266, "learning_rate": 8.826855161832321e-06, "logp/chosen": -358.0, "logp/rejected": -230.0, "loss": 0.597900390625, "reward/accuracy": 0.75, "reward/chosen": -1.9375, "reward/margin": 0.8203125, "reward/rejected": -2.75, "step": 743 }, { "approx. KL/chosen": 209.0, "approx. KL/rejected": 660.0, "epoch": 0.22315536892621476, "grad_norm": 5.280271530151367, "learning_rate": 8.823821226668942e-06, "logp/chosen": -482.0, "logp/rejected": -484.0, "loss": 0.45233154296875, "reward/accuracy": 0.8125, "reward/chosen": -1.6484375, "reward/margin": 1.609375, "reward/rejected": -3.265625, "step": 744 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 964.0, "epoch": 0.22345530893821236, "grad_norm": 2.9286489486694336, "learning_rate": 8.820783896299779e-06, "logp/chosen": -232.0, "logp/rejected": -346.0, "loss": 0.2135009765625, "reward/accuracy": 0.875, "reward/chosen": -1.734375, "reward/margin": 2.3125, "reward/rejected": -4.0625, "step": 745 }, { "approx. KL/chosen": 356.0, "approx. KL/rejected": 426.0, "epoch": 0.22375524895020996, "grad_norm": 7.457104206085205, "learning_rate": 8.817743173421705e-06, "logp/chosen": -342.0, "logp/rejected": -304.0, "loss": 0.762115478515625, "reward/accuracy": 0.625, "reward/chosen": -2.1875, "reward/margin": 0.51953125, "reward/rejected": -2.703125, "step": 746 }, { "approx. KL/chosen": 235.0, "approx. KL/rejected": 580.0, "epoch": 0.22405518896220755, "grad_norm": 4.793064594268799, "learning_rate": 8.814699060734608e-06, "logp/chosen": -500.0, "logp/rejected": -370.0, "loss": 0.54669189453125, "reward/accuracy": 0.875, "reward/chosen": -1.390625, "reward/margin": 1.7265625, "reward/rejected": -3.109375, "step": 747 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 520.0, "epoch": 0.22435512897420515, "grad_norm": 4.782389163970947, "learning_rate": 8.811651560941381e-06, "logp/chosen": -270.0, "logp/rejected": -290.0, "loss": 0.50360107421875, "reward/accuracy": 0.625, "reward/chosen": -1.6875, "reward/margin": 1.109375, "reward/rejected": -2.796875, "step": 748 }, { "approx. KL/chosen": 184.0, "approx. KL/rejected": 402.0, "epoch": 0.22465506898620277, "grad_norm": 7.97158145904541, "learning_rate": 8.808600676747928e-06, "logp/chosen": -476.0, "logp/rejected": -412.0, "loss": 0.614990234375, "reward/accuracy": 0.8125, "reward/chosen": -1.3203125, "reward/margin": 1.1171875, "reward/rejected": -2.4375, "step": 749 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 504.0, "epoch": 0.22495500899820037, "grad_norm": 4.762566089630127, "learning_rate": 8.805546410863156e-06, "logp/chosen": -472.0, "logp/rejected": -308.0, "loss": 0.376495361328125, "reward/accuracy": 0.8125, "reward/chosen": -1.1953125, "reward/margin": 1.65625, "reward/rejected": -2.859375, "step": 750 }, { "approx. KL/chosen": 255.0, "approx. KL/rejected": 684.0, "epoch": 0.22525494901019796, "grad_norm": 4.683937072753906, "learning_rate": 8.80248876599898e-06, "logp/chosen": -288.0, "logp/rejected": -404.0, "loss": 0.5947265625, "reward/accuracy": 0.625, "reward/chosen": -1.625, "reward/margin": 1.46875, "reward/rejected": -3.09375, "step": 751 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 664.0, "epoch": 0.22555488902219556, "grad_norm": 8.057100296020508, "learning_rate": 8.799427744870306e-06, "logp/chosen": -394.0, "logp/rejected": -548.0, "loss": 0.6464996337890625, "reward/accuracy": 0.625, "reward/chosen": -1.8671875, "reward/margin": 1.375, "reward/rejected": -3.234375, "step": 752 }, { "approx. KL/chosen": 154.0, "approx. KL/rejected": 394.0, "epoch": 0.22585482903419316, "grad_norm": 4.34982967376709, "learning_rate": 8.796363350195043e-06, "logp/chosen": -268.0, "logp/rejected": -282.0, "loss": 0.584228515625, "reward/accuracy": 0.625, "reward/chosen": -1.4609375, "reward/margin": 0.8046875, "reward/rejected": -2.265625, "step": 753 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 712.0, "epoch": 0.22615476904619075, "grad_norm": 4.263466835021973, "learning_rate": 8.793295584694096e-06, "logp/chosen": -378.0, "logp/rejected": -436.0, "loss": 0.333160400390625, "reward/accuracy": 0.875, "reward/chosen": -2.046875, "reward/margin": 1.46875, "reward/rejected": -3.515625, "step": 754 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 420.0, "epoch": 0.22645470905818837, "grad_norm": 4.258060455322266, "learning_rate": 8.790224451091362e-06, "logp/chosen": -340.0, "logp/rejected": -354.0, "loss": 0.5223388671875, "reward/accuracy": 0.625, "reward/chosen": -1.546875, "reward/margin": 0.84765625, "reward/rejected": -2.390625, "step": 755 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 496.0, "epoch": 0.22675464907018597, "grad_norm": 3.6732754707336426, "learning_rate": 8.787149952113729e-06, "logp/chosen": -310.0, "logp/rejected": -312.0, "loss": 0.48272705078125, "reward/accuracy": 0.8125, "reward/chosen": -1.4609375, "reward/margin": 1.2890625, "reward/rejected": -2.75, "step": 756 }, { "approx. KL/chosen": 226.0, "approx. KL/rejected": 488.0, "epoch": 0.22705458908218357, "grad_norm": 4.219877243041992, "learning_rate": 8.784072090491074e-06, "logp/chosen": -242.0, "logp/rejected": -286.0, "loss": 0.56781005859375, "reward/accuracy": 0.75, "reward/chosen": -1.875, "reward/margin": 0.921875, "reward/rejected": -2.796875, "step": 757 }, { "approx. KL/chosen": 124.5, "approx. KL/rejected": 414.0, "epoch": 0.22735452909418116, "grad_norm": 4.338681221008301, "learning_rate": 8.780990868956254e-06, "logp/chosen": -328.0, "logp/rejected": -404.0, "loss": 0.54205322265625, "reward/accuracy": 0.75, "reward/chosen": -0.96875, "reward/margin": 1.40625, "reward/rejected": -2.375, "step": 758 }, { "approx. KL/chosen": 249.0, "approx. KL/rejected": 812.0, "epoch": 0.22765446910617876, "grad_norm": 4.948858261108398, "learning_rate": 8.777906290245117e-06, "logp/chosen": -346.0, "logp/rejected": -368.0, "loss": 0.8900146484375, "reward/accuracy": 0.625, "reward/chosen": -1.921875, "reward/margin": 1.25, "reward/rejected": -3.171875, "step": 759 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 1208.0, "epoch": 0.22795440911817635, "grad_norm": 2.5660929679870605, "learning_rate": 8.774818357096489e-06, "logp/chosen": -402.0, "logp/rejected": -380.0, "loss": 0.14508056640625, "reward/accuracy": 0.9375, "reward/chosen": -1.2734375, "reward/margin": 3.140625, "reward/rejected": -4.40625, "step": 760 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 664.0, "epoch": 0.22825434913017398, "grad_norm": 4.756117820739746, "learning_rate": 8.771727072252175e-06, "logp/chosen": -350.0, "logp/rejected": -372.0, "loss": 0.4827880859375, "reward/accuracy": 0.75, "reward/chosen": -1.6796875, "reward/margin": 1.53125, "reward/rejected": -3.21875, "step": 761 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 494.0, "epoch": 0.22855428914217157, "grad_norm": 4.5556135177612305, "learning_rate": 8.76863243845695e-06, "logp/chosen": -324.0, "logp/rejected": -336.0, "loss": 0.38232421875, "reward/accuracy": 0.75, "reward/chosen": -1.6796875, "reward/margin": 1.25, "reward/rejected": -2.9375, "step": 762 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 420.0, "epoch": 0.22885422915416917, "grad_norm": 5.130988597869873, "learning_rate": 8.76553445845857e-06, "logp/chosen": -348.0, "logp/rejected": -442.0, "loss": 0.515716552734375, "reward/accuracy": 0.625, "reward/chosen": -1.0546875, "reward/margin": 1.0859375, "reward/rejected": -2.140625, "step": 763 }, { "approx. KL/chosen": 184.0, "approx. KL/rejected": 624.0, "epoch": 0.22915416916616677, "grad_norm": 5.0421929359436035, "learning_rate": 8.762433135007761e-06, "logp/chosen": -334.0, "logp/rejected": -264.0, "loss": 0.59185791015625, "reward/accuracy": 0.6875, "reward/chosen": -1.5546875, "reward/margin": 1.21875, "reward/rejected": -2.78125, "step": 764 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 644.0, "epoch": 0.22945410917816436, "grad_norm": 4.06766939163208, "learning_rate": 8.75932847085822e-06, "logp/chosen": -362.0, "logp/rejected": -312.0, "loss": 0.661865234375, "reward/accuracy": 0.75, "reward/chosen": -2.078125, "reward/margin": 1.0546875, "reward/rejected": -3.125, "step": 765 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 498.0, "epoch": 0.22975404919016196, "grad_norm": 4.971236705780029, "learning_rate": 8.756220468766599e-06, "logp/chosen": -328.0, "logp/rejected": -336.0, "loss": 0.60577392578125, "reward/accuracy": 0.625, "reward/chosen": -1.9765625, "reward/margin": 0.7421875, "reward/rejected": -2.71875, "step": 766 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 980.0, "epoch": 0.23005398920215958, "grad_norm": 3.362745523452759, "learning_rate": 8.753109131492525e-06, "logp/chosen": -312.0, "logp/rejected": -350.0, "loss": 0.2427825927734375, "reward/accuracy": 0.8125, "reward/chosen": -1.59375, "reward/margin": 2.453125, "reward/rejected": -4.0625, "step": 767 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 374.0, "epoch": 0.23035392921415718, "grad_norm": 5.1661152839660645, "learning_rate": 8.749994461798583e-06, "logp/chosen": -278.0, "logp/rejected": -316.0, "loss": 0.671875, "reward/accuracy": 0.5625, "reward/chosen": -1.5625, "reward/margin": 0.6640625, "reward/rejected": -2.234375, "step": 768 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 648.0, "epoch": 0.23065386922615477, "grad_norm": 5.9201202392578125, "learning_rate": 8.746876462450316e-06, "logp/chosen": -354.0, "logp/rejected": -400.0, "loss": 0.458648681640625, "reward/accuracy": 0.6875, "reward/chosen": -1.625, "reward/margin": 1.3828125, "reward/rejected": -3.0, "step": 769 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 384.0, "epoch": 0.23095380923815237, "grad_norm": 4.234123706817627, "learning_rate": 8.743755136216225e-06, "logp/chosen": -488.0, "logp/rejected": -408.0, "loss": 0.59210205078125, "reward/accuracy": 0.75, "reward/chosen": -1.2578125, "reward/margin": 1.0390625, "reward/rejected": -2.296875, "step": 770 }, { "approx. KL/chosen": 127.0, "approx. KL/rejected": 632.0, "epoch": 0.23125374925014996, "grad_norm": 3.036003351211548, "learning_rate": 8.740630485867764e-06, "logp/chosen": -308.0, "logp/rejected": -326.0, "loss": 0.3389892578125, "reward/accuracy": 0.875, "reward/chosen": -1.03125, "reward/margin": 2.046875, "reward/rejected": -3.0625, "step": 771 }, { "approx. KL/chosen": 117.5, "approx. KL/rejected": 620.0, "epoch": 0.23155368926214756, "grad_norm": 2.0331761837005615, "learning_rate": 8.737502514179337e-06, "logp/chosen": -390.0, "logp/rejected": -344.0, "loss": 0.155487060546875, "reward/accuracy": 1.0, "reward/chosen": -1.203125, "reward/margin": 2.125, "reward/rejected": -3.328125, "step": 772 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 350.0, "epoch": 0.23185362927414518, "grad_norm": 4.3891072273254395, "learning_rate": 8.7343712239283e-06, "logp/chosen": -274.0, "logp/rejected": -276.0, "loss": 0.502197265625, "reward/accuracy": 0.6875, "reward/chosen": -1.296875, "reward/margin": 0.984375, "reward/rejected": -2.28125, "step": 773 }, { "approx. KL/chosen": 328.0, "approx. KL/rejected": 332.0, "epoch": 0.23215356928614278, "grad_norm": 8.571084976196289, "learning_rate": 8.731236617894955e-06, "logp/chosen": -292.0, "logp/rejected": -274.0, "loss": 0.8436279296875, "reward/accuracy": 0.6875, "reward/chosen": -1.984375, "reward/margin": 0.31640625, "reward/rejected": -2.296875, "step": 774 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 592.0, "epoch": 0.23245350929814038, "grad_norm": 3.5301601886749268, "learning_rate": 8.728098698862546e-06, "logp/chosen": -460.0, "logp/rejected": -376.0, "loss": 0.2972412109375, "reward/accuracy": 0.8125, "reward/chosen": -1.203125, "reward/margin": 1.71875, "reward/rejected": -2.921875, "step": 775 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 532.0, "epoch": 0.23275344931013797, "grad_norm": 4.4055891036987305, "learning_rate": 8.72495746961726e-06, "logp/chosen": -354.0, "logp/rejected": -348.0, "loss": 0.603179931640625, "reward/accuracy": 0.6875, "reward/chosen": -1.6484375, "reward/margin": 0.91796875, "reward/rejected": -2.5625, "step": 776 }, { "approx. KL/chosen": 241.0, "approx. KL/rejected": 860.0, "epoch": 0.23305338932213557, "grad_norm": 3.7605502605438232, "learning_rate": 8.721812932948224e-06, "logp/chosen": -478.0, "logp/rejected": -412.0, "loss": 0.374542236328125, "reward/accuracy": 0.8125, "reward/chosen": -1.8046875, "reward/margin": 1.8359375, "reward/rejected": -3.640625, "step": 777 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 440.0, "epoch": 0.23335332933413316, "grad_norm": 4.429393291473389, "learning_rate": 8.718665091647499e-06, "logp/chosen": -466.0, "logp/rejected": -392.0, "loss": 0.505401611328125, "reward/accuracy": 0.75, "reward/chosen": -1.53125, "reward/margin": 1.03125, "reward/rejected": -2.5625, "step": 778 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 402.0, "epoch": 0.2336532693461308, "grad_norm": 5.111326217651367, "learning_rate": 8.715513948510083e-06, "logp/chosen": -312.0, "logp/rejected": -316.0, "loss": 0.73895263671875, "reward/accuracy": 0.5625, "reward/chosen": -1.625, "reward/margin": 0.578125, "reward/rejected": -2.203125, "step": 779 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 506.0, "epoch": 0.23395320935812838, "grad_norm": 4.3604736328125, "learning_rate": 8.712359506333908e-06, "logp/chosen": -208.0, "logp/rejected": -247.0, "loss": 0.62457275390625, "reward/accuracy": 0.6875, "reward/chosen": -1.84375, "reward/margin": 0.84375, "reward/rejected": -2.6875, "step": 780 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 524.0, "epoch": 0.23425314937012598, "grad_norm": 5.006486415863037, "learning_rate": 8.709201767919827e-06, "logp/chosen": -217.0, "logp/rejected": -230.0, "loss": 0.614959716796875, "reward/accuracy": 0.5625, "reward/chosen": -1.8984375, "reward/margin": 0.9609375, "reward/rejected": -2.859375, "step": 781 }, { "approx. KL/chosen": 362.0, "approx. KL/rejected": 856.0, "epoch": 0.23455308938212358, "grad_norm": 3.652580499649048, "learning_rate": 8.706040736071625e-06, "logp/chosen": -338.0, "logp/rejected": -340.0, "loss": 0.412445068359375, "reward/accuracy": 0.75, "reward/chosen": -2.203125, "reward/margin": 1.3984375, "reward/rejected": -3.59375, "step": 782 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 490.0, "epoch": 0.23485302939412117, "grad_norm": 6.3889312744140625, "learning_rate": 8.702876413596014e-06, "logp/chosen": -320.0, "logp/rejected": -352.0, "loss": 0.7667999267578125, "reward/accuracy": 0.5625, "reward/chosen": -2.046875, "reward/margin": 0.55859375, "reward/rejected": -2.609375, "step": 783 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 508.0, "epoch": 0.23515296940611877, "grad_norm": 6.006346702575684, "learning_rate": 8.699708803302624e-06, "logp/chosen": -276.0, "logp/rejected": -348.0, "loss": 0.5452880859375, "reward/accuracy": 0.5625, "reward/chosen": -1.3828125, "reward/margin": 1.34375, "reward/rejected": -2.71875, "step": 784 }, { "approx. KL/chosen": 72.5, "approx. KL/rejected": 274.0, "epoch": 0.23545290941811636, "grad_norm": 2.700594663619995, "learning_rate": 8.696537908004005e-06, "logp/chosen": -322.0, "logp/rejected": -330.0, "loss": 0.372314453125, "reward/accuracy": 0.8125, "reward/chosen": -0.87890625, "reward/margin": 1.2578125, "reward/rejected": -2.140625, "step": 785 }, { "approx. KL/chosen": 113.5, "approx. KL/rejected": 524.0, "epoch": 0.235752849430114, "grad_norm": 3.09871244430542, "learning_rate": 8.693363730515625e-06, "logp/chosen": -253.0, "logp/rejected": -308.0, "loss": 0.32745361328125, "reward/accuracy": 0.875, "reward/chosen": -1.234375, "reward/margin": 1.703125, "reward/rejected": -2.9375, "step": 786 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 544.0, "epoch": 0.23605278944211158, "grad_norm": 4.935006141662598, "learning_rate": 8.690186273655861e-06, "logp/chosen": -380.0, "logp/rejected": -330.0, "loss": 0.353271484375, "reward/accuracy": 0.8125, "reward/chosen": -1.3203125, "reward/margin": 1.765625, "reward/rejected": -3.09375, "step": 787 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 336.0, "epoch": 0.23635272945410918, "grad_norm": 3.9505274295806885, "learning_rate": 8.68700554024601e-06, "logp/chosen": -422.0, "logp/rejected": -328.0, "loss": 0.42083740234375, "reward/accuracy": 0.875, "reward/chosen": -1.328125, "reward/margin": 1.09375, "reward/rejected": -2.421875, "step": 788 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 410.0, "epoch": 0.23665266946610677, "grad_norm": 4.058772563934326, "learning_rate": 8.683821533110274e-06, "logp/chosen": -272.0, "logp/rejected": -276.0, "loss": 0.4522705078125, "reward/accuracy": 0.75, "reward/chosen": -1.5546875, "reward/margin": 0.984375, "reward/rejected": -2.53125, "step": 789 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 462.0, "epoch": 0.23695260947810437, "grad_norm": 4.677639961242676, "learning_rate": 8.68063425507576e-06, "logp/chosen": -298.0, "logp/rejected": -378.0, "loss": 0.68505859375, "reward/accuracy": 0.625, "reward/chosen": -1.828125, "reward/margin": 0.66015625, "reward/rejected": -2.484375, "step": 790 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 426.0, "epoch": 0.23725254949010197, "grad_norm": 5.119002342224121, "learning_rate": 8.677443708972481e-06, "logp/chosen": -588.0, "logp/rejected": -548.0, "loss": 0.6419677734375, "reward/accuracy": 0.5625, "reward/chosen": -1.7578125, "reward/margin": 0.8046875, "reward/rejected": -2.5625, "step": 791 }, { "approx. KL/chosen": 218.0, "approx. KL/rejected": 688.0, "epoch": 0.2375524895020996, "grad_norm": 5.162971496582031, "learning_rate": 8.674249897633355e-06, "logp/chosen": -274.0, "logp/rejected": -296.0, "loss": 0.54608154296875, "reward/accuracy": 0.8125, "reward/chosen": -1.7109375, "reward/margin": 1.5390625, "reward/rejected": -3.25, "step": 792 }, { "approx. KL/chosen": 213.0, "approx. KL/rejected": 360.0, "epoch": 0.23785242951409719, "grad_norm": 4.659543037414551, "learning_rate": 8.671052823894195e-06, "logp/chosen": -404.0, "logp/rejected": -410.0, "loss": 0.5257568359375, "reward/accuracy": 0.6875, "reward/chosen": -1.515625, "reward/margin": 0.89453125, "reward/rejected": -2.40625, "step": 793 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 632.0, "epoch": 0.23815236952609478, "grad_norm": 2.5582191944122314, "learning_rate": 8.667852490593712e-06, "logp/chosen": -368.0, "logp/rejected": -300.0, "loss": 0.27471923828125, "reward/accuracy": 0.9375, "reward/chosen": -1.5546875, "reward/margin": 1.6953125, "reward/rejected": -3.25, "step": 794 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 640.0, "epoch": 0.23845230953809238, "grad_norm": 4.055057048797607, "learning_rate": 8.66464890057351e-06, "logp/chosen": -532.0, "logp/rejected": -374.0, "loss": 0.38146209716796875, "reward/accuracy": 0.8125, "reward/chosen": -1.3046875, "reward/margin": 1.9375, "reward/rejected": -3.25, "step": 795 }, { "approx. KL/chosen": 142.0, "approx. KL/rejected": 856.0, "epoch": 0.23875224955008997, "grad_norm": 3.7959396839141846, "learning_rate": 8.661442056678084e-06, "logp/chosen": -258.0, "logp/rejected": -358.0, "loss": 0.3933982849121094, "reward/accuracy": 0.75, "reward/chosen": -1.3359375, "reward/margin": 2.125, "reward/rejected": -3.46875, "step": 796 }, { "approx. KL/chosen": 247.0, "approx. KL/rejected": 604.0, "epoch": 0.23905218956208757, "grad_norm": 5.032351493835449, "learning_rate": 8.658231961754825e-06, "logp/chosen": -360.0, "logp/rejected": -274.0, "loss": 0.494384765625, "reward/accuracy": 0.625, "reward/chosen": -2.09375, "reward/margin": 1.03125, "reward/rejected": -3.125, "step": 797 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 478.0, "epoch": 0.2393521295740852, "grad_norm": 3.9943394660949707, "learning_rate": 8.655018618654003e-06, "logp/chosen": -249.0, "logp/rejected": -290.0, "loss": 0.43310546875, "reward/accuracy": 0.75, "reward/chosen": -1.5078125, "reward/margin": 1.15625, "reward/rejected": -2.65625, "step": 798 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 648.0, "epoch": 0.2396520695860828, "grad_norm": 2.8164093494415283, "learning_rate": 8.651802030228772e-06, "logp/chosen": -322.0, "logp/rejected": -278.0, "loss": 0.362091064453125, "reward/accuracy": 0.8125, "reward/chosen": -1.5390625, "reward/margin": 1.671875, "reward/rejected": -3.203125, "step": 799 }, { "approx. KL/chosen": 251.0, "approx. KL/rejected": 664.0, "epoch": 0.23995200959808038, "grad_norm": 3.7016499042510986, "learning_rate": 8.648582199335177e-06, "logp/chosen": -244.0, "logp/rejected": -338.0, "loss": 0.410614013671875, "reward/accuracy": 0.875, "reward/chosen": -2.0, "reward/margin": 1.390625, "reward/rejected": -3.390625, "step": 800 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 442.0, "epoch": 0.24025194961007798, "grad_norm": 4.6925368309021, "learning_rate": 8.645359128832129e-06, "logp/chosen": -350.0, "logp/rejected": -354.0, "loss": 0.64569091796875, "reward/accuracy": 0.6875, "reward/chosen": -1.765625, "reward/margin": 0.796875, "reward/rejected": -2.5625, "step": 801 }, { "approx. KL/chosen": 308.0, "approx. KL/rejected": 592.0, "epoch": 0.24055188962207558, "grad_norm": 5.504211902618408, "learning_rate": 8.642132821581426e-06, "logp/chosen": -328.0, "logp/rejected": -392.0, "loss": 0.634063720703125, "reward/accuracy": 0.5625, "reward/chosen": -2.03125, "reward/margin": 0.9609375, "reward/rejected": -2.984375, "step": 802 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 1200.0, "epoch": 0.24085182963407317, "grad_norm": 2.0695154666900635, "learning_rate": 8.638903280447733e-06, "logp/chosen": -392.0, "logp/rejected": -268.0, "loss": 0.1745758056640625, "reward/accuracy": 0.9375, "reward/chosen": -1.3359375, "reward/margin": 3.0, "reward/rejected": -4.34375, "step": 803 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 968.0, "epoch": 0.2411517696460708, "grad_norm": 2.4251232147216797, "learning_rate": 8.635670508298589e-06, "logp/chosen": -324.0, "logp/rejected": -430.0, "loss": 0.2184600830078125, "reward/accuracy": 1.0, "reward/chosen": -1.984375, "reward/margin": 2.15625, "reward/rejected": -4.15625, "step": 804 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 676.0, "epoch": 0.2414517096580684, "grad_norm": 3.2364017963409424, "learning_rate": 8.632434508004402e-06, "logp/chosen": -208.0, "logp/rejected": -288.0, "loss": 0.31243896484375, "reward/accuracy": 0.9375, "reward/chosen": -1.484375, "reward/margin": 1.6171875, "reward/rejected": -3.09375, "step": 805 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 680.0, "epoch": 0.241751649670066, "grad_norm": 4.910468578338623, "learning_rate": 8.629195282438447e-06, "logp/chosen": -410.0, "logp/rejected": -344.0, "loss": 0.4405059814453125, "reward/accuracy": 0.8125, "reward/chosen": -1.3203125, "reward/margin": 1.9375, "reward/rejected": -3.265625, "step": 806 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 1096.0, "epoch": 0.24205158968206358, "grad_norm": 3.7104735374450684, "learning_rate": 8.625952834476863e-06, "logp/chosen": -364.0, "logp/rejected": -348.0, "loss": 0.4298095703125, "reward/accuracy": 0.75, "reward/chosen": -1.65625, "reward/margin": 2.03125, "reward/rejected": -3.6875, "step": 807 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 364.0, "epoch": 0.24235152969406118, "grad_norm": 7.26042366027832, "learning_rate": 8.622707166998648e-06, "logp/chosen": -310.0, "logp/rejected": -330.0, "loss": 0.9178466796875, "reward/accuracy": 0.375, "reward/chosen": -2.0625, "reward/margin": 0.310546875, "reward/rejected": -2.375, "step": 808 }, { "approx. KL/chosen": 360.0, "approx. KL/rejected": 1040.0, "epoch": 0.24265146970605878, "grad_norm": 3.70424222946167, "learning_rate": 8.619458282885658e-06, "logp/chosen": -494.0, "logp/rejected": -366.0, "loss": 0.332763671875, "reward/accuracy": 0.875, "reward/chosen": -2.28125, "reward/margin": 1.921875, "reward/rejected": -4.1875, "step": 809 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 776.0, "epoch": 0.2429514097180564, "grad_norm": 4.341169834136963, "learning_rate": 8.616206185022608e-06, "logp/chosen": -398.0, "logp/rejected": -398.0, "loss": 0.3637847900390625, "reward/accuracy": 0.875, "reward/chosen": -1.84375, "reward/margin": 1.7421875, "reward/rejected": -3.578125, "step": 810 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 484.0, "epoch": 0.243251349730054, "grad_norm": 4.880641937255859, "learning_rate": 8.612950876297068e-06, "logp/chosen": -440.0, "logp/rejected": -324.0, "loss": 0.6502685546875, "reward/accuracy": 0.6875, "reward/chosen": -2.078125, "reward/margin": 0.65234375, "reward/rejected": -2.71875, "step": 811 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 892.0, "epoch": 0.2435512897420516, "grad_norm": 6.825384140014648, "learning_rate": 8.609692359599455e-06, "logp/chosen": -340.0, "logp/rejected": -318.0, "loss": 0.7275238037109375, "reward/accuracy": 0.625, "reward/chosen": -1.984375, "reward/margin": 1.46875, "reward/rejected": -3.453125, "step": 812 }, { "approx. KL/chosen": 624.0, "approx. KL/rejected": 588.0, "epoch": 0.2438512297540492, "grad_norm": 9.240335464477539, "learning_rate": 8.606430637823033e-06, "logp/chosen": -470.0, "logp/rejected": -270.0, "loss": 0.7481689453125, "reward/accuracy": 0.5, "reward/chosen": -2.8125, "reward/margin": 0.23828125, "reward/rejected": -3.046875, "step": 813 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 724.0, "epoch": 0.24415116976604678, "grad_norm": 5.5843915939331055, "learning_rate": 8.60316571386392e-06, "logp/chosen": -350.0, "logp/rejected": -414.0, "loss": 0.407318115234375, "reward/accuracy": 0.8125, "reward/chosen": -1.7890625, "reward/margin": 1.7265625, "reward/rejected": -3.515625, "step": 814 }, { "approx. KL/chosen": 174.0, "approx. KL/rejected": 596.0, "epoch": 0.24445110977804438, "grad_norm": 3.8587565422058105, "learning_rate": 8.599897590621065e-06, "logp/chosen": -286.0, "logp/rejected": -274.0, "loss": 0.43017578125, "reward/accuracy": 0.75, "reward/chosen": -1.390625, "reward/margin": 1.6640625, "reward/rejected": -3.046875, "step": 815 }, { "approx. KL/chosen": 246.0, "approx. KL/rejected": 772.0, "epoch": 0.244751049790042, "grad_norm": 4.223028659820557, "learning_rate": 8.596626270996268e-06, "logp/chosen": -328.0, "logp/rejected": -358.0, "loss": 0.42333984375, "reward/accuracy": 0.75, "reward/chosen": -1.5, "reward/margin": 1.6875, "reward/rejected": -3.1875, "step": 816 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 348.0, "epoch": 0.2450509898020396, "grad_norm": 6.289716720581055, "learning_rate": 8.593351757894164e-06, "logp/chosen": -338.0, "logp/rejected": -274.0, "loss": 0.8966064453125, "reward/accuracy": 0.625, "reward/chosen": -1.8359375, "reward/margin": 0.328125, "reward/rejected": -2.15625, "step": 817 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 528.0, "epoch": 0.2453509298140372, "grad_norm": 4.75161075592041, "learning_rate": 8.590074054222224e-06, "logp/chosen": -310.0, "logp/rejected": -308.0, "loss": 0.700439453125, "reward/accuracy": 0.75, "reward/chosen": -1.7421875, "reward/margin": 0.921875, "reward/rejected": -2.671875, "step": 818 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 736.0, "epoch": 0.2456508698260348, "grad_norm": 3.4300243854522705, "learning_rate": 8.586793162890748e-06, "logp/chosen": -330.0, "logp/rejected": -358.0, "loss": 0.385772705078125, "reward/accuracy": 0.8125, "reward/chosen": -1.6328125, "reward/margin": 1.7421875, "reward/rejected": -3.375, "step": 819 }, { "approx. KL/chosen": 246.0, "approx. KL/rejected": 516.0, "epoch": 0.24595080983803239, "grad_norm": 5.239990711212158, "learning_rate": 8.58350908681287e-06, "logp/chosen": -462.0, "logp/rejected": -350.0, "loss": 0.632568359375, "reward/accuracy": 0.6875, "reward/chosen": -1.859375, "reward/margin": 1.015625, "reward/rejected": -2.875, "step": 820 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 560.0, "epoch": 0.24625074985002998, "grad_norm": 4.790522575378418, "learning_rate": 8.580221828904555e-06, "logp/chosen": -330.0, "logp/rejected": -322.0, "loss": 0.4107513427734375, "reward/accuracy": 0.75, "reward/chosen": -1.2734375, "reward/margin": 1.6328125, "reward/rejected": -2.90625, "step": 821 }, { "approx. KL/chosen": 290.0, "approx. KL/rejected": 924.0, "epoch": 0.2465506898620276, "grad_norm": 4.209156513214111, "learning_rate": 8.576931392084586e-06, "logp/chosen": -354.0, "logp/rejected": -386.0, "loss": 0.407958984375, "reward/accuracy": 0.8125, "reward/chosen": -1.8671875, "reward/margin": 1.796875, "reward/rejected": -3.65625, "step": 822 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 268.0, "epoch": 0.2468506298740252, "grad_norm": 5.084692001342773, "learning_rate": 8.57363777927457e-06, "logp/chosen": -220.0, "logp/rejected": -294.0, "loss": 0.711669921875, "reward/accuracy": 0.625, "reward/chosen": -1.6171875, "reward/margin": 0.365234375, "reward/rejected": -1.9765625, "step": 823 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 520.0, "epoch": 0.2471505698860228, "grad_norm": 3.9756343364715576, "learning_rate": 8.57034099339894e-06, "logp/chosen": -284.0, "logp/rejected": -238.0, "loss": 0.324462890625, "reward/accuracy": 0.875, "reward/chosen": -1.546875, "reward/margin": 1.3671875, "reward/rejected": -2.90625, "step": 824 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 1248.0, "epoch": 0.2474505098980204, "grad_norm": 3.529829740524292, "learning_rate": 8.56704103738494e-06, "logp/chosen": -220.0, "logp/rejected": -364.0, "loss": 0.30290985107421875, "reward/accuracy": 0.8125, "reward/chosen": -1.7578125, "reward/margin": 2.546875, "reward/rejected": -4.3125, "step": 825 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 580.0, "epoch": 0.247750449910018, "grad_norm": 3.987102746963501, "learning_rate": 8.563737914162631e-06, "logp/chosen": -322.0, "logp/rejected": -508.0, "loss": 0.387939453125, "reward/accuracy": 0.875, "reward/chosen": -1.4765625, "reward/margin": 1.421875, "reward/rejected": -2.90625, "step": 826 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 636.0, "epoch": 0.24805038992201559, "grad_norm": 3.4497928619384766, "learning_rate": 8.560431626664887e-06, "logp/chosen": -318.0, "logp/rejected": -308.0, "loss": 0.29034423828125, "reward/accuracy": 0.8125, "reward/chosen": -1.53125, "reward/margin": 1.7421875, "reward/rejected": -3.265625, "step": 827 }, { "approx. KL/chosen": 104.0, "approx. KL/rejected": 564.0, "epoch": 0.2483503299340132, "grad_norm": 3.1634151935577393, "learning_rate": 8.557122177827391e-06, "logp/chosen": -318.0, "logp/rejected": -258.0, "loss": 0.3726959228515625, "reward/accuracy": 0.75, "reward/chosen": -1.1484375, "reward/margin": 1.859375, "reward/rejected": -3.015625, "step": 828 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 560.0, "epoch": 0.2486502699460108, "grad_norm": 3.5510687828063965, "learning_rate": 8.553809570588631e-06, "logp/chosen": -356.0, "logp/rejected": -366.0, "loss": 0.26580810546875, "reward/accuracy": 0.875, "reward/chosen": -1.421875, "reward/margin": 1.7890625, "reward/rejected": -3.21875, "step": 829 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 520.0, "epoch": 0.2489502099580084, "grad_norm": 3.7674336433410645, "learning_rate": 8.550493807889903e-06, "logp/chosen": -332.0, "logp/rejected": -350.0, "loss": 0.398223876953125, "reward/accuracy": 0.875, "reward/chosen": -1.6015625, "reward/margin": 1.390625, "reward/rejected": -2.984375, "step": 830 }, { "approx. KL/chosen": 346.0, "approx. KL/rejected": 924.0, "epoch": 0.249250149970006, "grad_norm": 4.948805809020996, "learning_rate": 8.547174892675305e-06, "logp/chosen": -244.0, "logp/rejected": -334.0, "loss": 0.53857421875, "reward/accuracy": 0.625, "reward/chosen": -2.109375, "reward/margin": 1.546875, "reward/rejected": -3.65625, "step": 831 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 474.0, "epoch": 0.2495500899820036, "grad_norm": 8.044894218444824, "learning_rate": 8.543852827891729e-06, "logp/chosen": -396.0, "logp/rejected": -314.0, "loss": 0.59234619140625, "reward/accuracy": 0.625, "reward/chosen": -1.578125, "reward/margin": 0.95703125, "reward/rejected": -2.546875, "step": 832 }, { "approx. KL/chosen": 364.0, "approx. KL/rejected": 900.0, "epoch": 0.2498500299940012, "grad_norm": 5.911198139190674, "learning_rate": 8.540527616488868e-06, "logp/chosen": -346.0, "logp/rejected": -310.0, "loss": 0.36468505859375, "reward/accuracy": 0.8125, "reward/chosen": -2.21875, "reward/margin": 1.7734375, "reward/rejected": -3.984375, "step": 833 }, { "approx. KL/chosen": 340.0, "approx. KL/rejected": 660.0, "epoch": 0.2501499700059988, "grad_norm": 6.046184062957764, "learning_rate": 8.537199261419205e-06, "logp/chosen": -390.0, "logp/rejected": -386.0, "loss": 0.50665283203125, "reward/accuracy": 0.875, "reward/chosen": -1.8359375, "reward/margin": 1.5, "reward/rejected": -3.34375, "step": 834 }, { "approx. KL/chosen": 294.0, "approx. KL/rejected": 540.0, "epoch": 0.2504499100179964, "grad_norm": 4.095968723297119, "learning_rate": 8.53386776563802e-06, "logp/chosen": -284.0, "logp/rejected": -392.0, "loss": 0.491943359375, "reward/accuracy": 0.6875, "reward/chosen": -2.078125, "reward/margin": 0.8046875, "reward/rejected": -2.890625, "step": 835 }, { "approx. KL/chosen": 86.0, "approx. KL/rejected": 616.0, "epoch": 0.250749850029994, "grad_norm": 2.999201774597168, "learning_rate": 8.530533132103376e-06, "logp/chosen": -302.0, "logp/rejected": -372.0, "loss": 0.249542236328125, "reward/accuracy": 0.9375, "reward/chosen": -1.0390625, "reward/margin": 1.953125, "reward/rejected": -3.0, "step": 836 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 736.0, "epoch": 0.2510497900419916, "grad_norm": 3.089125394821167, "learning_rate": 8.52719536377613e-06, "logp/chosen": -219.0, "logp/rejected": -240.0, "loss": 0.30267333984375, "reward/accuracy": 0.8125, "reward/chosen": -1.6484375, "reward/margin": 1.921875, "reward/rejected": -3.5625, "step": 837 }, { "approx. KL/chosen": 116.5, "approx. KL/rejected": 506.0, "epoch": 0.2513497300539892, "grad_norm": 3.8498775959014893, "learning_rate": 8.523854463619906e-06, "logp/chosen": -404.0, "logp/rejected": -336.0, "loss": 0.3564453125, "reward/accuracy": 0.75, "reward/chosen": -0.8359375, "reward/margin": 1.8125, "reward/rejected": -2.65625, "step": 838 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 418.0, "epoch": 0.2516496700659868, "grad_norm": 6.340806484222412, "learning_rate": 8.520510434601133e-06, "logp/chosen": -374.0, "logp/rejected": -418.0, "loss": 0.683837890625, "reward/accuracy": 0.625, "reward/chosen": -1.421875, "reward/margin": 0.75, "reward/rejected": -2.171875, "step": 839 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 660.0, "epoch": 0.2519496100779844, "grad_norm": 4.263360500335693, "learning_rate": 8.517163279688996e-06, "logp/chosen": -358.0, "logp/rejected": -330.0, "loss": 0.3494873046875, "reward/accuracy": 0.75, "reward/chosen": -1.3515625, "reward/margin": 1.8046875, "reward/rejected": -3.15625, "step": 840 }, { "approx. KL/chosen": 125.0, "approx. KL/rejected": 536.0, "epoch": 0.252249550089982, "grad_norm": 4.0445146560668945, "learning_rate": 8.513813001855468e-06, "logp/chosen": -412.0, "logp/rejected": -376.0, "loss": 0.36572265625, "reward/accuracy": 0.8125, "reward/chosen": -1.2578125, "reward/margin": 1.671875, "reward/rejected": -2.9375, "step": 841 }, { "approx. KL/chosen": 422.0, "approx. KL/rejected": 820.0, "epoch": 0.2525494901019796, "grad_norm": 8.592973709106445, "learning_rate": 8.510459604075289e-06, "logp/chosen": -416.0, "logp/rejected": -336.0, "loss": 0.858795166015625, "reward/accuracy": 0.6875, "reward/chosen": -1.71875, "reward/margin": 1.5078125, "reward/rejected": -3.21875, "step": 842 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 972.0, "epoch": 0.2528494301139772, "grad_norm": 3.487729787826538, "learning_rate": 8.507103089325974e-06, "logp/chosen": -306.0, "logp/rejected": -320.0, "loss": 0.31296539306640625, "reward/accuracy": 0.8125, "reward/chosen": -1.078125, "reward/margin": 2.546875, "reward/rejected": -3.625, "step": 843 }, { "approx. KL/chosen": 520.0, "approx. KL/rejected": 868.0, "epoch": 0.2531493701259748, "grad_norm": 4.659121990203857, "learning_rate": 8.503743460587803e-06, "logp/chosen": -306.0, "logp/rejected": -288.0, "loss": 0.505218505859375, "reward/accuracy": 0.75, "reward/chosen": -2.921875, "reward/margin": 0.9921875, "reward/rejected": -3.921875, "step": 844 }, { "approx. KL/chosen": 232.0, "approx. KL/rejected": 1352.0, "epoch": 0.2534493101379724, "grad_norm": 2.4179718494415283, "learning_rate": 8.500380720843818e-06, "logp/chosen": -243.0, "logp/rejected": -436.0, "loss": 0.17531585693359375, "reward/accuracy": 1.0, "reward/chosen": -1.546875, "reward/margin": 3.15625, "reward/rejected": -4.71875, "step": 845 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 716.0, "epoch": 0.25374925014997, "grad_norm": 4.381037712097168, "learning_rate": 8.497014873079829e-06, "logp/chosen": -217.0, "logp/rejected": -262.0, "loss": 0.54229736328125, "reward/accuracy": 0.8125, "reward/chosen": -1.7578125, "reward/margin": 1.75, "reward/rejected": -3.5, "step": 846 }, { "approx. KL/chosen": 476.0, "approx. KL/rejected": 892.0, "epoch": 0.2540491901619676, "grad_norm": 5.668330192565918, "learning_rate": 8.493645920284401e-06, "logp/chosen": -402.0, "logp/rejected": -424.0, "loss": 0.5454864501953125, "reward/accuracy": 0.6875, "reward/chosen": -2.09375, "reward/margin": 1.4296875, "reward/rejected": -3.515625, "step": 847 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 712.0, "epoch": 0.2543491301739652, "grad_norm": 5.229055404663086, "learning_rate": 8.490273865448862e-06, "logp/chosen": -376.0, "logp/rejected": -414.0, "loss": 0.546539306640625, "reward/accuracy": 0.6875, "reward/chosen": -1.453125, "reward/margin": 1.6328125, "reward/rejected": -3.09375, "step": 848 }, { "approx. KL/chosen": 237.0, "approx. KL/rejected": 820.0, "epoch": 0.25464907018596283, "grad_norm": 3.8682398796081543, "learning_rate": 8.486898711567284e-06, "logp/chosen": -416.0, "logp/rejected": -404.0, "loss": 0.36368560791015625, "reward/accuracy": 0.75, "reward/chosen": -1.71875, "reward/margin": 1.875, "reward/rejected": -3.59375, "step": 849 }, { "approx. KL/chosen": 412.0, "approx. KL/rejected": 1208.0, "epoch": 0.2549490101979604, "grad_norm": 12.815052032470703, "learning_rate": 8.483520461636503e-06, "logp/chosen": -482.0, "logp/rejected": -438.0, "loss": 0.45531463623046875, "reward/accuracy": 0.8125, "reward/chosen": -1.9296875, "reward/margin": 2.40625, "reward/rejected": -4.34375, "step": 850 }, { "approx. KL/chosen": 444.0, "approx. KL/rejected": 700.0, "epoch": 0.255248950209958, "grad_norm": 5.647017955780029, "learning_rate": 8.480139118656094e-06, "logp/chosen": -282.0, "logp/rejected": -306.0, "loss": 0.47015380859375, "reward/accuracy": 0.75, "reward/chosen": -2.484375, "reward/margin": 0.98828125, "reward/rejected": -3.46875, "step": 851 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 900.0, "epoch": 0.2555488902219556, "grad_norm": 5.762203216552734, "learning_rate": 8.476754685628384e-06, "logp/chosen": -362.0, "logp/rejected": -229.0, "loss": 0.5998725891113281, "reward/accuracy": 0.6875, "reward/chosen": -1.6484375, "reward/margin": 1.8125, "reward/rejected": -3.46875, "step": 852 }, { "approx. KL/chosen": 154.0, "approx. KL/rejected": 856.0, "epoch": 0.2558488302339532, "grad_norm": 2.0697009563446045, "learning_rate": 8.473367165558443e-06, "logp/chosen": -237.0, "logp/rejected": -278.0, "loss": 0.24853515625, "reward/accuracy": 0.8125, "reward/chosen": -1.359375, "reward/margin": 2.375, "reward/rejected": -3.71875, "step": 853 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 1096.0, "epoch": 0.2561487702459508, "grad_norm": 2.8579628467559814, "learning_rate": 8.46997656145408e-06, "logp/chosen": -225.0, "logp/rejected": -328.0, "loss": 0.18772125244140625, "reward/accuracy": 0.875, "reward/chosen": -1.640625, "reward/margin": 2.6875, "reward/rejected": -4.34375, "step": 854 }, { "approx. KL/chosen": 123.0, "approx. KL/rejected": 924.0, "epoch": 0.2564487102579484, "grad_norm": 4.330235004425049, "learning_rate": 8.466582876325839e-06, "logp/chosen": -334.0, "logp/rejected": -270.0, "loss": 0.33197021484375, "reward/accuracy": 0.8125, "reward/chosen": -1.109375, "reward/margin": 2.375, "reward/rejected": -3.484375, "step": 855 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 636.0, "epoch": 0.25674865026994603, "grad_norm": 4.351962089538574, "learning_rate": 8.463186113187013e-06, "logp/chosen": -316.0, "logp/rejected": -402.0, "loss": 0.3839111328125, "reward/accuracy": 0.75, "reward/chosen": -1.40625, "reward/margin": 1.6953125, "reward/rejected": -3.109375, "step": 856 }, { "approx. KL/chosen": 466.0, "approx. KL/rejected": 1056.0, "epoch": 0.2570485902819436, "grad_norm": 4.915217876434326, "learning_rate": 8.459786275053609e-06, "logp/chosen": -302.0, "logp/rejected": -462.0, "loss": 0.579132080078125, "reward/accuracy": 0.6875, "reward/chosen": -2.21875, "reward/margin": 1.7734375, "reward/rejected": -4.0, "step": 857 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 420.0, "epoch": 0.2573485302939412, "grad_norm": 7.633713245391846, "learning_rate": 8.456383364944384e-06, "logp/chosen": -332.0, "logp/rejected": -298.0, "loss": 0.8819580078125, "reward/accuracy": 0.5625, "reward/chosen": -1.75, "reward/margin": 0.5234375, "reward/rejected": -2.28125, "step": 858 }, { "approx. KL/chosen": 247.0, "approx. KL/rejected": 904.0, "epoch": 0.2576484703059388, "grad_norm": 4.631514072418213, "learning_rate": 8.452977385880805e-06, "logp/chosen": -348.0, "logp/rejected": -334.0, "loss": 0.373382568359375, "reward/accuracy": 0.8125, "reward/chosen": -1.828125, "reward/margin": 1.9921875, "reward/rejected": -3.8125, "step": 859 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 1368.0, "epoch": 0.2579484103179364, "grad_norm": 3.3160977363586426, "learning_rate": 8.449568340887078e-06, "logp/chosen": -350.0, "logp/rejected": -336.0, "loss": 0.291656494140625, "reward/accuracy": 0.875, "reward/chosen": -2.15625, "reward/margin": 2.515625, "reward/rejected": -4.6875, "step": 860 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 608.0, "epoch": 0.25824835032993404, "grad_norm": 4.883181571960449, "learning_rate": 8.446156232990119e-06, "logp/chosen": -322.0, "logp/rejected": -300.0, "loss": 0.6343994140625, "reward/accuracy": 0.625, "reward/chosen": -1.71875, "reward/margin": 1.09375, "reward/rejected": -2.8125, "step": 861 }, { "approx. KL/chosen": 304.0, "approx. KL/rejected": 596.0, "epoch": 0.2585482903419316, "grad_norm": 5.85305118560791, "learning_rate": 8.442741065219575e-06, "logp/chosen": -306.0, "logp/rejected": -416.0, "loss": 0.84710693359375, "reward/accuracy": 0.625, "reward/chosen": -1.84375, "reward/margin": 0.9453125, "reward/rejected": -2.78125, "step": 862 }, { "approx. KL/chosen": 372.0, "approx. KL/rejected": 764.0, "epoch": 0.25884823035392923, "grad_norm": 6.4658284187316895, "learning_rate": 8.439322840607805e-06, "logp/chosen": -304.0, "logp/rejected": -316.0, "loss": 0.8687744140625, "reward/accuracy": 0.75, "reward/chosen": -1.9375, "reward/margin": 1.3046875, "reward/rejected": -3.25, "step": 863 }, { "approx. KL/chosen": 324.0, "approx. KL/rejected": 1024.0, "epoch": 0.2591481703659268, "grad_norm": 5.244470596313477, "learning_rate": 8.435901562189876e-06, "logp/chosen": -310.0, "logp/rejected": -388.0, "loss": 0.53753662109375, "reward/accuracy": 0.75, "reward/chosen": -2.0, "reward/margin": 1.953125, "reward/rejected": -3.953125, "step": 864 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 768.0, "epoch": 0.2594481103779244, "grad_norm": 5.531984329223633, "learning_rate": 8.432477233003578e-06, "logp/chosen": -346.0, "logp/rejected": -266.0, "loss": 0.6739654541015625, "reward/accuracy": 0.6875, "reward/chosen": -2.046875, "reward/margin": 1.3984375, "reward/rejected": -3.4375, "step": 865 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 788.0, "epoch": 0.259748050389922, "grad_norm": 4.440680980682373, "learning_rate": 8.429049856089402e-06, "logp/chosen": -322.0, "logp/rejected": -322.0, "loss": 0.43030548095703125, "reward/accuracy": 0.8125, "reward/chosen": -1.7421875, "reward/margin": 1.78125, "reward/rejected": -3.515625, "step": 866 }, { "approx. KL/chosen": 252.0, "approx. KL/rejected": 856.0, "epoch": 0.2600479904019196, "grad_norm": 2.992185592651367, "learning_rate": 8.425619434490548e-06, "logp/chosen": -272.0, "logp/rejected": -338.0, "loss": 0.3219451904296875, "reward/accuracy": 0.8125, "reward/chosen": -1.640625, "reward/margin": 1.9296875, "reward/rejected": -3.5625, "step": 867 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 616.0, "epoch": 0.26034793041391724, "grad_norm": 4.4247026443481445, "learning_rate": 8.422185971252917e-06, "logp/chosen": -284.0, "logp/rejected": -364.0, "loss": 0.416473388671875, "reward/accuracy": 0.8125, "reward/chosen": -1.7734375, "reward/margin": 1.4453125, "reward/rejected": -3.21875, "step": 868 }, { "approx. KL/chosen": 225.0, "approx. KL/rejected": 712.0, "epoch": 0.2606478704259148, "grad_norm": 3.825540781021118, "learning_rate": 8.418749469425114e-06, "logp/chosen": -376.0, "logp/rejected": -404.0, "loss": 0.4493408203125, "reward/accuracy": 0.8125, "reward/chosen": -1.390625, "reward/margin": 1.40625, "reward/rejected": -2.796875, "step": 869 }, { "approx. KL/chosen": 346.0, "approx. KL/rejected": 468.0, "epoch": 0.26094781043791243, "grad_norm": 5.126302719116211, "learning_rate": 8.415309932058438e-06, "logp/chosen": -270.0, "logp/rejected": -322.0, "loss": 0.69384765625, "reward/accuracy": 0.625, "reward/chosen": -1.765625, "reward/margin": 0.71484375, "reward/rejected": -2.484375, "step": 870 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 512.0, "epoch": 0.26124775044991, "grad_norm": 3.845933198928833, "learning_rate": 8.411867362206889e-06, "logp/chosen": -356.0, "logp/rejected": -356.0, "loss": 0.4364013671875, "reward/accuracy": 0.6875, "reward/chosen": -1.296875, "reward/margin": 1.484375, "reward/rejected": -2.78125, "step": 871 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 1040.0, "epoch": 0.2615476904619076, "grad_norm": 4.504059314727783, "learning_rate": 8.408421762927154e-06, "logp/chosen": -270.0, "logp/rejected": -330.0, "loss": 0.4850311279296875, "reward/accuracy": 0.75, "reward/chosen": -1.9921875, "reward/margin": 1.9453125, "reward/rejected": -3.9375, "step": 872 }, { "approx. KL/chosen": 177.0, "approx. KL/rejected": 932.0, "epoch": 0.26184763047390525, "grad_norm": 7.129112720489502, "learning_rate": 8.404973137278614e-06, "logp/chosen": -502.0, "logp/rejected": -364.0, "loss": 0.5075225830078125, "reward/accuracy": 0.75, "reward/chosen": -1.546875, "reward/margin": 2.296875, "reward/rejected": -3.828125, "step": 873 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 552.0, "epoch": 0.2621475704859028, "grad_norm": 4.320293426513672, "learning_rate": 8.401521488323334e-06, "logp/chosen": -370.0, "logp/rejected": -378.0, "loss": 0.4488525390625, "reward/accuracy": 0.8125, "reward/chosen": -1.4453125, "reward/margin": 1.4609375, "reward/rejected": -2.90625, "step": 874 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 632.0, "epoch": 0.26244751049790044, "grad_norm": 3.538012981414795, "learning_rate": 8.398066819126063e-06, "logp/chosen": -348.0, "logp/rejected": -294.0, "loss": 0.42046356201171875, "reward/accuracy": 0.75, "reward/chosen": -1.1953125, "reward/margin": 1.9609375, "reward/rejected": -3.15625, "step": 875 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 496.0, "epoch": 0.262747450509898, "grad_norm": 5.4286065101623535, "learning_rate": 8.394609132754236e-06, "logp/chosen": -380.0, "logp/rejected": -378.0, "loss": 0.555908203125, "reward/accuracy": 0.75, "reward/chosen": -1.3671875, "reward/margin": 1.1640625, "reward/rejected": -2.53125, "step": 876 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 584.0, "epoch": 0.26304739052189563, "grad_norm": 6.557967185974121, "learning_rate": 8.391148432277962e-06, "logp/chosen": -268.0, "logp/rejected": -240.0, "loss": 0.6405258178710938, "reward/accuracy": 0.625, "reward/chosen": -1.421875, "reward/margin": 1.265625, "reward/rejected": -2.6875, "step": 877 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 502.0, "epoch": 0.2633473305338932, "grad_norm": 4.454270362854004, "learning_rate": 8.387684720770032e-06, "logp/chosen": -260.0, "logp/rejected": -284.0, "loss": 0.386627197265625, "reward/accuracy": 0.8125, "reward/chosen": -1.28125, "reward/margin": 1.546875, "reward/rejected": -2.828125, "step": 878 }, { "approx. KL/chosen": 66.5, "approx. KL/rejected": 468.0, "epoch": 0.2636472705458908, "grad_norm": 2.481492042541504, "learning_rate": 8.384218001305904e-06, "logp/chosen": -222.0, "logp/rejected": -262.0, "loss": 0.3262939453125, "reward/accuracy": 0.9375, "reward/chosen": -0.9140625, "reward/margin": 1.59375, "reward/rejected": -2.5, "step": 879 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 876.0, "epoch": 0.26394721055788845, "grad_norm": 3.978890895843506, "learning_rate": 8.380748276963708e-06, "logp/chosen": -298.0, "logp/rejected": -320.0, "loss": 0.3737335205078125, "reward/accuracy": 0.8125, "reward/chosen": -1.515625, "reward/margin": 2.015625, "reward/rejected": -3.53125, "step": 880 }, { "approx. KL/chosen": 146.0, "approx. KL/rejected": 414.0, "epoch": 0.264247150569886, "grad_norm": 4.278883457183838, "learning_rate": 8.377275550824248e-06, "logp/chosen": -344.0, "logp/rejected": -340.0, "loss": 0.5113525390625, "reward/accuracy": 0.75, "reward/chosen": -1.34375, "reward/margin": 1.0703125, "reward/rejected": -2.40625, "step": 881 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 556.0, "epoch": 0.26454709058188364, "grad_norm": 5.318771839141846, "learning_rate": 8.37379982597099e-06, "logp/chosen": -314.0, "logp/rejected": -260.0, "loss": 0.5814208984375, "reward/accuracy": 0.6875, "reward/chosen": -1.4375, "reward/margin": 0.984375, "reward/rejected": -2.421875, "step": 882 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 764.0, "epoch": 0.2648470305938812, "grad_norm": 3.9637882709503174, "learning_rate": 8.370321105490058e-06, "logp/chosen": -440.0, "logp/rejected": -332.0, "loss": 0.421417236328125, "reward/accuracy": 0.75, "reward/chosen": -1.3359375, "reward/margin": 1.9453125, "reward/rejected": -3.28125, "step": 883 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 520.0, "epoch": 0.26514697060587883, "grad_norm": 4.018825054168701, "learning_rate": 8.36683939247024e-06, "logp/chosen": -336.0, "logp/rejected": -350.0, "loss": 0.33984375, "reward/accuracy": 0.8125, "reward/chosen": -1.4375, "reward/margin": 1.4453125, "reward/rejected": -2.875, "step": 884 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 520.0, "epoch": 0.2654469106178764, "grad_norm": 5.564828872680664, "learning_rate": 8.363354690002984e-06, "logp/chosen": -280.0, "logp/rejected": -328.0, "loss": 0.598419189453125, "reward/accuracy": 0.75, "reward/chosen": -1.6171875, "reward/margin": 1.203125, "reward/rejected": -2.8125, "step": 885 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 616.0, "epoch": 0.265746850629874, "grad_norm": 4.374059200286865, "learning_rate": 8.359867001182388e-06, "logp/chosen": -320.0, "logp/rejected": -470.0, "loss": 0.46076202392578125, "reward/accuracy": 0.75, "reward/chosen": -1.515625, "reward/margin": 1.4375, "reward/rejected": -2.953125, "step": 886 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 454.0, "epoch": 0.26604679064187164, "grad_norm": 6.261397838592529, "learning_rate": 8.3563763291052e-06, "logp/chosen": -436.0, "logp/rejected": -318.0, "loss": 0.731689453125, "reward/accuracy": 0.5, "reward/chosen": -2.0625, "reward/margin": 0.57421875, "reward/rejected": -2.640625, "step": 887 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 664.0, "epoch": 0.2663467306538692, "grad_norm": 4.487250804901123, "learning_rate": 8.352882676870821e-06, "logp/chosen": -286.0, "logp/rejected": -348.0, "loss": 0.4638671875, "reward/accuracy": 0.875, "reward/chosen": -1.609375, "reward/margin": 1.5078125, "reward/rejected": -3.109375, "step": 888 }, { "approx. KL/chosen": 185.0, "approx. KL/rejected": 416.0, "epoch": 0.26664667066586684, "grad_norm": 4.696677207946777, "learning_rate": 8.3493860475813e-06, "logp/chosen": -350.0, "logp/rejected": -322.0, "loss": 0.7495269775390625, "reward/accuracy": 0.6875, "reward/chosen": -1.171875, "reward/margin": 1.09375, "reward/rejected": -2.265625, "step": 889 }, { "approx. KL/chosen": 123.0, "approx. KL/rejected": 608.0, "epoch": 0.2669466106778644, "grad_norm": 3.5313308238983154, "learning_rate": 8.345886444341322e-06, "logp/chosen": -282.0, "logp/rejected": -282.0, "loss": 0.3702392578125, "reward/accuracy": 0.75, "reward/chosen": -1.1875, "reward/margin": 1.6640625, "reward/rejected": -2.859375, "step": 890 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 608.0, "epoch": 0.26724655068986203, "grad_norm": 2.796610116958618, "learning_rate": 8.34238387025822e-06, "logp/chosen": -223.0, "logp/rejected": -247.0, "loss": 0.262420654296875, "reward/accuracy": 0.9375, "reward/chosen": -1.3359375, "reward/margin": 1.921875, "reward/rejected": -3.25, "step": 891 }, { "approx. KL/chosen": 193.0, "approx. KL/rejected": 848.0, "epoch": 0.26754649070185965, "grad_norm": 4.052690029144287, "learning_rate": 8.338878328441957e-06, "logp/chosen": -368.0, "logp/rejected": -330.0, "loss": 0.38616943359375, "reward/accuracy": 0.8125, "reward/chosen": -1.65625, "reward/margin": 2.0, "reward/rejected": -3.65625, "step": 892 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 632.0, "epoch": 0.2678464307138572, "grad_norm": 6.706121921539307, "learning_rate": 8.335369822005138e-06, "logp/chosen": -368.0, "logp/rejected": -302.0, "loss": 0.5924530029296875, "reward/accuracy": 0.75, "reward/chosen": -1.7890625, "reward/margin": 1.40625, "reward/rejected": -3.203125, "step": 893 }, { "approx. KL/chosen": 245.0, "approx. KL/rejected": 760.0, "epoch": 0.26814637072585484, "grad_norm": 4.300452709197998, "learning_rate": 8.331858354063e-06, "logp/chosen": -322.0, "logp/rejected": -316.0, "loss": 0.45391845703125, "reward/accuracy": 0.6875, "reward/chosen": -1.8125, "reward/margin": 1.4921875, "reward/rejected": -3.3125, "step": 894 }, { "approx. KL/chosen": 254.0, "approx. KL/rejected": 498.0, "epoch": 0.2684463107378524, "grad_norm": 5.070827960968018, "learning_rate": 8.328343927733404e-06, "logp/chosen": -312.0, "logp/rejected": -362.0, "loss": 0.60986328125, "reward/accuracy": 0.625, "reward/chosen": -1.7265625, "reward/margin": 0.9453125, "reward/rejected": -2.671875, "step": 895 }, { "approx. KL/chosen": 146.0, "approx. KL/rejected": 476.0, "epoch": 0.26874625074985004, "grad_norm": 4.604249954223633, "learning_rate": 8.324826546136844e-06, "logp/chosen": -334.0, "logp/rejected": -288.0, "loss": 0.69677734375, "reward/accuracy": 0.5, "reward/chosen": -1.3828125, "reward/margin": 1.125, "reward/rejected": -2.5, "step": 896 }, { "approx. KL/chosen": 63.75, "approx. KL/rejected": 416.0, "epoch": 0.2690461907618476, "grad_norm": 3.201256275177002, "learning_rate": 8.321306212396435e-06, "logp/chosen": -322.0, "logp/rejected": -268.0, "loss": 0.3394775390625, "reward/accuracy": 0.75, "reward/chosen": -0.78515625, "reward/margin": 1.7578125, "reward/rejected": -2.546875, "step": 897 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 624.0, "epoch": 0.2693461307738452, "grad_norm": 3.8641059398651123, "learning_rate": 8.31778292963791e-06, "logp/chosen": -296.0, "logp/rejected": -298.0, "loss": 0.356964111328125, "reward/accuracy": 0.8125, "reward/chosen": -1.5234375, "reward/margin": 1.609375, "reward/rejected": -3.140625, "step": 898 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 740.0, "epoch": 0.26964607078584285, "grad_norm": 2.464050531387329, "learning_rate": 8.314256700989626e-06, "logp/chosen": -248.0, "logp/rejected": -286.0, "loss": 0.19549560546875, "reward/accuracy": 1.0, "reward/chosen": -1.3515625, "reward/margin": 2.0625, "reward/rejected": -3.421875, "step": 899 }, { "approx. KL/chosen": 354.0, "approx. KL/rejected": 872.0, "epoch": 0.2699460107978404, "grad_norm": 3.707273483276367, "learning_rate": 8.310727529582556e-06, "logp/chosen": -310.0, "logp/rejected": -348.0, "loss": 0.43511962890625, "reward/accuracy": 0.875, "reward/chosen": -2.296875, "reward/margin": 1.4296875, "reward/rejected": -3.734375, "step": 900 }, { "approx. KL/chosen": 219.0, "approx. KL/rejected": 462.0, "epoch": 0.27024595080983804, "grad_norm": 4.349726676940918, "learning_rate": 8.307195418550283e-06, "logp/chosen": -326.0, "logp/rejected": -248.0, "loss": 0.58966064453125, "reward/accuracy": 0.6875, "reward/chosen": -1.546875, "reward/margin": 1.1484375, "reward/rejected": -2.6875, "step": 901 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 804.0, "epoch": 0.2705458908218356, "grad_norm": 4.385776996612549, "learning_rate": 8.303660371028995e-06, "logp/chosen": -350.0, "logp/rejected": -260.0, "loss": 0.32672119140625, "reward/accuracy": 0.875, "reward/chosen": -1.9609375, "reward/margin": 1.8046875, "reward/rejected": -3.765625, "step": 902 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 528.0, "epoch": 0.27084583083383323, "grad_norm": 4.841426372528076, "learning_rate": 8.3001223901575e-06, "logp/chosen": -334.0, "logp/rejected": -296.0, "loss": 0.520751953125, "reward/accuracy": 0.6875, "reward/chosen": -1.453125, "reward/margin": 1.4375, "reward/rejected": -2.890625, "step": 903 }, { "approx. KL/chosen": 396.0, "approx. KL/rejected": 680.0, "epoch": 0.27114577084583086, "grad_norm": 5.421963214874268, "learning_rate": 8.2965814790772e-06, "logp/chosen": -394.0, "logp/rejected": -342.0, "loss": 0.642303466796875, "reward/accuracy": 0.75, "reward/chosen": -2.078125, "reward/margin": 1.0546875, "reward/rejected": -3.125, "step": 904 }, { "approx. KL/chosen": 184.0, "approx. KL/rejected": 588.0, "epoch": 0.2714457108578284, "grad_norm": 3.715214490890503, "learning_rate": 8.293037640932102e-06, "logp/chosen": -348.0, "logp/rejected": -310.0, "loss": 0.3858642578125, "reward/accuracy": 0.75, "reward/chosen": -1.4921875, "reward/margin": 1.59375, "reward/rejected": -3.078125, "step": 905 }, { "approx. KL/chosen": 328.0, "approx. KL/rejected": 800.0, "epoch": 0.27174565086982605, "grad_norm": 6.588594436645508, "learning_rate": 8.289490878868816e-06, "logp/chosen": -418.0, "logp/rejected": -376.0, "loss": 0.36480712890625, "reward/accuracy": 0.8125, "reward/chosen": -1.8828125, "reward/margin": 1.8359375, "reward/rejected": -3.71875, "step": 906 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 688.0, "epoch": 0.2720455908818236, "grad_norm": 3.2039999961853027, "learning_rate": 8.285941196036538e-06, "logp/chosen": -306.0, "logp/rejected": -390.0, "loss": 0.3822021484375, "reward/accuracy": 0.875, "reward/chosen": -1.75, "reward/margin": 1.5625, "reward/rejected": -3.3125, "step": 907 }, { "approx. KL/chosen": 350.0, "approx. KL/rejected": 680.0, "epoch": 0.27234553089382124, "grad_norm": 5.773646831512451, "learning_rate": 8.28238859558707e-06, "logp/chosen": -248.0, "logp/rejected": -352.0, "loss": 0.58465576171875, "reward/accuracy": 0.75, "reward/chosen": -2.265625, "reward/margin": 1.1640625, "reward/rejected": -3.421875, "step": 908 }, { "approx. KL/chosen": 154.0, "approx. KL/rejected": 660.0, "epoch": 0.2726454709058188, "grad_norm": 7.660150527954102, "learning_rate": 8.278833080674796e-06, "logp/chosen": -306.0, "logp/rejected": -362.0, "loss": 0.6726760864257812, "reward/accuracy": 0.8125, "reward/chosen": -1.109375, "reward/margin": 1.7890625, "reward/rejected": -2.890625, "step": 909 }, { "approx. KL/chosen": 147.0, "approx. KL/rejected": 392.0, "epoch": 0.27294541091781643, "grad_norm": 3.8935751914978027, "learning_rate": 8.275274654456689e-06, "logp/chosen": -384.0, "logp/rejected": -332.0, "loss": 0.4404296875, "reward/accuracy": 0.8125, "reward/chosen": -1.40625, "reward/margin": 1.109375, "reward/rejected": -2.515625, "step": 910 }, { "approx. KL/chosen": 248.0, "approx. KL/rejected": 640.0, "epoch": 0.27324535092981406, "grad_norm": 9.248619079589844, "learning_rate": 8.271713320092309e-06, "logp/chosen": -432.0, "logp/rejected": -342.0, "loss": 0.46783447265625, "reward/accuracy": 0.75, "reward/chosen": -1.71875, "reward/margin": 1.4609375, "reward/rejected": -3.1875, "step": 911 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 420.0, "epoch": 0.2735452909418116, "grad_norm": 4.2387895584106445, "learning_rate": 8.268149080743797e-06, "logp/chosen": -286.0, "logp/rejected": -334.0, "loss": 0.57476806640625, "reward/accuracy": 0.8125, "reward/chosen": -1.59375, "reward/margin": 0.9609375, "reward/rejected": -2.5625, "step": 912 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 316.0, "epoch": 0.27384523095380925, "grad_norm": 5.138131141662598, "learning_rate": 8.264581939575876e-06, "logp/chosen": -249.0, "logp/rejected": -157.0, "loss": 0.714111328125, "reward/accuracy": 0.6875, "reward/chosen": -1.7109375, "reward/margin": 0.5546875, "reward/rejected": -2.265625, "step": 913 }, { "approx. KL/chosen": 107.5, "approx. KL/rejected": 564.0, "epoch": 0.2741451709658068, "grad_norm": 4.575900077819824, "learning_rate": 8.26101189975584e-06, "logp/chosen": -312.0, "logp/rejected": -360.0, "loss": 0.49066162109375, "reward/accuracy": 0.8125, "reward/chosen": -1.1875, "reward/margin": 1.515625, "reward/rejected": -2.703125, "step": 914 }, { "approx. KL/chosen": 126.0, "approx. KL/rejected": 1012.0, "epoch": 0.27444511097780444, "grad_norm": 3.9748995304107666, "learning_rate": 8.257438964453561e-06, "logp/chosen": -296.0, "logp/rejected": -288.0, "loss": 0.377349853515625, "reward/accuracy": 0.875, "reward/chosen": -1.21875, "reward/margin": 2.1875, "reward/rejected": -3.421875, "step": 915 }, { "approx. KL/chosen": 65.0, "approx. KL/rejected": 704.0, "epoch": 0.27474505098980206, "grad_norm": 2.1003994941711426, "learning_rate": 8.25386313684148e-06, "logp/chosen": -414.0, "logp/rejected": -358.0, "loss": 0.178131103515625, "reward/accuracy": 1.0, "reward/chosen": -0.8828125, "reward/margin": 2.484375, "reward/rejected": -3.375, "step": 916 }, { "approx. KL/chosen": 119.0, "approx. KL/rejected": 552.0, "epoch": 0.27504499100179963, "grad_norm": 4.331458568572998, "learning_rate": 8.25028442009461e-06, "logp/chosen": -452.0, "logp/rejected": -406.0, "loss": 0.48459625244140625, "reward/accuracy": 0.75, "reward/chosen": -0.98046875, "reward/margin": 1.7265625, "reward/rejected": -2.703125, "step": 917 }, { "approx. KL/chosen": 99.5, "approx. KL/rejected": 528.0, "epoch": 0.27534493101379726, "grad_norm": 2.3422813415527344, "learning_rate": 8.246702817390523e-06, "logp/chosen": -340.0, "logp/rejected": -358.0, "loss": 0.2049560546875, "reward/accuracy": 1.0, "reward/chosen": -0.81640625, "reward/margin": 2.03125, "reward/rejected": -2.84375, "step": 918 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 464.0, "epoch": 0.2756448710257948, "grad_norm": 4.608014106750488, "learning_rate": 8.243118331909359e-06, "logp/chosen": -428.0, "logp/rejected": -404.0, "loss": 0.379150390625, "reward/accuracy": 0.8125, "reward/chosen": -1.1171875, "reward/margin": 1.578125, "reward/rejected": -2.703125, "step": 919 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 740.0, "epoch": 0.27594481103779245, "grad_norm": 3.83647084236145, "learning_rate": 8.239530966833815e-06, "logp/chosen": -245.0, "logp/rejected": -278.0, "loss": 0.34527587890625, "reward/accuracy": 0.75, "reward/chosen": -1.4453125, "reward/margin": 1.9140625, "reward/rejected": -3.359375, "step": 920 }, { "approx. KL/chosen": 115.0, "approx. KL/rejected": 494.0, "epoch": 0.27624475104979, "grad_norm": 3.411720037460327, "learning_rate": 8.235940725349146e-06, "logp/chosen": -376.0, "logp/rejected": -302.0, "loss": 0.409423828125, "reward/accuracy": 0.6875, "reward/chosen": -1.0, "reward/margin": 1.59375, "reward/rejected": -2.59375, "step": 921 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 352.0, "epoch": 0.27654469106178764, "grad_norm": 3.6549465656280518, "learning_rate": 8.232347610643156e-06, "logp/chosen": -280.0, "logp/rejected": -242.0, "loss": 0.5938720703125, "reward/accuracy": 0.6875, "reward/chosen": -1.390625, "reward/margin": 0.90234375, "reward/rejected": -2.296875, "step": 922 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 652.0, "epoch": 0.27684463107378526, "grad_norm": 4.134742259979248, "learning_rate": 8.228751625906208e-06, "logp/chosen": -274.0, "logp/rejected": -390.0, "loss": 0.355560302734375, "reward/accuracy": 0.875, "reward/chosen": -1.359375, "reward/margin": 1.7421875, "reward/rejected": -3.09375, "step": 923 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 556.0, "epoch": 0.27714457108578283, "grad_norm": 2.613734006881714, "learning_rate": 8.225152774331209e-06, "logp/chosen": -350.0, "logp/rejected": -288.0, "loss": 0.34649658203125, "reward/accuracy": 0.8125, "reward/chosen": -1.2578125, "reward/margin": 1.6328125, "reward/rejected": -2.890625, "step": 924 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 600.0, "epoch": 0.27744451109778046, "grad_norm": 3.765681505203247, "learning_rate": 8.221551059113613e-06, "logp/chosen": -270.0, "logp/rejected": -390.0, "loss": 0.50482177734375, "reward/accuracy": 0.75, "reward/chosen": -1.421875, "reward/margin": 1.5625, "reward/rejected": -2.984375, "step": 925 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 302.0, "epoch": 0.277744451109778, "grad_norm": 5.364788055419922, "learning_rate": 8.217946483451414e-06, "logp/chosen": -314.0, "logp/rejected": -250.0, "loss": 0.641448974609375, "reward/accuracy": 0.5625, "reward/chosen": -1.171875, "reward/margin": 0.76171875, "reward/rejected": -1.9375, "step": 926 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 420.0, "epoch": 0.27804439112177565, "grad_norm": 5.485270023345947, "learning_rate": 8.21433905054515e-06, "logp/chosen": -372.0, "logp/rejected": -346.0, "loss": 0.693817138671875, "reward/accuracy": 0.625, "reward/chosen": -1.3203125, "reward/margin": 1.0234375, "reward/rejected": -2.34375, "step": 927 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 384.0, "epoch": 0.27834433113377327, "grad_norm": 4.1380534172058105, "learning_rate": 8.21072876359789e-06, "logp/chosen": -266.0, "logp/rejected": -310.0, "loss": 0.51898193359375, "reward/accuracy": 0.75, "reward/chosen": -1.3125, "reward/margin": 1.1953125, "reward/rejected": -2.515625, "step": 928 }, { "approx. KL/chosen": 123.0, "approx. KL/rejected": 282.0, "epoch": 0.27864427114577084, "grad_norm": 3.5555617809295654, "learning_rate": 8.207115625815245e-06, "logp/chosen": -384.0, "logp/rejected": -328.0, "loss": 0.4132080078125, "reward/accuracy": 0.875, "reward/chosen": -0.8984375, "reward/margin": 1.03125, "reward/rejected": -1.9296875, "step": 929 }, { "approx. KL/chosen": 79.5, "approx. KL/rejected": 390.0, "epoch": 0.27894421115776846, "grad_norm": 4.100442886352539, "learning_rate": 8.203499640405351e-06, "logp/chosen": -332.0, "logp/rejected": -402.0, "loss": 0.36309814453125, "reward/accuracy": 0.875, "reward/chosen": -0.859375, "reward/margin": 1.5234375, "reward/rejected": -2.375, "step": 930 }, { "approx. KL/chosen": 172.0, "approx. KL/rejected": 520.0, "epoch": 0.27924415116976603, "grad_norm": 5.400422096252441, "learning_rate": 8.199880810578873e-06, "logp/chosen": -328.0, "logp/rejected": -316.0, "loss": 0.761474609375, "reward/accuracy": 0.6875, "reward/chosen": -1.4140625, "reward/margin": 1.203125, "reward/rejected": -2.625, "step": 931 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 788.0, "epoch": 0.27954409118176365, "grad_norm": 2.831602096557617, "learning_rate": 8.196259139549006e-06, "logp/chosen": -290.0, "logp/rejected": -382.0, "loss": 0.279541015625, "reward/accuracy": 0.8125, "reward/chosen": -1.359375, "reward/margin": 2.203125, "reward/rejected": -3.5625, "step": 932 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 632.0, "epoch": 0.2798440311937612, "grad_norm": 3.880296230316162, "learning_rate": 8.192634630531461e-06, "logp/chosen": -356.0, "logp/rejected": -372.0, "loss": 0.38946533203125, "reward/accuracy": 0.6875, "reward/chosen": -1.2265625, "reward/margin": 1.75, "reward/rejected": -2.96875, "step": 933 }, { "approx. KL/chosen": 206.0, "approx. KL/rejected": 428.0, "epoch": 0.28014397120575885, "grad_norm": 4.510580062866211, "learning_rate": 8.18900728674448e-06, "logp/chosen": -222.0, "logp/rejected": -213.0, "loss": 0.53094482421875, "reward/accuracy": 0.6875, "reward/chosen": -1.5546875, "reward/margin": 1.0859375, "reward/rejected": -2.640625, "step": 934 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 452.0, "epoch": 0.28044391121775647, "grad_norm": 4.686017036437988, "learning_rate": 8.185377111408804e-06, "logp/chosen": -466.0, "logp/rejected": -486.0, "loss": 0.4544677734375, "reward/accuracy": 0.625, "reward/chosen": -0.984375, "reward/margin": 1.3515625, "reward/rejected": -2.328125, "step": 935 }, { "approx. KL/chosen": 50.75, "approx. KL/rejected": 362.0, "epoch": 0.28074385122975404, "grad_norm": 3.9416048526763916, "learning_rate": 8.181744107747707e-06, "logp/chosen": -290.0, "logp/rejected": -376.0, "loss": 0.4295654296875, "reward/accuracy": 0.75, "reward/chosen": -0.5859375, "reward/margin": 1.484375, "reward/rejected": -2.078125, "step": 936 }, { "approx. KL/chosen": 146.0, "approx. KL/rejected": 486.0, "epoch": 0.28104379124175166, "grad_norm": 3.9575164318084717, "learning_rate": 8.178108278986963e-06, "logp/chosen": -326.0, "logp/rejected": -296.0, "loss": 0.3575439453125, "reward/accuracy": 0.9375, "reward/chosen": -1.171875, "reward/margin": 1.4921875, "reward/rejected": -2.671875, "step": 937 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 400.0, "epoch": 0.28134373125374923, "grad_norm": 4.0661940574646, "learning_rate": 8.174469628354857e-06, "logp/chosen": -284.0, "logp/rejected": -238.0, "loss": 0.4581298828125, "reward/accuracy": 0.75, "reward/chosen": -1.0234375, "reward/margin": 1.296875, "reward/rejected": -2.328125, "step": 938 }, { "approx. KL/chosen": 53.75, "approx. KL/rejected": 808.0, "epoch": 0.28164367126574685, "grad_norm": 2.776394844055176, "learning_rate": 8.170828159082184e-06, "logp/chosen": -342.0, "logp/rejected": -426.0, "loss": 0.25891876220703125, "reward/accuracy": 0.875, "reward/chosen": -0.5546875, "reward/margin": 2.734375, "reward/rejected": -3.28125, "step": 939 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 284.0, "epoch": 0.2819436112777445, "grad_norm": 4.802924633026123, "learning_rate": 8.167183874402234e-06, "logp/chosen": -286.0, "logp/rejected": -304.0, "loss": 0.62347412109375, "reward/accuracy": 0.5625, "reward/chosen": -0.99609375, "reward/margin": 0.546875, "reward/rejected": -1.546875, "step": 940 }, { "approx. KL/chosen": 112.5, "approx. KL/rejected": 416.0, "epoch": 0.28224355128974205, "grad_norm": 4.539976596832275, "learning_rate": 8.1635367775508e-06, "logp/chosen": -354.0, "logp/rejected": -338.0, "loss": 0.4085845947265625, "reward/accuracy": 0.75, "reward/chosen": -0.81640625, "reward/margin": 1.5625, "reward/rejected": -2.375, "step": 941 }, { "approx. KL/chosen": 91.0, "approx. KL/rejected": 402.0, "epoch": 0.28254349130173967, "grad_norm": 4.2165985107421875, "learning_rate": 8.159886871766177e-06, "logp/chosen": -408.0, "logp/rejected": -396.0, "loss": 0.4295654296875, "reward/accuracy": 0.8125, "reward/chosen": -0.75390625, "reward/margin": 1.515625, "reward/rejected": -2.28125, "step": 942 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 812.0, "epoch": 0.28284343131373724, "grad_norm": 4.522670269012451, "learning_rate": 8.156234160289145e-06, "logp/chosen": -236.0, "logp/rejected": -236.0, "loss": 0.315704345703125, "reward/accuracy": 0.875, "reward/chosen": -1.1953125, "reward/margin": 2.578125, "reward/rejected": -3.78125, "step": 943 }, { "approx. KL/chosen": 91.0, "approx. KL/rejected": 316.0, "epoch": 0.28314337132573486, "grad_norm": 4.189444541931152, "learning_rate": 8.152578646362984e-06, "logp/chosen": -384.0, "logp/rejected": -332.0, "loss": 0.541015625, "reward/accuracy": 0.6875, "reward/chosen": -0.7578125, "reward/margin": 1.046875, "reward/rejected": -1.8046875, "step": 944 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 524.0, "epoch": 0.28344331133773243, "grad_norm": 3.48127818107605, "learning_rate": 8.148920333233455e-06, "logp/chosen": -332.0, "logp/rejected": -326.0, "loss": 0.40972900390625, "reward/accuracy": 0.75, "reward/chosen": -1.4453125, "reward/margin": 1.2734375, "reward/rejected": -2.71875, "step": 945 }, { "approx. KL/chosen": 124.0, "approx. KL/rejected": 270.0, "epoch": 0.28374325134973005, "grad_norm": 4.351458549499512, "learning_rate": 8.145259224148807e-06, "logp/chosen": -314.0, "logp/rejected": -424.0, "loss": 0.663818359375, "reward/accuracy": 0.6875, "reward/chosen": -1.140625, "reward/margin": 0.671875, "reward/rejected": -1.8125, "step": 946 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 524.0, "epoch": 0.2840431913617277, "grad_norm": 5.793646335601807, "learning_rate": 8.141595322359775e-06, "logp/chosen": -366.0, "logp/rejected": -418.0, "loss": 0.5723876953125, "reward/accuracy": 0.8125, "reward/chosen": -1.609375, "reward/margin": 1.15625, "reward/rejected": -2.765625, "step": 947 }, { "approx. KL/chosen": 100.0, "approx. KL/rejected": 354.0, "epoch": 0.28434313137372524, "grad_norm": 4.328972816467285, "learning_rate": 8.137928631119569e-06, "logp/chosen": -446.0, "logp/rejected": -492.0, "loss": 0.5452880859375, "reward/accuracy": 0.625, "reward/chosen": -1.0625, "reward/margin": 1.03125, "reward/rejected": -2.09375, "step": 948 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 410.0, "epoch": 0.28464307138572287, "grad_norm": 5.390100002288818, "learning_rate": 8.134259153683881e-06, "logp/chosen": -253.0, "logp/rejected": -266.0, "loss": 0.5596923828125, "reward/accuracy": 0.6875, "reward/chosen": -1.6953125, "reward/margin": 0.78515625, "reward/rejected": -2.484375, "step": 949 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 428.0, "epoch": 0.28494301139772044, "grad_norm": 3.020235061645508, "learning_rate": 8.130586893310868e-06, "logp/chosen": -230.0, "logp/rejected": -220.0, "loss": 0.309295654296875, "reward/accuracy": 0.9375, "reward/chosen": -1.140625, "reward/margin": 1.46875, "reward/rejected": -2.609375, "step": 950 }, { "approx. KL/chosen": 116.5, "approx. KL/rejected": 468.0, "epoch": 0.28524295140971806, "grad_norm": 3.174663543701172, "learning_rate": 8.126911853261167e-06, "logp/chosen": -282.0, "logp/rejected": -314.0, "loss": 0.3853759765625, "reward/accuracy": 0.8125, "reward/chosen": -1.0234375, "reward/margin": 1.4765625, "reward/rejected": -2.5, "step": 951 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 490.0, "epoch": 0.2855428914217157, "grad_norm": 4.743523120880127, "learning_rate": 8.12323403679788e-06, "logp/chosen": -386.0, "logp/rejected": -414.0, "loss": 0.38421630859375, "reward/accuracy": 0.8125, "reward/chosen": -1.2734375, "reward/margin": 1.375, "reward/rejected": -2.65625, "step": 952 }, { "approx. KL/chosen": 98.5, "approx. KL/rejected": 480.0, "epoch": 0.28584283143371325, "grad_norm": 3.7169628143310547, "learning_rate": 8.119553447186573e-06, "logp/chosen": -360.0, "logp/rejected": -394.0, "loss": 0.3500518798828125, "reward/accuracy": 0.875, "reward/chosen": -1.0703125, "reward/margin": 1.6171875, "reward/rejected": -2.6875, "step": 953 }, { "approx. KL/chosen": 187.0, "approx. KL/rejected": 292.0, "epoch": 0.2861427714457109, "grad_norm": 5.895851135253906, "learning_rate": 8.115870087695275e-06, "logp/chosen": -334.0, "logp/rejected": -322.0, "loss": 0.802734375, "reward/accuracy": 0.625, "reward/chosen": -1.109375, "reward/margin": 0.640625, "reward/rejected": -1.75, "step": 954 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 612.0, "epoch": 0.28644271145770844, "grad_norm": 4.354063987731934, "learning_rate": 8.112183961594474e-06, "logp/chosen": -298.0, "logp/rejected": -330.0, "loss": 0.5088958740234375, "reward/accuracy": 0.75, "reward/chosen": -1.390625, "reward/margin": 1.4921875, "reward/rejected": -2.890625, "step": 955 }, { "approx. KL/chosen": 96.0, "approx. KL/rejected": 572.0, "epoch": 0.28674265146970607, "grad_norm": 2.782029151916504, "learning_rate": 8.108495072157114e-06, "logp/chosen": -314.0, "logp/rejected": -255.0, "loss": 0.2674407958984375, "reward/accuracy": 0.875, "reward/chosen": -0.68359375, "reward/margin": 2.109375, "reward/rejected": -2.796875, "step": 956 }, { "approx. KL/chosen": 116.5, "approx. KL/rejected": 544.0, "epoch": 0.28704259148170364, "grad_norm": 2.5227227210998535, "learning_rate": 8.104803422658598e-06, "logp/chosen": -332.0, "logp/rejected": -232.0, "loss": 0.3118896484375, "reward/accuracy": 0.875, "reward/chosen": -1.03125, "reward/margin": 1.7265625, "reward/rejected": -2.75, "step": 957 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 588.0, "epoch": 0.28734253149370126, "grad_norm": 4.513731479644775, "learning_rate": 8.10110901637677e-06, "logp/chosen": -256.0, "logp/rejected": -328.0, "loss": 0.391845703125, "reward/accuracy": 0.75, "reward/chosen": -1.3984375, "reward/margin": 1.3125, "reward/rejected": -2.703125, "step": 958 }, { "approx. KL/chosen": 112.5, "approx. KL/rejected": 476.0, "epoch": 0.2876424715056989, "grad_norm": 4.181436061859131, "learning_rate": 8.09741185659193e-06, "logp/chosen": -320.0, "logp/rejected": -298.0, "loss": 0.565185546875, "reward/accuracy": 0.75, "reward/chosen": -0.9609375, "reward/margin": 1.4140625, "reward/rejected": -2.375, "step": 959 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 496.0, "epoch": 0.28794241151769645, "grad_norm": 4.265212535858154, "learning_rate": 8.09371194658682e-06, "logp/chosen": -394.0, "logp/rejected": -392.0, "loss": 0.446044921875, "reward/accuracy": 0.8125, "reward/chosen": -1.2109375, "reward/margin": 1.3671875, "reward/rejected": -2.578125, "step": 960 }, { "approx. KL/chosen": 90.5, "approx. KL/rejected": 532.0, "epoch": 0.2882423515296941, "grad_norm": 3.322821855545044, "learning_rate": 8.090009289646623e-06, "logp/chosen": -334.0, "logp/rejected": -270.0, "loss": 0.442626953125, "reward/accuracy": 0.75, "reward/chosen": -0.87890625, "reward/margin": 1.6171875, "reward/rejected": -2.5, "step": 961 }, { "approx. KL/chosen": 95.5, "approx. KL/rejected": 736.0, "epoch": 0.28854229154169164, "grad_norm": 2.5309362411499023, "learning_rate": 8.086303889058964e-06, "logp/chosen": -141.0, "logp/rejected": -300.0, "loss": 0.212005615234375, "reward/accuracy": 0.9375, "reward/chosen": -0.9140625, "reward/margin": 2.421875, "reward/rejected": -3.34375, "step": 962 }, { "approx. KL/chosen": 61.0, "approx. KL/rejected": 500.0, "epoch": 0.28884223155368927, "grad_norm": 3.799968719482422, "learning_rate": 8.082595748113898e-06, "logp/chosen": -464.0, "logp/rejected": -382.0, "loss": 0.4078369140625, "reward/accuracy": 0.6875, "reward/chosen": -0.6796875, "reward/margin": 1.8515625, "reward/rejected": -2.53125, "step": 963 }, { "approx. KL/chosen": 130.0, "approx. KL/rejected": 452.0, "epoch": 0.2891421715656869, "grad_norm": 4.171777725219727, "learning_rate": 8.07888487010392e-06, "logp/chosen": -268.0, "logp/rejected": -330.0, "loss": 0.4366607666015625, "reward/accuracy": 0.75, "reward/chosen": -0.54296875, "reward/margin": 1.6875, "reward/rejected": -2.234375, "step": 964 }, { "approx. KL/chosen": 171.0, "approx. KL/rejected": 608.0, "epoch": 0.28944211157768446, "grad_norm": 3.4875152111053467, "learning_rate": 8.075171258323952e-06, "logp/chosen": -318.0, "logp/rejected": -358.0, "loss": 0.33815765380859375, "reward/accuracy": 0.875, "reward/chosen": -1.1484375, "reward/margin": 1.6640625, "reward/rejected": -2.8125, "step": 965 }, { "approx. KL/chosen": 172.0, "approx. KL/rejected": 792.0, "epoch": 0.2897420515896821, "grad_norm": 2.8316028118133545, "learning_rate": 8.071454916071345e-06, "logp/chosen": -316.0, "logp/rejected": -420.0, "loss": 0.3473052978515625, "reward/accuracy": 0.8125, "reward/chosen": -1.203125, "reward/margin": 2.203125, "reward/rejected": -3.40625, "step": 966 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 458.0, "epoch": 0.29004199160167965, "grad_norm": 5.761105060577393, "learning_rate": 8.067735846645868e-06, "logp/chosen": -358.0, "logp/rejected": -386.0, "loss": 0.7681884765625, "reward/accuracy": 0.6875, "reward/chosen": -1.546875, "reward/margin": 1.0, "reward/rejected": -2.546875, "step": 967 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 532.0, "epoch": 0.2903419316136773, "grad_norm": 3.824856758117676, "learning_rate": 8.064014053349724e-06, "logp/chosen": -274.0, "logp/rejected": -292.0, "loss": 0.42437744140625, "reward/accuracy": 0.75, "reward/chosen": -1.6484375, "reward/margin": 1.1328125, "reward/rejected": -2.78125, "step": 968 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 672.0, "epoch": 0.29064187162567484, "grad_norm": 4.991366386413574, "learning_rate": 8.060289539487523e-06, "logp/chosen": -245.0, "logp/rejected": -262.0, "loss": 0.6102485656738281, "reward/accuracy": 0.6875, "reward/chosen": -1.359375, "reward/margin": 1.6953125, "reward/rejected": -3.046875, "step": 969 }, { "approx. KL/chosen": 188.0, "approx. KL/rejected": 932.0, "epoch": 0.29094181163767247, "grad_norm": 2.717212200164795, "learning_rate": 8.056562308366296e-06, "logp/chosen": -340.0, "logp/rejected": -384.0, "loss": 0.24041748046875, "reward/accuracy": 0.875, "reward/chosen": -1.4375, "reward/margin": 2.265625, "reward/rejected": -3.703125, "step": 970 }, { "approx. KL/chosen": 47.25, "approx. KL/rejected": 294.0, "epoch": 0.2912417516496701, "grad_norm": 2.91646409034729, "learning_rate": 8.052832363295485e-06, "logp/chosen": -225.0, "logp/rejected": -262.0, "loss": 0.385498046875, "reward/accuracy": 0.875, "reward/chosen": -0.72265625, "reward/margin": 1.3671875, "reward/rejected": -2.09375, "step": 971 }, { "approx. KL/chosen": 69.5, "approx. KL/rejected": 632.0, "epoch": 0.29154169166166766, "grad_norm": 3.2216224670410156, "learning_rate": 8.049099707586943e-06, "logp/chosen": -221.0, "logp/rejected": -368.0, "loss": 0.32147216796875, "reward/accuracy": 0.8125, "reward/chosen": -0.63671875, "reward/margin": 2.0, "reward/rejected": -2.640625, "step": 972 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 1040.0, "epoch": 0.2918416316736653, "grad_norm": 2.515343189239502, "learning_rate": 8.04536434455493e-06, "logp/chosen": -186.0, "logp/rejected": -340.0, "loss": 0.2658653259277344, "reward/accuracy": 0.875, "reward/chosen": -1.359375, "reward/margin": 2.234375, "reward/rejected": -3.59375, "step": 973 }, { "approx. KL/chosen": 85.5, "approx. KL/rejected": 840.0, "epoch": 0.29214157168566285, "grad_norm": 4.079440593719482, "learning_rate": 8.041626277516105e-06, "logp/chosen": -282.0, "logp/rejected": -262.0, "loss": 0.393402099609375, "reward/accuracy": 0.75, "reward/chosen": -1.0625, "reward/margin": 2.21875, "reward/rejected": -3.28125, "step": 974 }, { "approx. KL/chosen": 226.0, "approx. KL/rejected": 852.0, "epoch": 0.2924415116976605, "grad_norm": 3.87296986579895, "learning_rate": 8.037885509789536e-06, "logp/chosen": -228.0, "logp/rejected": -324.0, "loss": 0.4338226318359375, "reward/accuracy": 0.75, "reward/chosen": -1.59375, "reward/margin": 2.09375, "reward/rejected": -3.6875, "step": 975 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 516.0, "epoch": 0.2927414517096581, "grad_norm": 2.7930281162261963, "learning_rate": 8.034142044696682e-06, "logp/chosen": -356.0, "logp/rejected": -330.0, "loss": 0.321533203125, "reward/accuracy": 0.875, "reward/chosen": -0.890625, "reward/margin": 1.6171875, "reward/rejected": -2.515625, "step": 976 }, { "approx. KL/chosen": 137.0, "approx. KL/rejected": 484.0, "epoch": 0.29304139172165566, "grad_norm": 4.663668632507324, "learning_rate": 8.030395885561401e-06, "logp/chosen": -354.0, "logp/rejected": -404.0, "loss": 0.541290283203125, "reward/accuracy": 0.75, "reward/chosen": -0.88671875, "reward/margin": 1.453125, "reward/rejected": -2.34375, "step": 977 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 202.0, "epoch": 0.2933413317336533, "grad_norm": 4.102541446685791, "learning_rate": 8.026647035709944e-06, "logp/chosen": -334.0, "logp/rejected": -286.0, "loss": 0.6083984375, "reward/accuracy": 0.75, "reward/chosen": -1.1640625, "reward/margin": 0.50390625, "reward/rejected": -1.671875, "step": 978 }, { "approx. KL/chosen": 145.0, "approx. KL/rejected": 828.0, "epoch": 0.29364127174565086, "grad_norm": 5.476723670959473, "learning_rate": 8.022895498470946e-06, "logp/chosen": -294.0, "logp/rejected": -346.0, "loss": 0.4268035888671875, "reward/accuracy": 0.75, "reward/chosen": -0.921875, "reward/margin": 2.4375, "reward/rejected": -3.34375, "step": 979 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 716.0, "epoch": 0.2939412117576485, "grad_norm": 4.216798305511475, "learning_rate": 8.019141277175433e-06, "logp/chosen": -250.0, "logp/rejected": -316.0, "loss": 0.5655517578125, "reward/accuracy": 0.5, "reward/chosen": -1.546875, "reward/margin": 1.046875, "reward/rejected": -2.59375, "step": 980 }, { "approx. KL/chosen": 62.25, "approx. KL/rejected": 358.0, "epoch": 0.29424115176964605, "grad_norm": 2.9020347595214844, "learning_rate": 8.015384375156812e-06, "logp/chosen": -306.0, "logp/rejected": -288.0, "loss": 0.29638671875, "reward/accuracy": 0.875, "reward/chosen": -0.6015625, "reward/margin": 1.6484375, "reward/rejected": -2.25, "step": 981 }, { "approx. KL/chosen": 171.0, "approx. KL/rejected": 624.0, "epoch": 0.29454109178164367, "grad_norm": 4.140937328338623, "learning_rate": 8.01162479575087e-06, "logp/chosen": -364.0, "logp/rejected": -364.0, "loss": 0.352569580078125, "reward/accuracy": 0.75, "reward/chosen": -0.98046875, "reward/margin": 1.953125, "reward/rejected": -2.9375, "step": 982 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 844.0, "epoch": 0.2948410317936413, "grad_norm": 4.3109283447265625, "learning_rate": 8.00786254229577e-06, "logp/chosen": -352.0, "logp/rejected": -350.0, "loss": 0.368499755859375, "reward/accuracy": 0.8125, "reward/chosen": -1.078125, "reward/margin": 1.96875, "reward/rejected": -3.046875, "step": 983 }, { "approx. KL/chosen": 91.0, "approx. KL/rejected": 636.0, "epoch": 0.29514097180563886, "grad_norm": 4.45871639251709, "learning_rate": 8.004097618132055e-06, "logp/chosen": -250.0, "logp/rejected": -280.0, "loss": 0.367645263671875, "reward/accuracy": 0.9375, "reward/chosen": -0.73828125, "reward/margin": 2.171875, "reward/rejected": -2.90625, "step": 984 }, { "approx. KL/chosen": 83.0, "approx. KL/rejected": 536.0, "epoch": 0.2954409118176365, "grad_norm": 4.080587863922119, "learning_rate": 8.000330026602632e-06, "logp/chosen": -229.0, "logp/rejected": -278.0, "loss": 0.5243988037109375, "reward/accuracy": 0.625, "reward/chosen": -0.6875, "reward/margin": 1.65625, "reward/rejected": -2.34375, "step": 985 }, { "approx. KL/chosen": 127.5, "approx. KL/rejected": 796.0, "epoch": 0.29574085182963405, "grad_norm": 3.699587821960449, "learning_rate": 7.996559771052783e-06, "logp/chosen": -268.0, "logp/rejected": -328.0, "loss": 0.363616943359375, "reward/accuracy": 0.8125, "reward/chosen": -1.0, "reward/margin": 2.296875, "reward/rejected": -3.296875, "step": 986 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 732.0, "epoch": 0.2960407918416317, "grad_norm": 5.065359115600586, "learning_rate": 7.992786854830152e-06, "logp/chosen": -382.0, "logp/rejected": -438.0, "loss": 0.669921875, "reward/accuracy": 0.625, "reward/chosen": -1.5390625, "reward/margin": 1.203125, "reward/rejected": -2.734375, "step": 987 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 976.0, "epoch": 0.29634073185362925, "grad_norm": 3.7634143829345703, "learning_rate": 7.989011281284743e-06, "logp/chosen": -352.0, "logp/rejected": -402.0, "loss": 0.42568206787109375, "reward/accuracy": 0.8125, "reward/chosen": -1.0859375, "reward/margin": 2.21875, "reward/rejected": -3.3125, "step": 988 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 480.0, "epoch": 0.29664067186562687, "grad_norm": 5.283275127410889, "learning_rate": 7.985233053768926e-06, "logp/chosen": -260.0, "logp/rejected": -252.0, "loss": 0.465576171875, "reward/accuracy": 0.6875, "reward/chosen": -1.4453125, "reward/margin": 1.203125, "reward/rejected": -2.640625, "step": 989 }, { "approx. KL/chosen": 180.0, "approx. KL/rejected": 446.0, "epoch": 0.2969406118776245, "grad_norm": 7.288679122924805, "learning_rate": 7.981452175637422e-06, "logp/chosen": -358.0, "logp/rejected": -300.0, "loss": 0.522705078125, "reward/accuracy": 0.6875, "reward/chosen": -1.4765625, "reward/margin": 1.1015625, "reward/rejected": -2.578125, "step": 990 }, { "approx. KL/chosen": 255.0, "approx. KL/rejected": 306.0, "epoch": 0.29724055188962206, "grad_norm": 6.455471038818359, "learning_rate": 7.977668650247306e-06, "logp/chosen": -296.0, "logp/rejected": -346.0, "loss": 0.657073974609375, "reward/accuracy": 0.625, "reward/chosen": -1.46875, "reward/margin": 0.71484375, "reward/rejected": -2.1875, "step": 991 }, { "approx. KL/chosen": 490.0, "approx. KL/rejected": 1064.0, "epoch": 0.2975404919016197, "grad_norm": 12.594501495361328, "learning_rate": 7.973882480958005e-06, "logp/chosen": -318.0, "logp/rejected": -270.0, "loss": 0.4068603515625, "reward/accuracy": 0.8125, "reward/chosen": -2.296875, "reward/margin": 1.828125, "reward/rejected": -4.125, "step": 992 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 524.0, "epoch": 0.29784043191361725, "grad_norm": 5.60983419418335, "learning_rate": 7.970093671131295e-06, "logp/chosen": -284.0, "logp/rejected": -284.0, "loss": 0.5210800170898438, "reward/accuracy": 0.75, "reward/chosen": -0.9765625, "reward/margin": 1.6640625, "reward/rejected": -2.640625, "step": 993 }, { "approx. KL/chosen": 65.0, "approx. KL/rejected": 772.0, "epoch": 0.2981403719256149, "grad_norm": 3.777188777923584, "learning_rate": 7.966302224131293e-06, "logp/chosen": -406.0, "logp/rejected": -488.0, "loss": 0.27916717529296875, "reward/accuracy": 0.75, "reward/chosen": -0.365234375, "reward/margin": 2.765625, "reward/rejected": -3.125, "step": 994 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 668.0, "epoch": 0.2984403119376125, "grad_norm": 3.0494706630706787, "learning_rate": 7.962508143324462e-06, "logp/chosen": -232.0, "logp/rejected": -320.0, "loss": 0.27490234375, "reward/accuracy": 0.9375, "reward/chosen": -1.3046875, "reward/margin": 1.7890625, "reward/rejected": -3.09375, "step": 995 }, { "approx. KL/chosen": 314.0, "approx. KL/rejected": 792.0, "epoch": 0.29874025194961007, "grad_norm": 6.163644313812256, "learning_rate": 7.958711432079598e-06, "logp/chosen": -408.0, "logp/rejected": -318.0, "loss": 0.3631744384765625, "reward/accuracy": 0.8125, "reward/chosen": -1.6484375, "reward/margin": 1.859375, "reward/rejected": -3.515625, "step": 996 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 956.0, "epoch": 0.2990401919616077, "grad_norm": 4.323972225189209, "learning_rate": 7.954912093767837e-06, "logp/chosen": -286.0, "logp/rejected": -290.0, "loss": 0.3220977783203125, "reward/accuracy": 0.75, "reward/chosen": -1.6171875, "reward/margin": 2.078125, "reward/rejected": -3.703125, "step": 997 }, { "approx. KL/chosen": 148.0, "approx. KL/rejected": 768.0, "epoch": 0.29934013197360526, "grad_norm": 2.5934791564941406, "learning_rate": 7.951110131762644e-06, "logp/chosen": -394.0, "logp/rejected": -410.0, "loss": 0.23541259765625, "reward/accuracy": 0.875, "reward/chosen": -1.3359375, "reward/margin": 2.265625, "reward/rejected": -3.609375, "step": 998 }, { "approx. KL/chosen": 388.0, "approx. KL/rejected": 808.0, "epoch": 0.2996400719856029, "grad_norm": 5.715371608734131, "learning_rate": 7.947305549439822e-06, "logp/chosen": -368.0, "logp/rejected": -364.0, "loss": 0.5670280456542969, "reward/accuracy": 0.625, "reward/chosen": -1.8125, "reward/margin": 1.546875, "reward/rejected": -3.359375, "step": 999 }, { "approx. KL/chosen": 148.0, "approx. KL/rejected": 696.0, "epoch": 0.29994001199760045, "grad_norm": 3.3270623683929443, "learning_rate": 7.943498350177487e-06, "logp/chosen": -220.0, "logp/rejected": -286.0, "loss": 0.2404937744140625, "reward/accuracy": 0.8125, "reward/chosen": -0.9375, "reward/margin": 2.390625, "reward/rejected": -3.328125, "step": 1000 }, { "approx. KL/chosen": 310.0, "approx. KL/rejected": 640.0, "epoch": 0.3002399520095981, "grad_norm": 5.558892250061035, "learning_rate": 7.939688537356092e-06, "logp/chosen": -352.0, "logp/rejected": -336.0, "loss": 0.464111328125, "reward/accuracy": 0.75, "reward/chosen": -1.875, "reward/margin": 1.1640625, "reward/rejected": -3.03125, "step": 1001 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 1272.0, "epoch": 0.3005398920215957, "grad_norm": 4.0377197265625, "learning_rate": 7.935876114358398e-06, "logp/chosen": -298.0, "logp/rejected": -352.0, "loss": 0.35784912109375, "reward/accuracy": 0.8125, "reward/chosen": -1.5390625, "reward/margin": 2.046875, "reward/rejected": -3.578125, "step": 1002 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 876.0, "epoch": 0.30083983203359327, "grad_norm": 4.690615177154541, "learning_rate": 7.932061084569496e-06, "logp/chosen": -264.0, "logp/rejected": -378.0, "loss": 0.39888763427734375, "reward/accuracy": 0.6875, "reward/chosen": -1.25, "reward/margin": 2.328125, "reward/rejected": -3.578125, "step": 1003 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 1064.0, "epoch": 0.3011397720455909, "grad_norm": 3.747795820236206, "learning_rate": 7.928243451376786e-06, "logp/chosen": -328.0, "logp/rejected": -316.0, "loss": 0.4248199462890625, "reward/accuracy": 0.875, "reward/chosen": -1.609375, "reward/margin": 2.703125, "reward/rejected": -4.3125, "step": 1004 }, { "approx. KL/chosen": 218.0, "approx. KL/rejected": 1088.0, "epoch": 0.30143971205758846, "grad_norm": 3.8141279220581055, "learning_rate": 7.924423218169972e-06, "logp/chosen": -358.0, "logp/rejected": -398.0, "loss": 0.4075927734375, "reward/accuracy": 0.8125, "reward/chosen": -1.5859375, "reward/margin": 2.328125, "reward/rejected": -3.90625, "step": 1005 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 1168.0, "epoch": 0.3017396520695861, "grad_norm": 3.190629482269287, "learning_rate": 7.920600388341084e-06, "logp/chosen": -196.0, "logp/rejected": -292.0, "loss": 0.24323654174804688, "reward/accuracy": 0.875, "reward/chosen": -1.3359375, "reward/margin": 3.03125, "reward/rejected": -4.375, "step": 1006 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 856.0, "epoch": 0.3020395920815837, "grad_norm": 3.4342544078826904, "learning_rate": 7.91677496528444e-06, "logp/chosen": -416.0, "logp/rejected": -312.0, "loss": 0.403564453125, "reward/accuracy": 0.6875, "reward/chosen": -1.78125, "reward/margin": 1.53125, "reward/rejected": -3.3125, "step": 1007 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 828.0, "epoch": 0.3023395320935813, "grad_norm": 6.018606662750244, "learning_rate": 7.91294695239667e-06, "logp/chosen": -247.0, "logp/rejected": -220.0, "loss": 0.60003662109375, "reward/accuracy": 0.6875, "reward/chosen": -1.703125, "reward/margin": 1.6328125, "reward/rejected": -3.34375, "step": 1008 }, { "approx. KL/chosen": 356.0, "approx. KL/rejected": 1160.0, "epoch": 0.3026394721055789, "grad_norm": 5.3238067626953125, "learning_rate": 7.909116353076704e-06, "logp/chosen": -378.0, "logp/rejected": -298.0, "loss": 0.3973236083984375, "reward/accuracy": 0.8125, "reward/chosen": -2.125, "reward/margin": 2.1875, "reward/rejected": -4.3125, "step": 1009 }, { "approx. KL/chosen": 244.0, "approx. KL/rejected": 840.0, "epoch": 0.30293941211757647, "grad_norm": 6.768001079559326, "learning_rate": 7.905283170725764e-06, "logp/chosen": -268.0, "logp/rejected": -314.0, "loss": 0.574951171875, "reward/accuracy": 0.6875, "reward/chosen": -1.8828125, "reward/margin": 1.5546875, "reward/rejected": -3.4375, "step": 1010 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 820.0, "epoch": 0.3032393521295741, "grad_norm": 4.379274845123291, "learning_rate": 7.901447408747369e-06, "logp/chosen": -318.0, "logp/rejected": -292.0, "loss": 0.455718994140625, "reward/accuracy": 0.8125, "reward/chosen": -1.8203125, "reward/margin": 1.5546875, "reward/rejected": -3.375, "step": 1011 }, { "approx. KL/chosen": 416.0, "approx. KL/rejected": 928.0, "epoch": 0.30353929214157166, "grad_norm": 5.094761848449707, "learning_rate": 7.897609070547324e-06, "logp/chosen": -396.0, "logp/rejected": -430.0, "loss": 0.58245849609375, "reward/accuracy": 0.75, "reward/chosen": -2.15625, "reward/margin": 1.3515625, "reward/rejected": -3.515625, "step": 1012 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 688.0, "epoch": 0.3038392321535693, "grad_norm": 4.942580223083496, "learning_rate": 7.893768159533727e-06, "logp/chosen": -296.0, "logp/rejected": -344.0, "loss": 0.5835723876953125, "reward/accuracy": 0.75, "reward/chosen": -1.84375, "reward/margin": 1.546875, "reward/rejected": -3.390625, "step": 1013 }, { "approx. KL/chosen": 229.0, "approx. KL/rejected": 1544.0, "epoch": 0.3041391721655669, "grad_norm": 2.71559476852417, "learning_rate": 7.889924679116959e-06, "logp/chosen": -245.0, "logp/rejected": -400.0, "loss": 0.22310638427734375, "reward/accuracy": 0.9375, "reward/chosen": -1.9921875, "reward/margin": 2.921875, "reward/rejected": -4.90625, "step": 1014 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 552.0, "epoch": 0.3044391121775645, "grad_norm": 3.471890449523926, "learning_rate": 7.886078632709678e-06, "logp/chosen": -376.0, "logp/rejected": -362.0, "loss": 0.381256103515625, "reward/accuracy": 0.875, "reward/chosen": -1.6015625, "reward/margin": 1.421875, "reward/rejected": -3.015625, "step": 1015 }, { "approx. KL/chosen": 132.0, "approx. KL/rejected": 604.0, "epoch": 0.3047390521895621, "grad_norm": 3.013066530227661, "learning_rate": 7.882230023726829e-06, "logp/chosen": -242.0, "logp/rejected": -258.0, "loss": 0.26416015625, "reward/accuracy": 0.875, "reward/chosen": -1.0546875, "reward/margin": 1.921875, "reward/rejected": -2.96875, "step": 1016 }, { "approx. KL/chosen": 314.0, "approx. KL/rejected": 752.0, "epoch": 0.30503899220155967, "grad_norm": 4.673606872558594, "learning_rate": 7.87837885558562e-06, "logp/chosen": -364.0, "logp/rejected": -432.0, "loss": 0.6697998046875, "reward/accuracy": 0.6875, "reward/chosen": -1.7890625, "reward/margin": 1.4453125, "reward/rejected": -3.234375, "step": 1017 }, { "approx. KL/chosen": 398.0, "approx. KL/rejected": 1200.0, "epoch": 0.3053389322135573, "grad_norm": 7.829784393310547, "learning_rate": 7.874525131705543e-06, "logp/chosen": -322.0, "logp/rejected": -312.0, "loss": 0.47760772705078125, "reward/accuracy": 0.75, "reward/chosen": -2.25, "reward/margin": 2.015625, "reward/rejected": -4.25, "step": 1018 }, { "approx. KL/chosen": 376.0, "approx. KL/rejected": 600.0, "epoch": 0.3056388722255549, "grad_norm": 4.832530975341797, "learning_rate": 7.870668855508354e-06, "logp/chosen": -376.0, "logp/rejected": -310.0, "loss": 0.71441650390625, "reward/accuracy": 0.6875, "reward/chosen": -2.140625, "reward/margin": 0.87890625, "reward/rejected": -3.03125, "step": 1019 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 932.0, "epoch": 0.3059388122375525, "grad_norm": 5.4601569175720215, "learning_rate": 7.866810030418074e-06, "logp/chosen": -398.0, "logp/rejected": -348.0, "loss": 0.6156005859375, "reward/accuracy": 0.5625, "reward/chosen": -2.1875, "reward/margin": 1.4609375, "reward/rejected": -3.65625, "step": 1020 }, { "approx. KL/chosen": 404.0, "approx. KL/rejected": 494.0, "epoch": 0.3062387522495501, "grad_norm": 8.132498741149902, "learning_rate": 7.862948659860994e-06, "logp/chosen": -410.0, "logp/rejected": -328.0, "loss": 0.73779296875, "reward/accuracy": 0.625, "reward/chosen": -2.1875, "reward/margin": 0.5, "reward/rejected": -2.6875, "step": 1021 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 1024.0, "epoch": 0.3065386922615477, "grad_norm": 3.7211031913757324, "learning_rate": 7.859084747265655e-06, "logp/chosen": -360.0, "logp/rejected": -340.0, "loss": 0.3218498229980469, "reward/accuracy": 0.8125, "reward/chosen": -1.8046875, "reward/margin": 2.34375, "reward/rejected": -4.15625, "step": 1022 }, { "approx. KL/chosen": 596.0, "approx. KL/rejected": 1456.0, "epoch": 0.3068386322735453, "grad_norm": 4.772787570953369, "learning_rate": 7.855218296062862e-06, "logp/chosen": -300.0, "logp/rejected": -314.0, "loss": 0.396484375, "reward/accuracy": 0.8125, "reward/chosen": -2.71875, "reward/margin": 2.15625, "reward/rejected": -4.875, "step": 1023 }, { "approx. KL/chosen": 454.0, "approx. KL/rejected": 864.0, "epoch": 0.30713857228554287, "grad_norm": 4.953396320343018, "learning_rate": 7.851349309685672e-06, "logp/chosen": -406.0, "logp/rejected": -460.0, "loss": 0.5567626953125, "reward/accuracy": 0.8125, "reward/chosen": -2.5625, "reward/margin": 1.1015625, "reward/rejected": -3.671875, "step": 1024 }, { "approx. KL/chosen": 416.0, "approx. KL/rejected": 804.0, "epoch": 0.3074385122975405, "grad_norm": 4.653481483459473, "learning_rate": 7.847477791569395e-06, "logp/chosen": -219.0, "logp/rejected": -249.0, "loss": 0.62860107421875, "reward/accuracy": 0.625, "reward/chosen": -2.171875, "reward/margin": 1.3671875, "reward/rejected": -3.53125, "step": 1025 }, { "approx. KL/chosen": 328.0, "approx. KL/rejected": 1216.0, "epoch": 0.3077384523095381, "grad_norm": 3.2253336906433105, "learning_rate": 7.843603745151584e-06, "logp/chosen": -404.0, "logp/rejected": -328.0, "loss": 0.3533973693847656, "reward/accuracy": 0.8125, "reward/chosen": -2.1875, "reward/margin": 2.203125, "reward/rejected": -4.375, "step": 1026 }, { "approx. KL/chosen": 432.0, "approx. KL/rejected": 1248.0, "epoch": 0.3080383923215357, "grad_norm": 2.7515010833740234, "learning_rate": 7.839727173872043e-06, "logp/chosen": -342.0, "logp/rejected": -318.0, "loss": 0.242218017578125, "reward/accuracy": 0.9375, "reward/chosen": -2.78125, "reward/margin": 2.046875, "reward/rejected": -4.8125, "step": 1027 }, { "approx. KL/chosen": 416.0, "approx. KL/rejected": 592.0, "epoch": 0.3083383323335333, "grad_norm": 7.324713230133057, "learning_rate": 7.835848081172814e-06, "logp/chosen": -412.0, "logp/rejected": -386.0, "loss": 0.88714599609375, "reward/accuracy": 0.625, "reward/chosen": -2.375, "reward/margin": 0.625, "reward/rejected": -3.0, "step": 1028 }, { "approx. KL/chosen": 318.0, "approx. KL/rejected": 940.0, "epoch": 0.3086382723455309, "grad_norm": 4.3312506675720215, "learning_rate": 7.83196647049818e-06, "logp/chosen": -324.0, "logp/rejected": -342.0, "loss": 0.476043701171875, "reward/accuracy": 0.6875, "reward/chosen": -1.9453125, "reward/margin": 2.015625, "reward/rejected": -3.953125, "step": 1029 }, { "approx. KL/chosen": 364.0, "approx. KL/rejected": 996.0, "epoch": 0.3089382123575285, "grad_norm": 6.858785152435303, "learning_rate": 7.828082345294656e-06, "logp/chosen": -217.0, "logp/rejected": -430.0, "loss": 0.491363525390625, "reward/accuracy": 0.8125, "reward/chosen": -2.34375, "reward/margin": 1.7109375, "reward/rejected": -4.03125, "step": 1030 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 1000.0, "epoch": 0.3092381523695261, "grad_norm": 3.3040049076080322, "learning_rate": 7.824195709010995e-06, "logp/chosen": -364.0, "logp/rejected": -256.0, "loss": 0.30767822265625, "reward/accuracy": 0.8125, "reward/chosen": -2.078125, "reward/margin": 2.125, "reward/rejected": -4.1875, "step": 1031 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 924.0, "epoch": 0.3095380923815237, "grad_norm": 2.9833221435546875, "learning_rate": 7.820306565098175e-06, "logp/chosen": -330.0, "logp/rejected": -384.0, "loss": 0.2547454833984375, "reward/accuracy": 0.9375, "reward/chosen": -1.5390625, "reward/margin": 2.296875, "reward/rejected": -3.84375, "step": 1032 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 1040.0, "epoch": 0.3098380323935213, "grad_norm": 3.8433308601379395, "learning_rate": 7.816414917009404e-06, "logp/chosen": -382.0, "logp/rejected": -354.0, "loss": 0.43474578857421875, "reward/accuracy": 0.8125, "reward/chosen": -1.8203125, "reward/margin": 2.359375, "reward/rejected": -4.1875, "step": 1033 }, { "approx. KL/chosen": 490.0, "approx. KL/rejected": 1320.0, "epoch": 0.3101379724055189, "grad_norm": 2.944399833679199, "learning_rate": 7.812520768200109e-06, "logp/chosen": -251.0, "logp/rejected": -290.0, "loss": 0.2852783203125, "reward/accuracy": 0.875, "reward/chosen": -2.890625, "reward/margin": 1.84375, "reward/rejected": -4.75, "step": 1034 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 912.0, "epoch": 0.3104379124175165, "grad_norm": 2.9089155197143555, "learning_rate": 7.808624122127945e-06, "logp/chosen": -396.0, "logp/rejected": -354.0, "loss": 0.2938232421875, "reward/accuracy": 0.875, "reward/chosen": -2.0, "reward/margin": 1.78125, "reward/rejected": -3.78125, "step": 1035 }, { "approx. KL/chosen": 392.0, "approx. KL/rejected": 564.0, "epoch": 0.31073785242951407, "grad_norm": 8.757598876953125, "learning_rate": 7.804724982252774e-06, "logp/chosen": -344.0, "logp/rejected": -302.0, "loss": 0.9001312255859375, "reward/accuracy": 0.5625, "reward/chosen": -2.296875, "reward/margin": 0.65625, "reward/rejected": -2.953125, "step": 1036 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 1224.0, "epoch": 0.3110377924415117, "grad_norm": 4.28113317489624, "learning_rate": 7.800823352036684e-06, "logp/chosen": -432.0, "logp/rejected": -384.0, "loss": 0.339508056640625, "reward/accuracy": 0.75, "reward/chosen": -2.296875, "reward/margin": 1.984375, "reward/rejected": -4.28125, "step": 1037 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 944.0, "epoch": 0.3113377324535093, "grad_norm": 3.609065532684326, "learning_rate": 7.796919234943963e-06, "logp/chosen": -356.0, "logp/rejected": -324.0, "loss": 0.27154541015625, "reward/accuracy": 0.9375, "reward/chosen": -2.015625, "reward/margin": 2.0625, "reward/rejected": -4.0625, "step": 1038 }, { "approx. KL/chosen": 380.0, "approx. KL/rejected": 800.0, "epoch": 0.3116376724655069, "grad_norm": 4.800187587738037, "learning_rate": 7.793012634441112e-06, "logp/chosen": -358.0, "logp/rejected": -386.0, "loss": 0.5653076171875, "reward/accuracy": 0.625, "reward/chosen": -2.28125, "reward/margin": 1.3515625, "reward/rejected": -3.625, "step": 1039 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 868.0, "epoch": 0.3119376124775045, "grad_norm": 2.8923699855804443, "learning_rate": 7.789103553996842e-06, "logp/chosen": -424.0, "logp/rejected": -414.0, "loss": 0.27508544921875, "reward/accuracy": 0.875, "reward/chosen": -1.8984375, "reward/margin": 1.9296875, "reward/rejected": -3.828125, "step": 1040 }, { "approx. KL/chosen": 316.0, "approx. KL/rejected": 840.0, "epoch": 0.3122375524895021, "grad_norm": 5.924295425415039, "learning_rate": 7.785191997082057e-06, "logp/chosen": -520.0, "logp/rejected": -418.0, "loss": 0.384521484375, "reward/accuracy": 0.875, "reward/chosen": -1.890625, "reward/margin": 1.8515625, "reward/rejected": -3.75, "step": 1041 }, { "approx. KL/chosen": 174.0, "approx. KL/rejected": 896.0, "epoch": 0.3125374925014997, "grad_norm": 4.083268642425537, "learning_rate": 7.781277967169868e-06, "logp/chosen": -336.0, "logp/rejected": -438.0, "loss": 0.36074066162109375, "reward/accuracy": 0.875, "reward/chosen": -1.640625, "reward/margin": 2.1875, "reward/rejected": -3.828125, "step": 1042 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 780.0, "epoch": 0.3128374325134973, "grad_norm": 3.606204032897949, "learning_rate": 7.777361467735574e-06, "logp/chosen": -348.0, "logp/rejected": -350.0, "loss": 0.36834716796875, "reward/accuracy": 0.75, "reward/chosen": -1.9296875, "reward/margin": 1.7421875, "reward/rejected": -3.671875, "step": 1043 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 880.0, "epoch": 0.3131373725254949, "grad_norm": 3.382415533065796, "learning_rate": 7.773442502256672e-06, "logp/chosen": -282.0, "logp/rejected": -342.0, "loss": 0.299652099609375, "reward/accuracy": 0.875, "reward/chosen": -1.7890625, "reward/margin": 2.140625, "reward/rejected": -3.9375, "step": 1044 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 652.0, "epoch": 0.3134373125374925, "grad_norm": 4.691741466522217, "learning_rate": 7.769521074212847e-06, "logp/chosen": -282.0, "logp/rejected": -258.0, "loss": 0.4482421875, "reward/accuracy": 0.75, "reward/chosen": -1.5859375, "reward/margin": 1.71875, "reward/rejected": -3.296875, "step": 1045 }, { "approx. KL/chosen": 247.0, "approx. KL/rejected": 824.0, "epoch": 0.3137372525494901, "grad_norm": 4.219309329986572, "learning_rate": 7.765597187085972e-06, "logp/chosen": -438.0, "logp/rejected": -378.0, "loss": 0.45725250244140625, "reward/accuracy": 0.8125, "reward/chosen": -1.8046875, "reward/margin": 1.90625, "reward/rejected": -3.703125, "step": 1046 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 744.0, "epoch": 0.3140371925614877, "grad_norm": 4.041093349456787, "learning_rate": 7.761670844360101e-06, "logp/chosen": -260.0, "logp/rejected": -324.0, "loss": 0.3613128662109375, "reward/accuracy": 0.8125, "reward/chosen": -1.7734375, "reward/margin": 1.65625, "reward/rejected": -3.4375, "step": 1047 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 1064.0, "epoch": 0.3143371325734853, "grad_norm": 3.3012726306915283, "learning_rate": 7.757742049521471e-06, "logp/chosen": -362.0, "logp/rejected": -342.0, "loss": 0.270233154296875, "reward/accuracy": 0.8125, "reward/chosen": -1.796875, "reward/margin": 2.25, "reward/rejected": -4.0625, "step": 1048 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 784.0, "epoch": 0.3146370725854829, "grad_norm": 4.253695964813232, "learning_rate": 7.753810806058493e-06, "logp/chosen": -224.0, "logp/rejected": -318.0, "loss": 0.3979949951171875, "reward/accuracy": 0.875, "reward/chosen": -1.5, "reward/margin": 1.75, "reward/rejected": -3.25, "step": 1049 }, { "approx. KL/chosen": 338.0, "approx. KL/rejected": 1168.0, "epoch": 0.3149370125974805, "grad_norm": 3.5781095027923584, "learning_rate": 7.749877117461752e-06, "logp/chosen": -368.0, "logp/rejected": -356.0, "loss": 0.308349609375, "reward/accuracy": 0.8125, "reward/chosen": -2.171875, "reward/margin": 2.234375, "reward/rejected": -4.40625, "step": 1050 }, { "approx. KL/chosen": 464.0, "approx. KL/rejected": 744.0, "epoch": 0.3152369526094781, "grad_norm": 6.436107635498047, "learning_rate": 7.745940987224013e-06, "logp/chosen": -364.0, "logp/rejected": -374.0, "loss": 0.91064453125, "reward/accuracy": 0.5625, "reward/chosen": -2.59375, "reward/margin": 0.6171875, "reward/rejected": -3.21875, "step": 1051 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 1136.0, "epoch": 0.3155368926214757, "grad_norm": 3.0787792205810547, "learning_rate": 7.742002418840195e-06, "logp/chosen": -304.0, "logp/rejected": -366.0, "loss": 0.266448974609375, "reward/accuracy": 0.8125, "reward/chosen": -1.78125, "reward/margin": 2.6875, "reward/rejected": -4.46875, "step": 1052 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 568.0, "epoch": 0.3158368326334733, "grad_norm": 3.2858970165252686, "learning_rate": 7.738061415807393e-06, "logp/chosen": -378.0, "logp/rejected": -368.0, "loss": 0.364501953125, "reward/accuracy": 0.8125, "reward/chosen": -1.5703125, "reward/margin": 1.5390625, "reward/rejected": -3.109375, "step": 1053 }, { "approx. KL/chosen": 222.0, "approx. KL/rejected": 976.0, "epoch": 0.3161367726454709, "grad_norm": 2.8843021392822266, "learning_rate": 7.734117981624857e-06, "logp/chosen": -266.0, "logp/rejected": -298.0, "loss": 0.26971435546875, "reward/accuracy": 0.9375, "reward/chosen": -1.8359375, "reward/margin": 2.140625, "reward/rejected": -3.984375, "step": 1054 }, { "approx. KL/chosen": 187.0, "approx. KL/rejected": 672.0, "epoch": 0.31643671265746853, "grad_norm": 4.3415117263793945, "learning_rate": 7.730172119793999e-06, "logp/chosen": -328.0, "logp/rejected": -328.0, "loss": 0.690673828125, "reward/accuracy": 0.6875, "reward/chosen": -1.6796875, "reward/margin": 1.0390625, "reward/rejected": -2.71875, "step": 1055 }, { "approx. KL/chosen": 126.5, "approx. KL/rejected": 708.0, "epoch": 0.3167366526694661, "grad_norm": 2.5916547775268555, "learning_rate": 7.726223833818386e-06, "logp/chosen": -338.0, "logp/rejected": -356.0, "loss": 0.276641845703125, "reward/accuracy": 0.8125, "reward/chosen": -1.2890625, "reward/margin": 1.9453125, "reward/rejected": -3.234375, "step": 1056 }, { "approx. KL/chosen": 386.0, "approx. KL/rejected": 1536.0, "epoch": 0.3170365926814637, "grad_norm": 5.242918968200684, "learning_rate": 7.722273127203739e-06, "logp/chosen": -171.0, "logp/rejected": -394.0, "loss": 0.20931243896484375, "reward/accuracy": 0.875, "reward/chosen": -2.0625, "reward/margin": 3.078125, "reward/rejected": -5.125, "step": 1057 }, { "approx. KL/chosen": 426.0, "approx. KL/rejected": 704.0, "epoch": 0.3173365326934613, "grad_norm": 8.456933975219727, "learning_rate": 7.718320003457924e-06, "logp/chosen": -436.0, "logp/rejected": -276.0, "loss": 0.6879730224609375, "reward/accuracy": 0.625, "reward/chosen": -2.28125, "reward/margin": 1.0, "reward/rejected": -3.28125, "step": 1058 }, { "approx. KL/chosen": 386.0, "approx. KL/rejected": 1200.0, "epoch": 0.3176364727054589, "grad_norm": 5.032470703125, "learning_rate": 7.714364466090955e-06, "logp/chosen": -258.0, "logp/rejected": -346.0, "loss": 0.4082183837890625, "reward/accuracy": 0.875, "reward/chosen": -2.0, "reward/margin": 2.40625, "reward/rejected": -4.40625, "step": 1059 }, { "approx. KL/chosen": 346.0, "approx. KL/rejected": 1240.0, "epoch": 0.3179364127174565, "grad_norm": 4.775336742401123, "learning_rate": 7.710406518614992e-06, "logp/chosen": -328.0, "logp/rejected": -288.0, "loss": 0.565765380859375, "reward/accuracy": 0.8125, "reward/chosen": -2.140625, "reward/margin": 2.40625, "reward/rejected": -4.53125, "step": 1060 }, { "approx. KL/chosen": 221.0, "approx. KL/rejected": 960.0, "epoch": 0.3182363527294541, "grad_norm": 4.143626689910889, "learning_rate": 7.706446164544334e-06, "logp/chosen": -272.0, "logp/rejected": -316.0, "loss": 0.40433502197265625, "reward/accuracy": 0.8125, "reward/chosen": -1.78125, "reward/margin": 1.9609375, "reward/rejected": -3.734375, "step": 1061 }, { "approx. KL/chosen": 442.0, "approx. KL/rejected": 1144.0, "epoch": 0.31853629274145173, "grad_norm": 5.364813327789307, "learning_rate": 7.702483407395412e-06, "logp/chosen": -286.0, "logp/rejected": -356.0, "loss": 0.59893798828125, "reward/accuracy": 0.6875, "reward/chosen": -2.453125, "reward/margin": 1.8046875, "reward/rejected": -4.25, "step": 1062 }, { "approx. KL/chosen": 420.0, "approx. KL/rejected": 1032.0, "epoch": 0.3188362327534493, "grad_norm": 2.932356119155884, "learning_rate": 7.698518250686794e-06, "logp/chosen": -288.0, "logp/rejected": -308.0, "loss": 0.3367919921875, "reward/accuracy": 0.875, "reward/chosen": -2.390625, "reward/margin": 1.6875, "reward/rejected": -4.09375, "step": 1063 }, { "approx. KL/chosen": 338.0, "approx. KL/rejected": 832.0, "epoch": 0.3191361727654469, "grad_norm": 4.48844051361084, "learning_rate": 7.694550697939183e-06, "logp/chosen": -320.0, "logp/rejected": -378.0, "loss": 0.5050048828125, "reward/accuracy": 0.6875, "reward/chosen": -2.046875, "reward/margin": 1.4375, "reward/rejected": -3.484375, "step": 1064 }, { "approx. KL/chosen": 304.0, "approx. KL/rejected": 1144.0, "epoch": 0.3194361127774445, "grad_norm": 4.227438449859619, "learning_rate": 7.6905807526754e-06, "logp/chosen": -334.0, "logp/rejected": -324.0, "loss": 0.3537101745605469, "reward/accuracy": 0.75, "reward/chosen": -1.828125, "reward/margin": 2.421875, "reward/rejected": -4.25, "step": 1065 }, { "approx. KL/chosen": 386.0, "approx. KL/rejected": 884.0, "epoch": 0.3197360527894421, "grad_norm": 5.729616165161133, "learning_rate": 7.6866084184204e-06, "logp/chosen": -308.0, "logp/rejected": -502.0, "loss": 0.5708866119384766, "reward/accuracy": 0.5625, "reward/chosen": -1.859375, "reward/margin": 1.7109375, "reward/rejected": -3.578125, "step": 1066 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 904.0, "epoch": 0.32003599280143974, "grad_norm": 5.194962978363037, "learning_rate": 7.682633698701254e-06, "logp/chosen": -338.0, "logp/rejected": -388.0, "loss": 0.6253547668457031, "reward/accuracy": 0.75, "reward/chosen": -1.34375, "reward/margin": 2.125, "reward/rejected": -3.46875, "step": 1067 }, { "approx. KL/chosen": 404.0, "approx. KL/rejected": 740.0, "epoch": 0.3203359328134373, "grad_norm": 5.956152439117432, "learning_rate": 7.678656597047149e-06, "logp/chosen": -426.0, "logp/rejected": -410.0, "loss": 0.66400146484375, "reward/accuracy": 0.75, "reward/chosen": -1.890625, "reward/margin": 1.4296875, "reward/rejected": -3.3125, "step": 1068 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 628.0, "epoch": 0.32063587282543493, "grad_norm": 4.055654525756836, "learning_rate": 7.674677116989395e-06, "logp/chosen": -270.0, "logp/rejected": -312.0, "loss": 0.4874267578125, "reward/accuracy": 0.6875, "reward/chosen": -1.5, "reward/margin": 1.5859375, "reward/rejected": -3.078125, "step": 1069 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 660.0, "epoch": 0.3209358128374325, "grad_norm": 4.412778377532959, "learning_rate": 7.670695262061403e-06, "logp/chosen": -356.0, "logp/rejected": -420.0, "loss": 0.443328857421875, "reward/accuracy": 0.6875, "reward/chosen": -1.484375, "reward/margin": 1.5078125, "reward/rejected": -3.0, "step": 1070 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 428.0, "epoch": 0.3212357528494301, "grad_norm": 4.021968841552734, "learning_rate": 7.666711035798705e-06, "logp/chosen": -366.0, "logp/rejected": -398.0, "loss": 0.43511962890625, "reward/accuracy": 0.75, "reward/chosen": -1.328125, "reward/margin": 1.1796875, "reward/rejected": -2.515625, "step": 1071 }, { "approx. KL/chosen": 123.0, "approx. KL/rejected": 1072.0, "epoch": 0.3215356928614277, "grad_norm": 2.5877397060394287, "learning_rate": 7.662724441738925e-06, "logp/chosen": -304.0, "logp/rejected": -342.0, "loss": 0.189208984375, "reward/accuracy": 0.9375, "reward/chosen": -1.1640625, "reward/margin": 2.96875, "reward/rejected": -4.125, "step": 1072 }, { "approx. KL/chosen": 87.5, "approx. KL/rejected": 446.0, "epoch": 0.3218356328734253, "grad_norm": 4.996105194091797, "learning_rate": 7.6587354834218e-06, "logp/chosen": -390.0, "logp/rejected": -350.0, "loss": 0.4759521484375, "reward/accuracy": 0.75, "reward/chosen": -0.97265625, "reward/margin": 1.265625, "reward/rejected": -2.234375, "step": 1073 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 876.0, "epoch": 0.32213557288542294, "grad_norm": 4.295595645904541, "learning_rate": 7.654744164389161e-06, "logp/chosen": -288.0, "logp/rejected": -296.0, "loss": 0.387939453125, "reward/accuracy": 0.8125, "reward/chosen": -1.6171875, "reward/margin": 2.140625, "reward/rejected": -3.765625, "step": 1074 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 588.0, "epoch": 0.3224355128974205, "grad_norm": 5.251885414123535, "learning_rate": 7.650750488184937e-06, "logp/chosen": -280.0, "logp/rejected": -247.0, "loss": 0.5809326171875, "reward/accuracy": 0.625, "reward/chosen": -1.640625, "reward/margin": 0.8515625, "reward/rejected": -2.484375, "step": 1075 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 452.0, "epoch": 0.32273545290941813, "grad_norm": 3.4261953830718994, "learning_rate": 7.646754458355152e-06, "logp/chosen": -318.0, "logp/rejected": -354.0, "loss": 0.3665771484375, "reward/accuracy": 0.8125, "reward/chosen": -1.09375, "reward/margin": 1.3984375, "reward/rejected": -2.5, "step": 1076 }, { "approx. KL/chosen": 334.0, "approx. KL/rejected": 1288.0, "epoch": 0.3230353929214157, "grad_norm": 4.430963039398193, "learning_rate": 7.642756078447914e-06, "logp/chosen": -402.0, "logp/rejected": -382.0, "loss": 0.44673919677734375, "reward/accuracy": 0.875, "reward/chosen": -1.84375, "reward/margin": 2.53125, "reward/rejected": -4.375, "step": 1077 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 394.0, "epoch": 0.3233353329334133, "grad_norm": 5.941277027130127, "learning_rate": 7.638755352013424e-06, "logp/chosen": -268.0, "logp/rejected": -354.0, "loss": 0.66717529296875, "reward/accuracy": 0.6875, "reward/chosen": -1.3359375, "reward/margin": 1.109375, "reward/rejected": -2.453125, "step": 1078 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 624.0, "epoch": 0.3236352729454109, "grad_norm": 3.3124794960021973, "learning_rate": 7.634752282603961e-06, "logp/chosen": -312.0, "logp/rejected": -324.0, "loss": 0.41986083984375, "reward/accuracy": 0.875, "reward/chosen": -1.3359375, "reward/margin": 1.390625, "reward/rejected": -2.71875, "step": 1079 }, { "approx. KL/chosen": 97.5, "approx. KL/rejected": 708.0, "epoch": 0.3239352129574085, "grad_norm": 2.9945290088653564, "learning_rate": 7.630746873773888e-06, "logp/chosen": -402.0, "logp/rejected": -422.0, "loss": 0.2991485595703125, "reward/accuracy": 0.9375, "reward/chosen": -0.8671875, "reward/margin": 2.421875, "reward/rejected": -3.296875, "step": 1080 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 672.0, "epoch": 0.32423515296940614, "grad_norm": 4.547024726867676, "learning_rate": 7.626739129079643e-06, "logp/chosen": -217.0, "logp/rejected": -288.0, "loss": 0.56121826171875, "reward/accuracy": 0.625, "reward/chosen": -1.3984375, "reward/margin": 1.5234375, "reward/rejected": -2.921875, "step": 1081 }, { "approx. KL/chosen": 103.5, "approx. KL/rejected": 412.0, "epoch": 0.3245350929814037, "grad_norm": 3.535050392150879, "learning_rate": 7.62272905207974e-06, "logp/chosen": -242.0, "logp/rejected": -322.0, "loss": 0.35943603515625, "reward/accuracy": 0.75, "reward/chosen": -1.140625, "reward/margin": 1.515625, "reward/rejected": -2.65625, "step": 1082 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 516.0, "epoch": 0.32483503299340133, "grad_norm": 4.4857659339904785, "learning_rate": 7.6187166463347635e-06, "logp/chosen": -394.0, "logp/rejected": -370.0, "loss": 0.4000244140625, "reward/accuracy": 0.75, "reward/chosen": -1.4375, "reward/margin": 1.40625, "reward/rejected": -2.84375, "step": 1083 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 856.0, "epoch": 0.3251349730053989, "grad_norm": 5.583559513092041, "learning_rate": 7.614701915407366e-06, "logp/chosen": -235.0, "logp/rejected": -334.0, "loss": 0.4251060485839844, "reward/accuracy": 0.75, "reward/chosen": -1.515625, "reward/margin": 1.953125, "reward/rejected": -3.46875, "step": 1084 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 828.0, "epoch": 0.3254349130173965, "grad_norm": 4.177850246429443, "learning_rate": 7.610684862862261e-06, "logp/chosen": -300.0, "logp/rejected": -568.0, "loss": 0.3950042724609375, "reward/accuracy": 0.6875, "reward/chosen": -1.234375, "reward/margin": 2.15625, "reward/rejected": -3.390625, "step": 1085 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 668.0, "epoch": 0.32573485302939414, "grad_norm": 3.933288097381592, "learning_rate": 7.606665492266228e-06, "logp/chosen": -350.0, "logp/rejected": -402.0, "loss": 0.39067840576171875, "reward/accuracy": 0.8125, "reward/chosen": -1.2265625, "reward/margin": 1.953125, "reward/rejected": -3.171875, "step": 1086 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 628.0, "epoch": 0.3260347930413917, "grad_norm": 3.919003963470459, "learning_rate": 7.602643807188101e-06, "logp/chosen": -192.0, "logp/rejected": -248.0, "loss": 0.456787109375, "reward/accuracy": 0.875, "reward/chosen": -1.3828125, "reward/margin": 1.53125, "reward/rejected": -2.921875, "step": 1087 }, { "approx. KL/chosen": 148.0, "approx. KL/rejected": 624.0, "epoch": 0.32633473305338934, "grad_norm": 3.085798501968384, "learning_rate": 7.598619811198773e-06, "logp/chosen": -390.0, "logp/rejected": -432.0, "loss": 0.312744140625, "reward/accuracy": 0.8125, "reward/chosen": -1.0078125, "reward/margin": 2.0, "reward/rejected": -3.0, "step": 1088 }, { "approx. KL/chosen": 123.0, "approx. KL/rejected": 372.0, "epoch": 0.3266346730653869, "grad_norm": 5.537982940673828, "learning_rate": 7.5945935078711865e-06, "logp/chosen": -203.0, "logp/rejected": -206.0, "loss": 0.4923095703125, "reward/accuracy": 0.75, "reward/chosen": -1.1328125, "reward/margin": 1.09375, "reward/rejected": -2.21875, "step": 1089 }, { "approx. KL/chosen": 94.5, "approx. KL/rejected": 560.0, "epoch": 0.32693461307738453, "grad_norm": 2.132340669631958, "learning_rate": 7.5905649007803324e-06, "logp/chosen": -348.0, "logp/rejected": -292.0, "loss": 0.1771240234375, "reward/accuracy": 0.9375, "reward/chosen": -0.8671875, "reward/margin": 2.296875, "reward/rejected": -3.171875, "step": 1090 }, { "approx. KL/chosen": 180.0, "approx. KL/rejected": 820.0, "epoch": 0.3272345530893821, "grad_norm": 5.107758522033691, "learning_rate": 7.586533993503249e-06, "logp/chosen": -304.0, "logp/rejected": -288.0, "loss": 0.6080780029296875, "reward/accuracy": 0.5625, "reward/chosen": -1.34375, "reward/margin": 2.03125, "reward/rejected": -3.375, "step": 1091 }, { "approx. KL/chosen": 175.0, "approx. KL/rejected": 836.0, "epoch": 0.3275344931013797, "grad_norm": 4.80789041519165, "learning_rate": 7.582500789619014e-06, "logp/chosen": -380.0, "logp/rejected": -402.0, "loss": 0.319732666015625, "reward/accuracy": 0.8125, "reward/chosen": -1.4921875, "reward/margin": 2.234375, "reward/rejected": -3.71875, "step": 1092 }, { "approx. KL/chosen": 241.0, "approx. KL/rejected": 1004.0, "epoch": 0.32783443311337734, "grad_norm": 4.179173946380615, "learning_rate": 7.578465292708746e-06, "logp/chosen": -360.0, "logp/rejected": -418.0, "loss": 0.512542724609375, "reward/accuracy": 0.6875, "reward/chosen": -1.53125, "reward/margin": 2.03125, "reward/rejected": -3.5625, "step": 1093 }, { "approx. KL/chosen": 470.0, "approx. KL/rejected": 1352.0, "epoch": 0.3281343731253749, "grad_norm": 7.350104331970215, "learning_rate": 7.574427506355599e-06, "logp/chosen": -406.0, "logp/rejected": -364.0, "loss": 0.555145263671875, "reward/accuracy": 0.8125, "reward/chosen": -2.234375, "reward/margin": 2.3125, "reward/rejected": -4.5625, "step": 1094 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 976.0, "epoch": 0.32843431313737254, "grad_norm": 4.0483479499816895, "learning_rate": 7.570387434144765e-06, "logp/chosen": -406.0, "logp/rejected": -386.0, "loss": 0.5396728515625, "reward/accuracy": 0.8125, "reward/chosen": -1.7890625, "reward/margin": 1.9296875, "reward/rejected": -3.71875, "step": 1095 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 616.0, "epoch": 0.3287342531493701, "grad_norm": 4.548734664916992, "learning_rate": 7.566345079663455e-06, "logp/chosen": -239.0, "logp/rejected": -346.0, "loss": 0.53338623046875, "reward/accuracy": 0.875, "reward/chosen": -1.46875, "reward/margin": 1.5078125, "reward/rejected": -2.96875, "step": 1096 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 804.0, "epoch": 0.3290341931613677, "grad_norm": 5.672098159790039, "learning_rate": 7.5623004465009155e-06, "logp/chosen": -350.0, "logp/rejected": -384.0, "loss": 0.6374053955078125, "reward/accuracy": 0.5625, "reward/chosen": -1.8984375, "reward/margin": 1.3359375, "reward/rejected": -3.234375, "step": 1097 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 528.0, "epoch": 0.32933413317336535, "grad_norm": 5.953968524932861, "learning_rate": 7.558253538248413e-06, "logp/chosen": -328.0, "logp/rejected": -354.0, "loss": 0.67083740234375, "reward/accuracy": 0.625, "reward/chosen": -1.9140625, "reward/margin": 0.984375, "reward/rejected": -2.90625, "step": 1098 }, { "approx. KL/chosen": 205.0, "approx. KL/rejected": 888.0, "epoch": 0.3296340731853629, "grad_norm": 4.3125715255737305, "learning_rate": 7.554204358499232e-06, "logp/chosen": -278.0, "logp/rejected": -376.0, "loss": 0.5536880493164062, "reward/accuracy": 0.6875, "reward/chosen": -1.703125, "reward/margin": 1.828125, "reward/rejected": -3.53125, "step": 1099 }, { "approx. KL/chosen": 248.0, "approx. KL/rejected": 1004.0, "epoch": 0.32993401319736054, "grad_norm": 5.877591609954834, "learning_rate": 7.55015291084868e-06, "logp/chosen": -360.0, "logp/rejected": -424.0, "loss": 0.331024169921875, "reward/accuracy": 0.875, "reward/chosen": -1.5390625, "reward/margin": 2.40625, "reward/rejected": -3.9375, "step": 1100 }, { "approx. KL/chosen": 138.0, "approx. KL/rejected": 880.0, "epoch": 0.3302339532093581, "grad_norm": 3.8479743003845215, "learning_rate": 7.546099198894072e-06, "logp/chosen": -380.0, "logp/rejected": -384.0, "loss": 0.2948760986328125, "reward/accuracy": 0.8125, "reward/chosen": -1.046875, "reward/margin": 2.71875, "reward/rejected": -3.765625, "step": 1101 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 708.0, "epoch": 0.33053389322135573, "grad_norm": 3.5783333778381348, "learning_rate": 7.542043226234736e-06, "logp/chosen": -260.0, "logp/rejected": -308.0, "loss": 0.376129150390625, "reward/accuracy": 0.75, "reward/chosen": -1.265625, "reward/margin": 1.84375, "reward/rejected": -3.109375, "step": 1102 }, { "approx. KL/chosen": 209.0, "approx. KL/rejected": 532.0, "epoch": 0.3308338332333533, "grad_norm": 4.365312576293945, "learning_rate": 7.537984996472009e-06, "logp/chosen": -316.0, "logp/rejected": -390.0, "loss": 0.471923828125, "reward/accuracy": 0.8125, "reward/chosen": -1.4140625, "reward/margin": 1.2890625, "reward/rejected": -2.703125, "step": 1103 }, { "approx. KL/chosen": 111.0, "approx. KL/rejected": 1040.0, "epoch": 0.3311337732453509, "grad_norm": 3.3901045322418213, "learning_rate": 7.533924513209228e-06, "logp/chosen": -420.0, "logp/rejected": -482.0, "loss": 0.4799346923828125, "reward/accuracy": 0.75, "reward/chosen": -1.0546875, "reward/margin": 2.421875, "reward/rejected": -3.484375, "step": 1104 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 900.0, "epoch": 0.33143371325734855, "grad_norm": 2.206024169921875, "learning_rate": 7.529861780051733e-06, "logp/chosen": -240.0, "logp/rejected": -358.0, "loss": 0.22711944580078125, "reward/accuracy": 0.875, "reward/chosen": -1.1640625, "reward/margin": 2.65625, "reward/rejected": -3.8125, "step": 1105 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 564.0, "epoch": 0.3317336532693461, "grad_norm": 4.963653087615967, "learning_rate": 7.525796800606864e-06, "logp/chosen": -362.0, "logp/rejected": -470.0, "loss": 0.54132080078125, "reward/accuracy": 0.5625, "reward/chosen": -1.65625, "reward/margin": 1.3046875, "reward/rejected": -2.953125, "step": 1106 }, { "approx. KL/chosen": 88.0, "approx. KL/rejected": 572.0, "epoch": 0.33203359328134374, "grad_norm": 3.254586935043335, "learning_rate": 7.521729578483951e-06, "logp/chosen": -298.0, "logp/rejected": -306.0, "loss": 0.44732666015625, "reward/accuracy": 0.6875, "reward/chosen": -0.89453125, "reward/margin": 1.65625, "reward/rejected": -2.546875, "step": 1107 }, { "approx. KL/chosen": 174.0, "approx. KL/rejected": 584.0, "epoch": 0.3323335332933413, "grad_norm": 4.669183731079102, "learning_rate": 7.517660117294319e-06, "logp/chosen": -462.0, "logp/rejected": -362.0, "loss": 0.54278564453125, "reward/accuracy": 0.625, "reward/chosen": -1.5859375, "reward/margin": 1.2109375, "reward/rejected": -2.796875, "step": 1108 }, { "approx. KL/chosen": 245.0, "approx. KL/rejected": 680.0, "epoch": 0.33263347330533893, "grad_norm": 4.916403770446777, "learning_rate": 7.513588420651281e-06, "logp/chosen": -338.0, "logp/rejected": -394.0, "loss": 0.4353179931640625, "reward/accuracy": 0.6875, "reward/chosen": -1.7265625, "reward/margin": 1.453125, "reward/rejected": -3.171875, "step": 1109 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 764.0, "epoch": 0.33293341331733656, "grad_norm": 5.77745246887207, "learning_rate": 7.509514492170131e-06, "logp/chosen": -456.0, "logp/rejected": -382.0, "loss": 0.7045249938964844, "reward/accuracy": 0.625, "reward/chosen": -1.875, "reward/margin": 1.5546875, "reward/rejected": -3.4375, "step": 1110 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 528.0, "epoch": 0.3332333533293341, "grad_norm": 5.284433841705322, "learning_rate": 7.505438335468148e-06, "logp/chosen": -226.0, "logp/rejected": -255.0, "loss": 0.50921630859375, "reward/accuracy": 0.8125, "reward/chosen": -1.7265625, "reward/margin": 1.140625, "reward/rejected": -2.875, "step": 1111 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 484.0, "epoch": 0.33353329334133175, "grad_norm": 4.402475833892822, "learning_rate": 7.501359954164591e-06, "logp/chosen": -266.0, "logp/rejected": -428.0, "loss": 0.4608154296875, "reward/accuracy": 0.875, "reward/chosen": -0.9765625, "reward/margin": 1.453125, "reward/rejected": -2.4375, "step": 1112 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 398.0, "epoch": 0.3338332333533293, "grad_norm": 6.295234203338623, "learning_rate": 7.4972793518806885e-06, "logp/chosen": -308.0, "logp/rejected": -320.0, "loss": 0.721923828125, "reward/accuracy": 0.5, "reward/chosen": -1.7109375, "reward/margin": 0.72265625, "reward/rejected": -2.4375, "step": 1113 }, { "approx. KL/chosen": 117.0, "approx. KL/rejected": 268.0, "epoch": 0.33413317336532694, "grad_norm": 5.305148601531982, "learning_rate": 7.493196532239648e-06, "logp/chosen": -318.0, "logp/rejected": -302.0, "loss": 0.766845703125, "reward/accuracy": 0.5625, "reward/chosen": -1.2109375, "reward/margin": 0.5, "reward/rejected": -1.7109375, "step": 1114 }, { "approx. KL/chosen": 100.5, "approx. KL/rejected": 498.0, "epoch": 0.3344331133773245, "grad_norm": 3.6976747512817383, "learning_rate": 7.489111498866639e-06, "logp/chosen": -205.0, "logp/rejected": -237.0, "loss": 0.427215576171875, "reward/accuracy": 0.8125, "reward/chosen": -0.60546875, "reward/margin": 2.0625, "reward/rejected": -2.671875, "step": 1115 }, { "approx. KL/chosen": 59.0, "approx. KL/rejected": 394.0, "epoch": 0.33473305338932213, "grad_norm": 3.9945197105407715, "learning_rate": 7.485024255388803e-06, "logp/chosen": -245.0, "logp/rejected": -198.0, "loss": 0.52142333984375, "reward/accuracy": 0.75, "reward/chosen": -0.7421875, "reward/margin": 1.296875, "reward/rejected": -2.046875, "step": 1116 }, { "approx. KL/chosen": 117.0, "approx. KL/rejected": 450.0, "epoch": 0.33503299340131976, "grad_norm": 4.79279088973999, "learning_rate": 7.480934805435238e-06, "logp/chosen": -280.0, "logp/rejected": -340.0, "loss": 0.402374267578125, "reward/accuracy": 0.8125, "reward/chosen": -0.87890625, "reward/margin": 1.53125, "reward/rejected": -2.40625, "step": 1117 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 580.0, "epoch": 0.3353329334133173, "grad_norm": 5.475229740142822, "learning_rate": 7.476843152637005e-06, "logp/chosen": -346.0, "logp/rejected": -314.0, "loss": 0.472747802734375, "reward/accuracy": 0.8125, "reward/chosen": -1.6875, "reward/margin": 1.109375, "reward/rejected": -2.796875, "step": 1118 }, { "approx. KL/chosen": 218.0, "approx. KL/rejected": 564.0, "epoch": 0.33563287342531495, "grad_norm": 3.4551777839660645, "learning_rate": 7.472749300627122e-06, "logp/chosen": -316.0, "logp/rejected": -320.0, "loss": 0.3511962890625, "reward/accuracy": 0.875, "reward/chosen": -1.3828125, "reward/margin": 1.625, "reward/rejected": -3.015625, "step": 1119 }, { "approx. KL/chosen": 91.5, "approx. KL/rejected": 330.0, "epoch": 0.3359328134373125, "grad_norm": 4.275612831115723, "learning_rate": 7.468653253040555e-06, "logp/chosen": -266.0, "logp/rejected": -306.0, "loss": 0.45849609375, "reward/accuracy": 0.75, "reward/chosen": -0.87109375, "reward/margin": 1.203125, "reward/rejected": -2.0625, "step": 1120 }, { "approx. KL/chosen": 77.5, "approx. KL/rejected": 442.0, "epoch": 0.33623275344931014, "grad_norm": 4.489099979400635, "learning_rate": 7.464555013514224e-06, "logp/chosen": -404.0, "logp/rejected": -472.0, "loss": 0.5078125, "reward/accuracy": 0.8125, "reward/chosen": -0.84375, "reward/margin": 1.328125, "reward/rejected": -2.171875, "step": 1121 }, { "approx. KL/chosen": 69.5, "approx. KL/rejected": 366.0, "epoch": 0.33653269346130776, "grad_norm": 4.063255786895752, "learning_rate": 7.46045458568699e-06, "logp/chosen": -358.0, "logp/rejected": -292.0, "loss": 0.4747314453125, "reward/accuracy": 0.75, "reward/chosen": -0.80078125, "reward/margin": 1.1328125, "reward/rejected": -1.9296875, "step": 1122 }, { "approx. KL/chosen": 115.0, "approx. KL/rejected": 388.0, "epoch": 0.33683263347330533, "grad_norm": 3.422454833984375, "learning_rate": 7.456351973199664e-06, "logp/chosen": -314.0, "logp/rejected": -304.0, "loss": 0.314910888671875, "reward/accuracy": 0.8125, "reward/chosen": -0.984375, "reward/margin": 1.421875, "reward/rejected": -2.40625, "step": 1123 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 454.0, "epoch": 0.33713257348530296, "grad_norm": 5.981118202209473, "learning_rate": 7.452247179694991e-06, "logp/chosen": -306.0, "logp/rejected": -318.0, "loss": 0.564208984375, "reward/accuracy": 0.6875, "reward/chosen": -1.75, "reward/margin": 0.9921875, "reward/rejected": -2.75, "step": 1124 }, { "approx. KL/chosen": 110.0, "approx. KL/rejected": 664.0, "epoch": 0.3374325134973005, "grad_norm": 2.0661368370056152, "learning_rate": 7.448140208817655e-06, "logp/chosen": -280.0, "logp/rejected": -382.0, "loss": 0.204132080078125, "reward/accuracy": 1.0, "reward/chosen": -1.109375, "reward/margin": 2.21875, "reward/rejected": -3.328125, "step": 1125 }, { "approx. KL/chosen": 454.0, "approx. KL/rejected": 612.0, "epoch": 0.33773245350929815, "grad_norm": 5.3927788734436035, "learning_rate": 7.444031064214274e-06, "logp/chosen": -370.0, "logp/rejected": -368.0, "loss": 0.911346435546875, "reward/accuracy": 0.75, "reward/chosen": -2.125, "reward/margin": 0.69140625, "reward/rejected": -2.8125, "step": 1126 }, { "approx. KL/chosen": 108.0, "approx. KL/rejected": 400.0, "epoch": 0.3380323935212957, "grad_norm": 4.73514986038208, "learning_rate": 7.439919749533393e-06, "logp/chosen": -382.0, "logp/rejected": -374.0, "loss": 0.44580078125, "reward/accuracy": 0.75, "reward/chosen": -0.77734375, "reward/margin": 1.4140625, "reward/rejected": -2.203125, "step": 1127 }, { "approx. KL/chosen": 93.5, "approx. KL/rejected": 564.0, "epoch": 0.33833233353329334, "grad_norm": 3.166121482849121, "learning_rate": 7.435806268425489e-06, "logp/chosen": -616.0, "logp/rejected": -496.0, "loss": 0.2369384765625, "reward/accuracy": 0.9375, "reward/chosen": -1.0234375, "reward/margin": 1.9375, "reward/rejected": -2.953125, "step": 1128 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 568.0, "epoch": 0.33863227354529096, "grad_norm": 7.40391731262207, "learning_rate": 7.431690624542954e-06, "logp/chosen": -312.0, "logp/rejected": -300.0, "loss": 0.6547393798828125, "reward/accuracy": 0.75, "reward/chosen": -1.3984375, "reward/margin": 1.3984375, "reward/rejected": -2.796875, "step": 1129 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 498.0, "epoch": 0.33893221355728853, "grad_norm": 4.143594741821289, "learning_rate": 7.427572821540111e-06, "logp/chosen": -235.0, "logp/rejected": -264.0, "loss": 0.530914306640625, "reward/accuracy": 0.6875, "reward/chosen": -1.1953125, "reward/margin": 1.5234375, "reward/rejected": -2.71875, "step": 1130 }, { "approx. KL/chosen": 103.5, "approx. KL/rejected": 596.0, "epoch": 0.33923215356928615, "grad_norm": 3.244610548019409, "learning_rate": 7.423452863073194e-06, "logp/chosen": -302.0, "logp/rejected": -266.0, "loss": 0.368255615234375, "reward/accuracy": 0.8125, "reward/chosen": -1.0390625, "reward/margin": 1.8984375, "reward/rejected": -2.9375, "step": 1131 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 708.0, "epoch": 0.3395320935812837, "grad_norm": 4.605023384094238, "learning_rate": 7.41933075280035e-06, "logp/chosen": -324.0, "logp/rejected": -282.0, "loss": 0.40423583984375, "reward/accuracy": 0.8125, "reward/chosen": -1.7109375, "reward/margin": 1.8125, "reward/rejected": -3.515625, "step": 1132 }, { "approx. KL/chosen": 126.5, "approx. KL/rejected": 592.0, "epoch": 0.33983203359328135, "grad_norm": 4.420886993408203, "learning_rate": 7.415206494381639e-06, "logp/chosen": -340.0, "logp/rejected": -342.0, "loss": 0.5440673828125, "reward/accuracy": 0.8125, "reward/chosen": -1.0703125, "reward/margin": 1.59375, "reward/rejected": -2.671875, "step": 1133 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 840.0, "epoch": 0.34013197360527897, "grad_norm": 2.3504040241241455, "learning_rate": 7.4110800914790284e-06, "logp/chosen": -266.0, "logp/rejected": -242.0, "loss": 0.21353530883789062, "reward/accuracy": 0.9375, "reward/chosen": -1.1484375, "reward/margin": 2.515625, "reward/rejected": -3.65625, "step": 1134 }, { "approx. KL/chosen": 240.0, "approx. KL/rejected": 780.0, "epoch": 0.34043191361727654, "grad_norm": 8.236926078796387, "learning_rate": 7.406951547756387e-06, "logp/chosen": -430.0, "logp/rejected": -434.0, "loss": 0.449066162109375, "reward/accuracy": 0.75, "reward/chosen": -1.6875, "reward/margin": 1.6640625, "reward/rejected": -3.359375, "step": 1135 }, { "approx. KL/chosen": 234.0, "approx. KL/rejected": 748.0, "epoch": 0.34073185362927416, "grad_norm": 4.136873245239258, "learning_rate": 7.40282086687949e-06, "logp/chosen": -470.0, "logp/rejected": -482.0, "loss": 0.265625, "reward/accuracy": 0.875, "reward/chosen": -1.3125, "reward/margin": 2.171875, "reward/rejected": -3.484375, "step": 1136 }, { "approx. KL/chosen": 348.0, "approx. KL/rejected": 372.0, "epoch": 0.34103179364127173, "grad_norm": 7.754759311676025, "learning_rate": 7.3986880525160034e-06, "logp/chosen": -338.0, "logp/rejected": -229.0, "loss": 0.899658203125, "reward/accuracy": 0.4375, "reward/chosen": -2.265625, "reward/margin": -0.00634765625, "reward/rejected": -2.25, "step": 1137 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 732.0, "epoch": 0.34133173365326935, "grad_norm": 3.397489547729492, "learning_rate": 7.394553108335494e-06, "logp/chosen": -482.0, "logp/rejected": -418.0, "loss": 0.380096435546875, "reward/accuracy": 0.75, "reward/chosen": -1.484375, "reward/margin": 1.6328125, "reward/rejected": -3.109375, "step": 1138 }, { "approx. KL/chosen": 103.5, "approx. KL/rejected": 324.0, "epoch": 0.3416316736652669, "grad_norm": 3.841637134552002, "learning_rate": 7.390416038009412e-06, "logp/chosen": -446.0, "logp/rejected": -322.0, "loss": 0.508544921875, "reward/accuracy": 0.625, "reward/chosen": -0.9609375, "reward/margin": 1.109375, "reward/rejected": -2.078125, "step": 1139 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 584.0, "epoch": 0.34193161367726455, "grad_norm": 3.4330646991729736, "learning_rate": 7.386276845211105e-06, "logp/chosen": -332.0, "logp/rejected": -396.0, "loss": 0.3455810546875, "reward/accuracy": 0.8125, "reward/chosen": -1.34375, "reward/margin": 1.75, "reward/rejected": -3.09375, "step": 1140 }, { "approx. KL/chosen": 101.5, "approx. KL/rejected": 424.0, "epoch": 0.34223155368926217, "grad_norm": 4.090948581695557, "learning_rate": 7.3821355336157955e-06, "logp/chosen": -296.0, "logp/rejected": -264.0, "loss": 0.466552734375, "reward/accuracy": 0.75, "reward/chosen": -0.85546875, "reward/margin": 1.34375, "reward/rejected": -2.203125, "step": 1141 }, { "approx. KL/chosen": 128.0, "approx. KL/rejected": 740.0, "epoch": 0.34253149370125974, "grad_norm": 2.5270564556121826, "learning_rate": 7.377992106900594e-06, "logp/chosen": -206.0, "logp/rejected": -322.0, "loss": 0.27884674072265625, "reward/accuracy": 0.8125, "reward/chosen": -1.1328125, "reward/margin": 2.171875, "reward/rejected": -3.296875, "step": 1142 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 676.0, "epoch": 0.34283143371325736, "grad_norm": 3.2012932300567627, "learning_rate": 7.373846568744487e-06, "logp/chosen": -434.0, "logp/rejected": -332.0, "loss": 0.34197998046875, "reward/accuracy": 0.8125, "reward/chosen": -1.484375, "reward/margin": 1.625, "reward/rejected": -3.109375, "step": 1143 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 488.0, "epoch": 0.34313137372525493, "grad_norm": 3.7505526542663574, "learning_rate": 7.369698922828336e-06, "logp/chosen": -288.0, "logp/rejected": -258.0, "loss": 0.420257568359375, "reward/accuracy": 0.8125, "reward/chosen": -1.453125, "reward/margin": 1.3125, "reward/rejected": -2.765625, "step": 1144 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 784.0, "epoch": 0.34343131373725255, "grad_norm": 4.817116737365723, "learning_rate": 7.365549172834874e-06, "logp/chosen": -398.0, "logp/rejected": -456.0, "loss": 0.49676513671875, "reward/accuracy": 0.6875, "reward/chosen": -1.6015625, "reward/margin": 1.75, "reward/rejected": -3.34375, "step": 1145 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 560.0, "epoch": 0.3437312537492502, "grad_norm": 3.9422757625579834, "learning_rate": 7.361397322448699e-06, "logp/chosen": -282.0, "logp/rejected": -286.0, "loss": 0.6283950805664062, "reward/accuracy": 0.75, "reward/chosen": -0.85546875, "reward/margin": 1.734375, "reward/rejected": -2.59375, "step": 1146 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 616.0, "epoch": 0.34403119376124774, "grad_norm": 2.689926862716675, "learning_rate": 7.35724337535628e-06, "logp/chosen": -288.0, "logp/rejected": -314.0, "loss": 0.3165283203125, "reward/accuracy": 0.875, "reward/chosen": -1.453125, "reward/margin": 1.2890625, "reward/rejected": -2.75, "step": 1147 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 548.0, "epoch": 0.34433113377324537, "grad_norm": 4.198612689971924, "learning_rate": 7.353087335245944e-06, "logp/chosen": -378.0, "logp/rejected": -382.0, "loss": 0.330230712890625, "reward/accuracy": 0.875, "reward/chosen": -1.296875, "reward/margin": 1.5703125, "reward/rejected": -2.875, "step": 1148 }, { "approx. KL/chosen": 188.0, "approx. KL/rejected": 776.0, "epoch": 0.34463107378524294, "grad_norm": 5.033273220062256, "learning_rate": 7.348929205807876e-06, "logp/chosen": -300.0, "logp/rejected": -374.0, "loss": 0.482452392578125, "reward/accuracy": 0.75, "reward/chosen": -1.578125, "reward/margin": 1.6328125, "reward/rejected": -3.21875, "step": 1149 }, { "approx. KL/chosen": 69.5, "approx. KL/rejected": 700.0, "epoch": 0.34493101379724056, "grad_norm": 3.1137473583221436, "learning_rate": 7.344768990734119e-06, "logp/chosen": -298.0, "logp/rejected": -300.0, "loss": 0.41115570068359375, "reward/accuracy": 0.75, "reward/chosen": -0.6640625, "reward/margin": 2.40625, "reward/rejected": -3.0625, "step": 1150 }, { "approx. KL/chosen": 350.0, "approx. KL/rejected": 928.0, "epoch": 0.34523095380923813, "grad_norm": 4.781032562255859, "learning_rate": 7.340606693718563e-06, "logp/chosen": -342.0, "logp/rejected": -378.0, "loss": 0.4984130859375, "reward/accuracy": 0.6875, "reward/chosen": -1.953125, "reward/margin": 1.4921875, "reward/rejected": -3.4375, "step": 1151 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 956.0, "epoch": 0.34553089382123575, "grad_norm": 4.472346305847168, "learning_rate": 7.336442318456952e-06, "logp/chosen": -199.0, "logp/rejected": -300.0, "loss": 0.546630859375, "reward/accuracy": 0.625, "reward/chosen": -1.5546875, "reward/margin": 1.8828125, "reward/rejected": -3.453125, "step": 1152 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 438.0, "epoch": 0.3458308338332334, "grad_norm": 5.917614936828613, "learning_rate": 7.332275868646871e-06, "logp/chosen": -440.0, "logp/rejected": -326.0, "loss": 0.67144775390625, "reward/accuracy": 0.625, "reward/chosen": -1.3984375, "reward/margin": 0.984375, "reward/rejected": -2.390625, "step": 1153 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 322.0, "epoch": 0.34613077384523094, "grad_norm": 5.325502395629883, "learning_rate": 7.3281073479877495e-06, "logp/chosen": -394.0, "logp/rejected": -316.0, "loss": 0.577880859375, "reward/accuracy": 0.625, "reward/chosen": -1.3359375, "reward/margin": 0.7109375, "reward/rejected": -2.046875, "step": 1154 }, { "approx. KL/chosen": 73.5, "approx. KL/rejected": 374.0, "epoch": 0.34643071385722857, "grad_norm": 3.698976993560791, "learning_rate": 7.323936760180857e-06, "logp/chosen": -394.0, "logp/rejected": -370.0, "loss": 0.372650146484375, "reward/accuracy": 0.75, "reward/chosen": -0.76171875, "reward/margin": 1.3828125, "reward/rejected": -2.140625, "step": 1155 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 572.0, "epoch": 0.34673065386922614, "grad_norm": 4.252513885498047, "learning_rate": 7.319764108929294e-06, "logp/chosen": -384.0, "logp/rejected": -506.0, "loss": 0.491455078125, "reward/accuracy": 0.6875, "reward/chosen": -1.609375, "reward/margin": 1.1953125, "reward/rejected": -2.8125, "step": 1156 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 572.0, "epoch": 0.34703059388122376, "grad_norm": 5.404839992523193, "learning_rate": 7.315589397937997e-06, "logp/chosen": -272.0, "logp/rejected": -406.0, "loss": 0.6015625, "reward/accuracy": 0.5625, "reward/chosen": -1.75, "reward/margin": 1.0, "reward/rejected": -2.75, "step": 1157 }, { "approx. KL/chosen": 133.0, "approx. KL/rejected": 1112.0, "epoch": 0.3473305338932214, "grad_norm": 5.471554279327393, "learning_rate": 7.311412630913728e-06, "logp/chosen": -358.0, "logp/rejected": -402.0, "loss": 0.4627685546875, "reward/accuracy": 0.875, "reward/chosen": -1.4453125, "reward/margin": 1.9375, "reward/rejected": -3.390625, "step": 1158 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 592.0, "epoch": 0.34763047390521895, "grad_norm": 2.355163097381592, "learning_rate": 7.307233811565078e-06, "logp/chosen": -500.0, "logp/rejected": -366.0, "loss": 0.251739501953125, "reward/accuracy": 0.9375, "reward/chosen": -0.94921875, "reward/margin": 1.8125, "reward/rejected": -2.765625, "step": 1159 }, { "approx. KL/chosen": 111.5, "approx. KL/rejected": 386.0, "epoch": 0.3479304139172166, "grad_norm": 3.1021721363067627, "learning_rate": 7.30305294360246e-06, "logp/chosen": -380.0, "logp/rejected": -390.0, "loss": 0.388427734375, "reward/accuracy": 0.8125, "reward/chosen": -1.0078125, "reward/margin": 1.1484375, "reward/rejected": -2.15625, "step": 1160 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 396.0, "epoch": 0.34823035392921414, "grad_norm": 5.981139659881592, "learning_rate": 7.298870030738103e-06, "logp/chosen": -221.0, "logp/rejected": -354.0, "loss": 0.79449462890625, "reward/accuracy": 0.5625, "reward/chosen": -1.84375, "reward/margin": 0.546875, "reward/rejected": -2.390625, "step": 1161 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 648.0, "epoch": 0.34853029394121177, "grad_norm": 6.833940029144287, "learning_rate": 7.294685076686055e-06, "logp/chosen": -280.0, "logp/rejected": -236.0, "loss": 0.31829833984375, "reward/accuracy": 0.9375, "reward/chosen": -1.34375, "reward/margin": 1.859375, "reward/rejected": -3.203125, "step": 1162 }, { "approx. KL/chosen": 85.0, "approx. KL/rejected": 472.0, "epoch": 0.34883023395320933, "grad_norm": 3.6556148529052734, "learning_rate": 7.290498085162173e-06, "logp/chosen": -412.0, "logp/rejected": -362.0, "loss": 0.42498779296875, "reward/accuracy": 0.875, "reward/chosen": -0.78125, "reward/margin": 1.8203125, "reward/rejected": -2.59375, "step": 1163 }, { "approx. KL/chosen": 219.0, "approx. KL/rejected": 900.0, "epoch": 0.34913017396520696, "grad_norm": 5.674472332000732, "learning_rate": 7.286309059884128e-06, "logp/chosen": -376.0, "logp/rejected": -346.0, "loss": 0.4951171875, "reward/accuracy": 0.625, "reward/chosen": -1.3515625, "reward/margin": 2.203125, "reward/rejected": -3.5625, "step": 1164 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 796.0, "epoch": 0.3494301139772046, "grad_norm": 5.021270751953125, "learning_rate": 7.2821180045713904e-06, "logp/chosen": -322.0, "logp/rejected": -316.0, "loss": 0.6241722106933594, "reward/accuracy": 0.625, "reward/chosen": -1.5234375, "reward/margin": 1.53125, "reward/rejected": -3.046875, "step": 1165 }, { "approx. KL/chosen": 147.0, "approx. KL/rejected": 458.0, "epoch": 0.34973005398920215, "grad_norm": 3.4719371795654297, "learning_rate": 7.277924922945239e-06, "logp/chosen": -276.0, "logp/rejected": -276.0, "loss": 0.39105224609375, "reward/accuracy": 0.8125, "reward/chosen": -1.3125, "reward/margin": 1.328125, "reward/rejected": -2.640625, "step": 1166 }, { "approx. KL/chosen": 180.0, "approx. KL/rejected": 482.0, "epoch": 0.3500299940011998, "grad_norm": 3.919583559036255, "learning_rate": 7.273729818728747e-06, "logp/chosen": -358.0, "logp/rejected": -354.0, "loss": 0.4066162109375, "reward/accuracy": 0.875, "reward/chosen": -1.0390625, "reward/margin": 1.6015625, "reward/rejected": -2.640625, "step": 1167 }, { "approx. KL/chosen": 120.0, "approx. KL/rejected": 472.0, "epoch": 0.35032993401319734, "grad_norm": 4.590411186218262, "learning_rate": 7.269532695646789e-06, "logp/chosen": -366.0, "logp/rejected": -256.0, "loss": 0.488983154296875, "reward/accuracy": 0.6875, "reward/chosen": -0.8515625, "reward/margin": 1.6875, "reward/rejected": -2.546875, "step": 1168 }, { "approx. KL/chosen": 128.0, "approx. KL/rejected": 438.0, "epoch": 0.35062987402519497, "grad_norm": 4.401933670043945, "learning_rate": 7.265333557426026e-06, "logp/chosen": -272.0, "logp/rejected": -268.0, "loss": 0.39080810546875, "reward/accuracy": 0.75, "reward/chosen": -1.09375, "reward/margin": 1.5, "reward/rejected": -2.59375, "step": 1169 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 712.0, "epoch": 0.3509298140371926, "grad_norm": 4.43110466003418, "learning_rate": 7.261132407794911e-06, "logp/chosen": -304.0, "logp/rejected": -312.0, "loss": 0.33356475830078125, "reward/accuracy": 0.875, "reward/chosen": -1.2734375, "reward/margin": 2.09375, "reward/rejected": -3.359375, "step": 1170 }, { "approx. KL/chosen": 120.0, "approx. KL/rejected": 864.0, "epoch": 0.35122975404919016, "grad_norm": 2.8720200061798096, "learning_rate": 7.256929250483685e-06, "logp/chosen": -342.0, "logp/rejected": -300.0, "loss": 0.3322296142578125, "reward/accuracy": 0.75, "reward/chosen": -1.2109375, "reward/margin": 2.203125, "reward/rejected": -3.40625, "step": 1171 }, { "approx. KL/chosen": 168.0, "approx. KL/rejected": 592.0, "epoch": 0.3515296940611878, "grad_norm": 3.9539904594421387, "learning_rate": 7.25272408922437e-06, "logp/chosen": -460.0, "logp/rejected": -508.0, "loss": 0.397552490234375, "reward/accuracy": 0.75, "reward/chosen": -1.1953125, "reward/margin": 1.3828125, "reward/rejected": -2.578125, "step": 1172 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 412.0, "epoch": 0.35182963407318535, "grad_norm": 5.210118770599365, "learning_rate": 7.248516927750765e-06, "logp/chosen": -246.0, "logp/rejected": -274.0, "loss": 0.731964111328125, "reward/accuracy": 0.5625, "reward/chosen": -1.46875, "reward/margin": 0.5234375, "reward/rejected": -1.9921875, "step": 1173 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 290.0, "epoch": 0.352129574085183, "grad_norm": 4.330535411834717, "learning_rate": 7.244307769798449e-06, "logp/chosen": -362.0, "logp/rejected": -354.0, "loss": 0.5302734375, "reward/accuracy": 0.75, "reward/chosen": -1.359375, "reward/margin": 0.6875, "reward/rejected": -2.046875, "step": 1174 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 604.0, "epoch": 0.35242951409718054, "grad_norm": 5.518300533294678, "learning_rate": 7.24009661910477e-06, "logp/chosen": -430.0, "logp/rejected": -390.0, "loss": 0.63751220703125, "reward/accuracy": 0.6875, "reward/chosen": -1.453125, "reward/margin": 1.1953125, "reward/rejected": -2.640625, "step": 1175 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 556.0, "epoch": 0.35272945410917816, "grad_norm": 5.062828063964844, "learning_rate": 7.23588347940885e-06, "logp/chosen": -384.0, "logp/rejected": -332.0, "loss": 0.60418701171875, "reward/accuracy": 0.75, "reward/chosen": -1.5234375, "reward/margin": 1.3125, "reward/rejected": -2.84375, "step": 1176 }, { "approx. KL/chosen": 127.5, "approx. KL/rejected": 342.0, "epoch": 0.3530293941211758, "grad_norm": 5.8231587409973145, "learning_rate": 7.231668354451571e-06, "logp/chosen": -306.0, "logp/rejected": -298.0, "loss": 0.5465087890625, "reward/accuracy": 0.6875, "reward/chosen": -1.046875, "reward/margin": 1.1015625, "reward/rejected": -2.15625, "step": 1177 }, { "approx. KL/chosen": 193.0, "approx. KL/rejected": 266.0, "epoch": 0.35332933413317336, "grad_norm": 5.507146835327148, "learning_rate": 7.227451247975581e-06, "logp/chosen": -308.0, "logp/rejected": -332.0, "loss": 0.7034912109375, "reward/accuracy": 0.625, "reward/chosen": -1.40625, "reward/margin": 0.51171875, "reward/rejected": -1.921875, "step": 1178 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 672.0, "epoch": 0.353629274145171, "grad_norm": 4.477617263793945, "learning_rate": 7.223232163725291e-06, "logp/chosen": -436.0, "logp/rejected": -430.0, "loss": 0.4532470703125, "reward/accuracy": 0.75, "reward/chosen": -1.484375, "reward/margin": 1.5390625, "reward/rejected": -3.015625, "step": 1179 }, { "approx. KL/chosen": 206.0, "approx. KL/rejected": 684.0, "epoch": 0.35392921415716855, "grad_norm": 3.660447359085083, "learning_rate": 7.2190111054468605e-06, "logp/chosen": -344.0, "logp/rejected": -312.0, "loss": 0.42755126953125, "reward/accuracy": 0.75, "reward/chosen": -1.3046875, "reward/margin": 1.7734375, "reward/rejected": -3.078125, "step": 1180 }, { "approx. KL/chosen": 130.0, "approx. KL/rejected": 572.0, "epoch": 0.35422915416916617, "grad_norm": 3.812239408493042, "learning_rate": 7.214788076888208e-06, "logp/chosen": -366.0, "logp/rejected": -234.0, "loss": 0.2987060546875, "reward/accuracy": 0.8125, "reward/chosen": -1.1953125, "reward/margin": 1.796875, "reward/rejected": -3.0, "step": 1181 }, { "approx. KL/chosen": 72.0, "approx. KL/rejected": 644.0, "epoch": 0.35452909418116374, "grad_norm": 3.150585651397705, "learning_rate": 7.210563081798998e-06, "logp/chosen": -428.0, "logp/rejected": -418.0, "loss": 0.2926673889160156, "reward/accuracy": 0.875, "reward/chosen": -0.94140625, "reward/margin": 1.9921875, "reward/rejected": -2.9375, "step": 1182 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 612.0, "epoch": 0.35482903419316136, "grad_norm": 3.574704647064209, "learning_rate": 7.2063361239306425e-06, "logp/chosen": -472.0, "logp/rejected": -374.0, "loss": 0.4118194580078125, "reward/accuracy": 0.8125, "reward/chosen": -1.296875, "reward/margin": 1.6796875, "reward/rejected": -2.96875, "step": 1183 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 612.0, "epoch": 0.355128974205159, "grad_norm": 4.427652835845947, "learning_rate": 7.202107207036295e-06, "logp/chosen": -332.0, "logp/rejected": -368.0, "loss": 0.51904296875, "reward/accuracy": 0.625, "reward/chosen": -2.046875, "reward/margin": 1.03125, "reward/rejected": -3.078125, "step": 1184 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 1024.0, "epoch": 0.35542891421715656, "grad_norm": 6.330018520355225, "learning_rate": 7.1978763348708524e-06, "logp/chosen": -360.0, "logp/rejected": -314.0, "loss": 0.4645538330078125, "reward/accuracy": 0.6875, "reward/chosen": -1.7421875, "reward/margin": 2.234375, "reward/rejected": -3.984375, "step": 1185 }, { "approx. KL/chosen": 454.0, "approx. KL/rejected": 506.0, "epoch": 0.3557288542291542, "grad_norm": 5.584761619567871, "learning_rate": 7.1936435111909416e-06, "logp/chosen": -338.0, "logp/rejected": -244.0, "loss": 0.720458984375, "reward/accuracy": 0.75, "reward/chosen": -2.25, "reward/margin": 0.65625, "reward/rejected": -2.90625, "step": 1186 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 502.0, "epoch": 0.35602879424115175, "grad_norm": 8.111717224121094, "learning_rate": 7.189408739754927e-06, "logp/chosen": -304.0, "logp/rejected": -255.0, "loss": 0.683837890625, "reward/accuracy": 0.625, "reward/chosen": -1.984375, "reward/margin": 0.73046875, "reward/rejected": -2.71875, "step": 1187 }, { "approx. KL/chosen": 128.0, "approx. KL/rejected": 428.0, "epoch": 0.35632873425314937, "grad_norm": 3.792501211166382, "learning_rate": 7.185172024322902e-06, "logp/chosen": -288.0, "logp/rejected": -286.0, "loss": 0.3580322265625, "reward/accuracy": 0.875, "reward/chosen": -1.0859375, "reward/margin": 1.3984375, "reward/rejected": -2.484375, "step": 1188 }, { "approx. KL/chosen": 314.0, "approx. KL/rejected": 408.0, "epoch": 0.356628674265147, "grad_norm": 5.456514358520508, "learning_rate": 7.180933368656681e-06, "logp/chosen": -354.0, "logp/rejected": -268.0, "loss": 0.5845947265625, "reward/accuracy": 0.8125, "reward/chosen": -1.84375, "reward/margin": 0.76953125, "reward/rejected": -2.609375, "step": 1189 }, { "approx. KL/chosen": 334.0, "approx. KL/rejected": 664.0, "epoch": 0.35692861427714456, "grad_norm": 4.153448104858398, "learning_rate": 7.176692776519809e-06, "logp/chosen": -260.0, "logp/rejected": -362.0, "loss": 0.511260986328125, "reward/accuracy": 0.6875, "reward/chosen": -2.171875, "reward/margin": 1.078125, "reward/rejected": -3.25, "step": 1190 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 736.0, "epoch": 0.3572285542891422, "grad_norm": 3.4140238761901855, "learning_rate": 7.172450251677543e-06, "logp/chosen": -255.0, "logp/rejected": -384.0, "loss": 0.293701171875, "reward/accuracy": 0.875, "reward/chosen": -1.421875, "reward/margin": 1.9609375, "reward/rejected": -3.375, "step": 1191 }, { "approx. KL/chosen": 316.0, "approx. KL/rejected": 432.0, "epoch": 0.35752849430113975, "grad_norm": 6.198919773101807, "learning_rate": 7.168205797896862e-06, "logp/chosen": -370.0, "logp/rejected": -364.0, "loss": 0.618408203125, "reward/accuracy": 0.625, "reward/chosen": -1.859375, "reward/margin": 0.57421875, "reward/rejected": -2.4375, "step": 1192 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 696.0, "epoch": 0.3578284343131374, "grad_norm": 5.759828567504883, "learning_rate": 7.1639594189464555e-06, "logp/chosen": -284.0, "logp/rejected": -223.0, "loss": 0.6382827758789062, "reward/accuracy": 0.75, "reward/chosen": -1.7109375, "reward/margin": 1.5546875, "reward/rejected": -3.265625, "step": 1193 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 768.0, "epoch": 0.35812837432513495, "grad_norm": 5.184229850769043, "learning_rate": 7.159711118596721e-06, "logp/chosen": -320.0, "logp/rejected": -318.0, "loss": 0.494781494140625, "reward/accuracy": 0.8125, "reward/chosen": -1.78125, "reward/margin": 1.546875, "reward/rejected": -3.328125, "step": 1194 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 560.0, "epoch": 0.35842831433713257, "grad_norm": 5.786222457885742, "learning_rate": 7.155460900619765e-06, "logp/chosen": -382.0, "logp/rejected": -374.0, "loss": 0.83837890625, "reward/accuracy": 0.6875, "reward/chosen": -1.8359375, "reward/margin": 0.98046875, "reward/rejected": -2.8125, "step": 1195 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 1496.0, "epoch": 0.3587282543491302, "grad_norm": 3.558359384536743, "learning_rate": 7.151208768789391e-06, "logp/chosen": -486.0, "logp/rejected": -416.0, "loss": 0.31781005859375, "reward/accuracy": 0.8125, "reward/chosen": -1.75, "reward/margin": 2.8125, "reward/rejected": -4.5625, "step": 1196 }, { "approx. KL/chosen": 247.0, "approx. KL/rejected": 684.0, "epoch": 0.35902819436112776, "grad_norm": 4.169587135314941, "learning_rate": 7.14695472688111e-06, "logp/chosen": -414.0, "logp/rejected": -346.0, "loss": 0.3636474609375, "reward/accuracy": 0.9375, "reward/chosen": -1.734375, "reward/margin": 1.625, "reward/rejected": -3.359375, "step": 1197 }, { "approx. KL/chosen": 137.0, "approx. KL/rejected": 406.0, "epoch": 0.3593281343731254, "grad_norm": 4.5074872970581055, "learning_rate": 7.142698778672121e-06, "logp/chosen": -388.0, "logp/rejected": -340.0, "loss": 0.60150146484375, "reward/accuracy": 0.625, "reward/chosen": -1.359375, "reward/margin": 0.8671875, "reward/rejected": -2.234375, "step": 1198 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 692.0, "epoch": 0.35962807438512295, "grad_norm": 2.95103120803833, "learning_rate": 7.138440927941324e-06, "logp/chosen": -338.0, "logp/rejected": -334.0, "loss": 0.288116455078125, "reward/accuracy": 0.9375, "reward/chosen": -1.1875, "reward/margin": 2.03125, "reward/rejected": -3.21875, "step": 1199 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 796.0, "epoch": 0.3599280143971206, "grad_norm": 3.306480646133423, "learning_rate": 7.134181178469297e-06, "logp/chosen": -346.0, "logp/rejected": -304.0, "loss": 0.40625, "reward/accuracy": 0.8125, "reward/chosen": -1.109375, "reward/margin": 2.0625, "reward/rejected": -3.1875, "step": 1200 }, { "approx. KL/chosen": 129.0, "approx. KL/rejected": 388.0, "epoch": 0.3602279544091182, "grad_norm": 3.8588204383850098, "learning_rate": 7.129919534038314e-06, "logp/chosen": -394.0, "logp/rejected": -324.0, "loss": 0.43414306640625, "reward/accuracy": 0.75, "reward/chosen": -1.0078125, "reward/margin": 1.3203125, "reward/rejected": -2.328125, "step": 1201 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 1012.0, "epoch": 0.36052789442111577, "grad_norm": 4.421947002410889, "learning_rate": 7.125655998432331e-06, "logp/chosen": -386.0, "logp/rejected": -404.0, "loss": 0.4508171081542969, "reward/accuracy": 0.8125, "reward/chosen": -1.703125, "reward/margin": 2.140625, "reward/rejected": -3.84375, "step": 1202 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 592.0, "epoch": 0.3608278344331134, "grad_norm": 3.7226498126983643, "learning_rate": 7.121390575436975e-06, "logp/chosen": -294.0, "logp/rejected": -330.0, "loss": 0.413818359375, "reward/accuracy": 0.75, "reward/chosen": -1.484375, "reward/margin": 1.3984375, "reward/rejected": -2.875, "step": 1203 }, { "approx. KL/chosen": 132.0, "approx. KL/rejected": 928.0, "epoch": 0.36112777444511096, "grad_norm": 2.1991002559661865, "learning_rate": 7.117123268839557e-06, "logp/chosen": -320.0, "logp/rejected": -406.0, "loss": 0.2064971923828125, "reward/accuracy": 0.9375, "reward/chosen": -1.34375, "reward/margin": 2.5625, "reward/rejected": -3.90625, "step": 1204 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 506.0, "epoch": 0.3614277144571086, "grad_norm": 5.361876010894775, "learning_rate": 7.112854082429057e-06, "logp/chosen": -352.0, "logp/rejected": -320.0, "loss": 0.65625, "reward/accuracy": 0.6875, "reward/chosen": -1.3046875, "reward/margin": 1.171875, "reward/rejected": -2.484375, "step": 1205 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 780.0, "epoch": 0.36172765446910615, "grad_norm": 3.1251778602600098, "learning_rate": 7.108583019996124e-06, "logp/chosen": -274.0, "logp/rejected": -320.0, "loss": 0.364959716796875, "reward/accuracy": 0.8125, "reward/chosen": -1.546875, "reward/margin": 1.875, "reward/rejected": -3.421875, "step": 1206 }, { "approx. KL/chosen": 448.0, "approx. KL/rejected": 608.0, "epoch": 0.3620275944811038, "grad_norm": 5.361334323883057, "learning_rate": 7.104310085333073e-06, "logp/chosen": -424.0, "logp/rejected": -378.0, "loss": 0.687713623046875, "reward/accuracy": 0.75, "reward/chosen": -2.484375, "reward/margin": 0.65625, "reward/rejected": -3.140625, "step": 1207 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 636.0, "epoch": 0.3623275344931014, "grad_norm": 3.5696511268615723, "learning_rate": 7.100035282233883e-06, "logp/chosen": -398.0, "logp/rejected": -342.0, "loss": 0.37060546875, "reward/accuracy": 0.875, "reward/chosen": -1.65625, "reward/margin": 1.4296875, "reward/rejected": -3.09375, "step": 1208 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 548.0, "epoch": 0.36262747450509897, "grad_norm": 4.821598529815674, "learning_rate": 7.09575861449419e-06, "logp/chosen": -524.0, "logp/rejected": -468.0, "loss": 0.4713134765625, "reward/accuracy": 0.625, "reward/chosen": -1.875, "reward/margin": 1.046875, "reward/rejected": -2.921875, "step": 1209 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 1012.0, "epoch": 0.3629274145170966, "grad_norm": 4.962735176086426, "learning_rate": 7.091480085911286e-06, "logp/chosen": -414.0, "logp/rejected": -410.0, "loss": 0.3980865478515625, "reward/accuracy": 0.875, "reward/chosen": -2.296875, "reward/margin": 1.859375, "reward/rejected": -4.15625, "step": 1210 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 632.0, "epoch": 0.36322735452909416, "grad_norm": 4.185608386993408, "learning_rate": 7.087199700284117e-06, "logp/chosen": -368.0, "logp/rejected": -380.0, "loss": 0.4534912109375, "reward/accuracy": 0.8125, "reward/chosen": -1.7265625, "reward/margin": 1.3671875, "reward/rejected": -3.09375, "step": 1211 }, { "approx. KL/chosen": 115.5, "approx. KL/rejected": 888.0, "epoch": 0.3635272945410918, "grad_norm": 2.185736894607544, "learning_rate": 7.082917461413273e-06, "logp/chosen": -486.0, "logp/rejected": -366.0, "loss": 0.1621246337890625, "reward/accuracy": 0.9375, "reward/chosen": -1.0, "reward/margin": 2.984375, "reward/rejected": -3.984375, "step": 1212 }, { "approx. KL/chosen": 334.0, "approx. KL/rejected": 784.0, "epoch": 0.3638272345530894, "grad_norm": 5.507760047912598, "learning_rate": 7.078633373100998e-06, "logp/chosen": -278.0, "logp/rejected": -304.0, "loss": 0.520660400390625, "reward/accuracy": 0.625, "reward/chosen": -2.203125, "reward/margin": 1.2421875, "reward/rejected": -3.4375, "step": 1213 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 984.0, "epoch": 0.364127174565087, "grad_norm": 3.6548829078674316, "learning_rate": 7.07434743915117e-06, "logp/chosen": -462.0, "logp/rejected": -352.0, "loss": 0.3960113525390625, "reward/accuracy": 0.875, "reward/chosen": -2.203125, "reward/margin": 1.828125, "reward/rejected": -4.03125, "step": 1214 }, { "approx. KL/chosen": 193.0, "approx. KL/rejected": 604.0, "epoch": 0.3644271145770846, "grad_norm": 3.875882148742676, "learning_rate": 7.07005966336931e-06, "logp/chosen": -348.0, "logp/rejected": -326.0, "loss": 0.436798095703125, "reward/accuracy": 0.75, "reward/chosen": -1.5546875, "reward/margin": 1.4140625, "reward/rejected": -2.96875, "step": 1215 }, { "approx. KL/chosen": 342.0, "approx. KL/rejected": 520.0, "epoch": 0.36472705458908217, "grad_norm": 5.309492111206055, "learning_rate": 7.065770049562572e-06, "logp/chosen": -358.0, "logp/rejected": -366.0, "loss": 0.737548828125, "reward/accuracy": 0.6875, "reward/chosen": -2.09375, "reward/margin": 0.58984375, "reward/rejected": -2.6875, "step": 1216 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 656.0, "epoch": 0.3650269946010798, "grad_norm": 4.710912704467773, "learning_rate": 7.0614786015397445e-06, "logp/chosen": -278.0, "logp/rejected": -304.0, "loss": 0.6131591796875, "reward/accuracy": 0.6875, "reward/chosen": -2.109375, "reward/margin": 1.109375, "reward/rejected": -3.21875, "step": 1217 }, { "approx. KL/chosen": 187.0, "approx. KL/rejected": 904.0, "epoch": 0.36532693461307736, "grad_norm": 3.4270834922790527, "learning_rate": 7.057185323111242e-06, "logp/chosen": -308.0, "logp/rejected": -270.0, "loss": 0.3283882141113281, "reward/accuracy": 0.8125, "reward/chosen": -1.4453125, "reward/margin": 2.140625, "reward/rejected": -3.59375, "step": 1218 }, { "approx. KL/chosen": 223.0, "approx. KL/rejected": 552.0, "epoch": 0.365626874625075, "grad_norm": 4.725757598876953, "learning_rate": 7.052890218089106e-06, "logp/chosen": -251.0, "logp/rejected": -320.0, "loss": 0.61981201171875, "reward/accuracy": 0.6875, "reward/chosen": -1.8359375, "reward/margin": 0.9140625, "reward/rejected": -2.75, "step": 1219 }, { "approx. KL/chosen": 432.0, "approx. KL/rejected": 912.0, "epoch": 0.3659268146370726, "grad_norm": 7.1391167640686035, "learning_rate": 7.048593290287002e-06, "logp/chosen": -296.0, "logp/rejected": -260.0, "loss": 0.3043670654296875, "reward/accuracy": 0.875, "reward/chosen": -1.859375, "reward/margin": 2.0625, "reward/rejected": -3.921875, "step": 1220 }, { "approx. KL/chosen": 107.0, "approx. KL/rejected": 616.0, "epoch": 0.3662267546490702, "grad_norm": 5.125776767730713, "learning_rate": 7.044294543520207e-06, "logp/chosen": -360.0, "logp/rejected": -520.0, "loss": 0.4972991943359375, "reward/accuracy": 0.6875, "reward/chosen": -1.1171875, "reward/margin": 1.546875, "reward/rejected": -2.65625, "step": 1221 }, { "approx. KL/chosen": 185.0, "approx. KL/rejected": 516.0, "epoch": 0.3665266946610678, "grad_norm": 5.194803714752197, "learning_rate": 7.039993981605623e-06, "logp/chosen": -280.0, "logp/rejected": -274.0, "loss": 0.50909423828125, "reward/accuracy": 0.6875, "reward/chosen": -1.453125, "reward/margin": 1.28125, "reward/rejected": -2.734375, "step": 1222 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 948.0, "epoch": 0.36682663467306537, "grad_norm": 3.9764740467071533, "learning_rate": 7.035691608361752e-06, "logp/chosen": -217.0, "logp/rejected": -394.0, "loss": 0.495361328125, "reward/accuracy": 0.8125, "reward/chosen": -1.640625, "reward/margin": 1.6328125, "reward/rejected": -3.265625, "step": 1223 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 612.0, "epoch": 0.367126574685063, "grad_norm": 3.6500141620635986, "learning_rate": 7.0313874276087155e-06, "logp/chosen": -410.0, "logp/rejected": -406.0, "loss": 0.450439453125, "reward/accuracy": 0.75, "reward/chosen": -1.3203125, "reward/margin": 1.4453125, "reward/rejected": -2.765625, "step": 1224 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 1080.0, "epoch": 0.3674265146970606, "grad_norm": 3.644559621810913, "learning_rate": 7.027081443168233e-06, "logp/chosen": -240.0, "logp/rejected": -310.0, "loss": 0.2552490234375, "reward/accuracy": 0.9375, "reward/chosen": -1.4140625, "reward/margin": 2.734375, "reward/rejected": -4.15625, "step": 1225 }, { "approx. KL/chosen": 128.0, "approx. KL/rejected": 808.0, "epoch": 0.3677264547090582, "grad_norm": 3.8890857696533203, "learning_rate": 7.022773658863626e-06, "logp/chosen": -272.0, "logp/rejected": -268.0, "loss": 0.325347900390625, "reward/accuracy": 0.875, "reward/chosen": -1.265625, "reward/margin": 2.25, "reward/rejected": -3.515625, "step": 1226 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 432.0, "epoch": 0.3680263947210558, "grad_norm": 3.9518446922302246, "learning_rate": 7.018464078519819e-06, "logp/chosen": -414.0, "logp/rejected": -334.0, "loss": 0.44403076171875, "reward/accuracy": 0.75, "reward/chosen": -1.2890625, "reward/margin": 1.2890625, "reward/rejected": -2.578125, "step": 1227 }, { "approx. KL/chosen": 180.0, "approx. KL/rejected": 466.0, "epoch": 0.3683263347330534, "grad_norm": 5.786788463592529, "learning_rate": 7.014152705963324e-06, "logp/chosen": -376.0, "logp/rejected": -410.0, "loss": 0.7257080078125, "reward/accuracy": 0.6875, "reward/chosen": -1.546875, "reward/margin": 1.0, "reward/rejected": -2.546875, "step": 1228 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 544.0, "epoch": 0.368626274745051, "grad_norm": 4.025776386260986, "learning_rate": 7.0098395450222515e-06, "logp/chosen": -416.0, "logp/rejected": -396.0, "loss": 0.486328125, "reward/accuracy": 0.6875, "reward/chosen": -1.4921875, "reward/margin": 1.3046875, "reward/rejected": -2.796875, "step": 1229 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 588.0, "epoch": 0.36892621475704857, "grad_norm": 6.129549980163574, "learning_rate": 7.005524599526293e-06, "logp/chosen": -240.0, "logp/rejected": -306.0, "loss": 0.822998046875, "reward/accuracy": 0.625, "reward/chosen": -1.5859375, "reward/margin": 1.03125, "reward/rejected": -2.609375, "step": 1230 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 692.0, "epoch": 0.3692261547690462, "grad_norm": 4.455682754516602, "learning_rate": 7.001207873306731e-06, "logp/chosen": -223.0, "logp/rejected": -264.0, "loss": 0.4705810546875, "reward/accuracy": 0.8125, "reward/chosen": -1.3984375, "reward/margin": 1.859375, "reward/rejected": -3.265625, "step": 1231 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 438.0, "epoch": 0.3695260947810438, "grad_norm": 5.4254584312438965, "learning_rate": 6.996889370196424e-06, "logp/chosen": -450.0, "logp/rejected": -314.0, "loss": 0.702392578125, "reward/accuracy": 0.6875, "reward/chosen": -1.78125, "reward/margin": 0.875, "reward/rejected": -2.65625, "step": 1232 }, { "approx. KL/chosen": 229.0, "approx. KL/rejected": 888.0, "epoch": 0.3698260347930414, "grad_norm": 2.907653570175171, "learning_rate": 6.992569094029811e-06, "logp/chosen": -428.0, "logp/rejected": -484.0, "loss": 0.22357177734375, "reward/accuracy": 0.9375, "reward/chosen": -1.5234375, "reward/margin": 2.296875, "reward/rejected": -3.828125, "step": 1233 }, { "approx. KL/chosen": 145.0, "approx. KL/rejected": 572.0, "epoch": 0.370125974805039, "grad_norm": 2.2408995628356934, "learning_rate": 6.9882470486429055e-06, "logp/chosen": -334.0, "logp/rejected": -298.0, "loss": 0.258087158203125, "reward/accuracy": 0.9375, "reward/chosen": -1.2890625, "reward/margin": 1.78125, "reward/rejected": -3.078125, "step": 1234 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 652.0, "epoch": 0.3704259148170366, "grad_norm": 2.807201385498047, "learning_rate": 6.983923237873289e-06, "logp/chosen": -366.0, "logp/rejected": -288.0, "loss": 0.33612060546875, "reward/accuracy": 0.875, "reward/chosen": -1.59375, "reward/margin": 1.7109375, "reward/rejected": -3.296875, "step": 1235 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 772.0, "epoch": 0.3707258548290342, "grad_norm": 4.398697376251221, "learning_rate": 6.979597665560116e-06, "logp/chosen": -368.0, "logp/rejected": -324.0, "loss": 0.5490798950195312, "reward/accuracy": 0.8125, "reward/chosen": -1.453125, "reward/margin": 1.765625, "reward/rejected": -3.234375, "step": 1236 }, { "approx. KL/chosen": 243.0, "approx. KL/rejected": 1112.0, "epoch": 0.3710257948410318, "grad_norm": 3.5578298568725586, "learning_rate": 6.975270335544099e-06, "logp/chosen": -324.0, "logp/rejected": -372.0, "loss": 0.3058624267578125, "reward/accuracy": 0.875, "reward/chosen": -1.796875, "reward/margin": 2.484375, "reward/rejected": -4.28125, "step": 1237 }, { "approx. KL/chosen": 234.0, "approx. KL/rejected": 636.0, "epoch": 0.3713257348530294, "grad_norm": 5.002593040466309, "learning_rate": 6.970941251667513e-06, "logp/chosen": -380.0, "logp/rejected": -346.0, "loss": 0.49139404296875, "reward/accuracy": 0.75, "reward/chosen": -1.640625, "reward/margin": 1.5234375, "reward/rejected": -3.171875, "step": 1238 }, { "approx. KL/chosen": 174.0, "approx. KL/rejected": 756.0, "epoch": 0.371625674865027, "grad_norm": 3.610478401184082, "learning_rate": 6.966610417774195e-06, "logp/chosen": -388.0, "logp/rejected": -430.0, "loss": 0.24700927734375, "reward/accuracy": 0.9375, "reward/chosen": -1.109375, "reward/margin": 2.125, "reward/rejected": -3.234375, "step": 1239 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 700.0, "epoch": 0.3719256148770246, "grad_norm": 3.282184600830078, "learning_rate": 6.9622778377095275e-06, "logp/chosen": -272.0, "logp/rejected": -254.0, "loss": 0.4404296875, "reward/accuracy": 0.8125, "reward/chosen": -1.609375, "reward/margin": 1.7265625, "reward/rejected": -3.34375, "step": 1240 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 1008.0, "epoch": 0.3722255548890222, "grad_norm": 4.953652858734131, "learning_rate": 6.957943515320452e-06, "logp/chosen": -260.0, "logp/rejected": -356.0, "loss": 0.46625518798828125, "reward/accuracy": 0.875, "reward/chosen": -2.296875, "reward/margin": 1.7421875, "reward/rejected": -4.03125, "step": 1241 }, { "approx. KL/chosen": 218.0, "approx. KL/rejected": 672.0, "epoch": 0.37252549490101977, "grad_norm": 2.84183931350708, "learning_rate": 6.953607454455449e-06, "logp/chosen": -368.0, "logp/rejected": -412.0, "loss": 0.326873779296875, "reward/accuracy": 0.875, "reward/chosen": -1.5, "reward/margin": 1.6171875, "reward/rejected": -3.125, "step": 1242 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 944.0, "epoch": 0.3728254349130174, "grad_norm": 3.5882582664489746, "learning_rate": 6.949269658964551e-06, "logp/chosen": -434.0, "logp/rejected": -330.0, "loss": 0.441314697265625, "reward/accuracy": 0.8125, "reward/chosen": -1.5625, "reward/margin": 2.1875, "reward/rejected": -3.75, "step": 1243 }, { "approx. KL/chosen": 76.5, "approx. KL/rejected": 444.0, "epoch": 0.373125374925015, "grad_norm": 3.4679276943206787, "learning_rate": 6.94493013269932e-06, "logp/chosen": -346.0, "logp/rejected": -314.0, "loss": 0.39202880859375, "reward/accuracy": 0.875, "reward/chosen": -1.0234375, "reward/margin": 1.4609375, "reward/rejected": -2.484375, "step": 1244 }, { "approx. KL/chosen": 240.0, "approx. KL/rejected": 596.0, "epoch": 0.3734253149370126, "grad_norm": 4.566427707672119, "learning_rate": 6.940588879512866e-06, "logp/chosen": -362.0, "logp/rejected": -452.0, "loss": 0.512054443359375, "reward/accuracy": 0.6875, "reward/chosen": -1.578125, "reward/margin": 1.484375, "reward/rejected": -3.0625, "step": 1245 }, { "approx. KL/chosen": 412.0, "approx. KL/rejected": 1016.0, "epoch": 0.3737252549490102, "grad_norm": 3.513392210006714, "learning_rate": 6.9362459032598264e-06, "logp/chosen": -374.0, "logp/rejected": -376.0, "loss": 0.34735107421875, "reward/accuracy": 0.75, "reward/chosen": -1.9375, "reward/margin": 1.96875, "reward/rejected": -3.90625, "step": 1246 }, { "approx. KL/chosen": 314.0, "approx. KL/rejected": 932.0, "epoch": 0.3740251949610078, "grad_norm": 4.378271102905273, "learning_rate": 6.931901207796369e-06, "logp/chosen": -302.0, "logp/rejected": -296.0, "loss": 0.534759521484375, "reward/accuracy": 0.75, "reward/chosen": -1.96875, "reward/margin": 1.7734375, "reward/rejected": -3.734375, "step": 1247 }, { "approx. KL/chosen": 394.0, "approx. KL/rejected": 1072.0, "epoch": 0.3743251349730054, "grad_norm": 4.91889762878418, "learning_rate": 6.927554796980189e-06, "logp/chosen": -294.0, "logp/rejected": -322.0, "loss": 0.656097412109375, "reward/accuracy": 0.625, "reward/chosen": -2.578125, "reward/margin": 1.34375, "reward/rejected": -3.921875, "step": 1248 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 1072.0, "epoch": 0.374625074985003, "grad_norm": 1.7033274173736572, "learning_rate": 6.923206674670503e-06, "logp/chosen": -290.0, "logp/rejected": -348.0, "loss": 0.1380615234375, "reward/accuracy": 1.0, "reward/chosen": -1.390625, "reward/margin": 2.875, "reward/rejected": -4.25, "step": 1249 }, { "approx. KL/chosen": 124.0, "approx. KL/rejected": 780.0, "epoch": 0.3749250149970006, "grad_norm": 7.422412395477295, "learning_rate": 6.918856844728049e-06, "logp/chosen": -338.0, "logp/rejected": -312.0, "loss": 0.2581977844238281, "reward/accuracy": 0.9375, "reward/chosen": -0.93359375, "reward/margin": 2.6875, "reward/rejected": -3.625, "step": 1250 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 836.0, "epoch": 0.3752249550089982, "grad_norm": 2.9356935024261475, "learning_rate": 6.914505311015083e-06, "logp/chosen": -448.0, "logp/rejected": -438.0, "loss": 0.274200439453125, "reward/accuracy": 0.875, "reward/chosen": -1.265625, "reward/margin": 2.28125, "reward/rejected": -3.546875, "step": 1251 }, { "approx. KL/chosen": 235.0, "approx. KL/rejected": 644.0, "epoch": 0.3755248950209958, "grad_norm": 5.4376654624938965, "learning_rate": 6.910152077395368e-06, "logp/chosen": -328.0, "logp/rejected": -316.0, "loss": 0.57086181640625, "reward/accuracy": 0.8125, "reward/chosen": -1.78125, "reward/margin": 1.359375, "reward/rejected": -3.140625, "step": 1252 }, { "approx. KL/chosen": 386.0, "approx. KL/rejected": 1120.0, "epoch": 0.3758248350329934, "grad_norm": 8.590632438659668, "learning_rate": 6.905797147734183e-06, "logp/chosen": -294.0, "logp/rejected": -354.0, "loss": 0.7242813110351562, "reward/accuracy": 0.625, "reward/chosen": -2.328125, "reward/margin": 1.7734375, "reward/rejected": -4.09375, "step": 1253 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 656.0, "epoch": 0.376124775044991, "grad_norm": 2.9167959690093994, "learning_rate": 6.9014405258983064e-06, "logp/chosen": -386.0, "logp/rejected": -286.0, "loss": 0.28900146484375, "reward/accuracy": 0.9375, "reward/chosen": -1.609375, "reward/margin": 1.6171875, "reward/rejected": -3.234375, "step": 1254 }, { "approx. KL/chosen": 374.0, "approx. KL/rejected": 956.0, "epoch": 0.3764247150569886, "grad_norm": 4.1013102531433105, "learning_rate": 6.897082215756026e-06, "logp/chosen": -178.0, "logp/rejected": -244.0, "loss": 0.33538818359375, "reward/accuracy": 0.8125, "reward/chosen": -2.0, "reward/margin": 1.8984375, "reward/rejected": -3.90625, "step": 1255 }, { "approx. KL/chosen": 416.0, "approx. KL/rejected": 1192.0, "epoch": 0.3767246550689862, "grad_norm": 2.663708209991455, "learning_rate": 6.8927222211771215e-06, "logp/chosen": -384.0, "logp/rejected": -334.0, "loss": 0.3193359375, "reward/accuracy": 0.75, "reward/chosen": -2.15625, "reward/margin": 2.125, "reward/rejected": -4.28125, "step": 1256 }, { "approx. KL/chosen": 168.0, "approx. KL/rejected": 524.0, "epoch": 0.3770245950809838, "grad_norm": 4.593882083892822, "learning_rate": 6.888360546032875e-06, "logp/chosen": -430.0, "logp/rejected": -396.0, "loss": 0.5672607421875, "reward/accuracy": 0.6875, "reward/chosen": -1.40625, "reward/margin": 1.4609375, "reward/rejected": -2.875, "step": 1257 }, { "approx. KL/chosen": 238.0, "approx. KL/rejected": 992.0, "epoch": 0.3773245350929814, "grad_norm": 4.331949710845947, "learning_rate": 6.883997194196058e-06, "logp/chosen": -326.0, "logp/rejected": -364.0, "loss": 0.35870361328125, "reward/accuracy": 0.8125, "reward/chosen": -1.75, "reward/margin": 2.09375, "reward/rejected": -3.84375, "step": 1258 }, { "approx. KL/chosen": 188.0, "approx. KL/rejected": 708.0, "epoch": 0.377624475104979, "grad_norm": 3.8573782444000244, "learning_rate": 6.879632169540927e-06, "logp/chosen": -221.0, "logp/rejected": -290.0, "loss": 0.308502197265625, "reward/accuracy": 0.875, "reward/chosen": -1.3984375, "reward/margin": 1.90625, "reward/rejected": -3.3125, "step": 1259 }, { "approx. KL/chosen": 219.0, "approx. KL/rejected": 1152.0, "epoch": 0.3779244151169766, "grad_norm": 3.4557857513427734, "learning_rate": 6.875265475943232e-06, "logp/chosen": -422.0, "logp/rejected": -404.0, "loss": 0.385498046875, "reward/accuracy": 0.8125, "reward/chosen": -1.6328125, "reward/margin": 2.109375, "reward/rejected": -3.734375, "step": 1260 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 1200.0, "epoch": 0.37822435512897423, "grad_norm": 2.5125794410705566, "learning_rate": 6.870897117280195e-06, "logp/chosen": -328.0, "logp/rejected": -392.0, "loss": 0.20953369140625, "reward/accuracy": 1.0, "reward/chosen": -1.828125, "reward/margin": 2.375, "reward/rejected": -4.21875, "step": 1261 }, { "approx. KL/chosen": 406.0, "approx. KL/rejected": 1000.0, "epoch": 0.3785242951409718, "grad_norm": 6.280098915100098, "learning_rate": 6.8665270974305246e-06, "logp/chosen": -308.0, "logp/rejected": -346.0, "loss": 0.641357421875, "reward/accuracy": 0.6875, "reward/chosen": -2.234375, "reward/margin": 1.2421875, "reward/rejected": -3.46875, "step": 1262 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 516.0, "epoch": 0.3788242351529694, "grad_norm": 3.9223527908325195, "learning_rate": 6.862155420274402e-06, "logp/chosen": -362.0, "logp/rejected": -386.0, "loss": 0.5635986328125, "reward/accuracy": 0.75, "reward/chosen": -1.2421875, "reward/margin": 1.21875, "reward/rejected": -2.46875, "step": 1263 }, { "approx. KL/chosen": 128.0, "approx. KL/rejected": 482.0, "epoch": 0.379124175164967, "grad_norm": 3.3827321529388428, "learning_rate": 6.857782089693476e-06, "logp/chosen": -370.0, "logp/rejected": -294.0, "loss": 0.387054443359375, "reward/accuracy": 0.8125, "reward/chosen": -0.81640625, "reward/margin": 1.5859375, "reward/rejected": -2.40625, "step": 1264 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 752.0, "epoch": 0.3794241151769646, "grad_norm": 4.055634021759033, "learning_rate": 6.853407109570871e-06, "logp/chosen": -370.0, "logp/rejected": -418.0, "loss": 0.419403076171875, "reward/accuracy": 0.625, "reward/chosen": -1.1015625, "reward/margin": 1.9609375, "reward/rejected": -3.0625, "step": 1265 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 484.0, "epoch": 0.3797240551889622, "grad_norm": 4.761357307434082, "learning_rate": 6.849030483791168e-06, "logp/chosen": -344.0, "logp/rejected": -312.0, "loss": 0.6507568359375, "reward/accuracy": 0.5625, "reward/chosen": -1.78125, "reward/margin": 0.640625, "reward/rejected": -2.421875, "step": 1266 }, { "approx. KL/chosen": 86.5, "approx. KL/rejected": 316.0, "epoch": 0.3800239952009598, "grad_norm": 4.495398044586182, "learning_rate": 6.844652216240416e-06, "logp/chosen": -249.0, "logp/rejected": -286.0, "loss": 0.5869140625, "reward/accuracy": 0.5625, "reward/chosen": -0.96484375, "reward/margin": 0.76953125, "reward/rejected": -1.734375, "step": 1267 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 428.0, "epoch": 0.38032393521295743, "grad_norm": 4.023683071136475, "learning_rate": 6.8402723108061154e-06, "logp/chosen": -398.0, "logp/rejected": -468.0, "loss": 0.4718017578125, "reward/accuracy": 0.8125, "reward/chosen": -1.3828125, "reward/margin": 1.1171875, "reward/rejected": -2.5, "step": 1268 }, { "approx. KL/chosen": 171.0, "approx. KL/rejected": 482.0, "epoch": 0.380623875224955, "grad_norm": 5.502941608428955, "learning_rate": 6.8358907713772275e-06, "logp/chosen": -233.0, "logp/rejected": -272.0, "loss": 0.481842041015625, "reward/accuracy": 0.8125, "reward/chosen": -1.34375, "reward/margin": 1.265625, "reward/rejected": -2.609375, "step": 1269 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 458.0, "epoch": 0.3809238152369526, "grad_norm": 7.663415431976318, "learning_rate": 6.8315076018441595e-06, "logp/chosen": -350.0, "logp/rejected": -470.0, "loss": 0.83892822265625, "reward/accuracy": 0.625, "reward/chosen": -1.6953125, "reward/margin": 0.6953125, "reward/rejected": -2.390625, "step": 1270 }, { "approx. KL/chosen": 308.0, "approx. KL/rejected": 1088.0, "epoch": 0.3812237552489502, "grad_norm": 6.464401721954346, "learning_rate": 6.827122806098767e-06, "logp/chosen": -430.0, "logp/rejected": -436.0, "loss": 0.544677734375, "reward/accuracy": 0.6875, "reward/chosen": -1.3359375, "reward/margin": 2.59375, "reward/rejected": -3.9375, "step": 1271 }, { "approx. KL/chosen": 63.5, "approx. KL/rejected": 604.0, "epoch": 0.3815236952609478, "grad_norm": 2.3770077228546143, "learning_rate": 6.822736388034353e-06, "logp/chosen": -326.0, "logp/rejected": -410.0, "loss": 0.18316650390625, "reward/accuracy": 1.0, "reward/chosen": -0.640625, "reward/margin": 2.40625, "reward/rejected": -3.046875, "step": 1272 }, { "approx. KL/chosen": 420.0, "approx. KL/rejected": 436.0, "epoch": 0.38182363527294544, "grad_norm": 6.837953090667725, "learning_rate": 6.818348351545655e-06, "logp/chosen": -408.0, "logp/rejected": -356.0, "loss": 0.83154296875, "reward/accuracy": 0.75, "reward/chosen": -2.0, "reward/margin": 0.49609375, "reward/rejected": -2.5, "step": 1273 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 620.0, "epoch": 0.382123575284943, "grad_norm": 4.731329917907715, "learning_rate": 6.813958700528851e-06, "logp/chosen": -211.0, "logp/rejected": -220.0, "loss": 0.37432861328125, "reward/accuracy": 0.875, "reward/chosen": -1.25, "reward/margin": 1.734375, "reward/rejected": -2.984375, "step": 1274 }, { "approx. KL/chosen": 188.0, "approx. KL/rejected": 320.0, "epoch": 0.38242351529694063, "grad_norm": 4.744907855987549, "learning_rate": 6.809567438881556e-06, "logp/chosen": -336.0, "logp/rejected": -298.0, "loss": 0.5167236328125, "reward/accuracy": 0.75, "reward/chosen": -1.15625, "reward/margin": 0.9609375, "reward/rejected": -2.109375, "step": 1275 }, { "approx. KL/chosen": 148.0, "approx. KL/rejected": 744.0, "epoch": 0.3827234553089382, "grad_norm": 3.629058837890625, "learning_rate": 6.805174570502806e-06, "logp/chosen": -436.0, "logp/rejected": -380.0, "loss": 0.3782958984375, "reward/accuracy": 0.75, "reward/chosen": -1.3046875, "reward/margin": 1.9375, "reward/rejected": -3.234375, "step": 1276 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 324.0, "epoch": 0.3830233953209358, "grad_norm": 4.724060535430908, "learning_rate": 6.800780099293073e-06, "logp/chosen": -350.0, "logp/rejected": -326.0, "loss": 0.751953125, "reward/accuracy": 0.6875, "reward/chosen": -1.4453125, "reward/margin": 0.578125, "reward/rejected": -2.03125, "step": 1277 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 828.0, "epoch": 0.3833233353329334, "grad_norm": 3.85445237159729, "learning_rate": 6.796384029154246e-06, "logp/chosen": -326.0, "logp/rejected": -248.0, "loss": 0.389862060546875, "reward/accuracy": 0.8125, "reward/chosen": -1.3046875, "reward/margin": 2.0, "reward/rejected": -3.3125, "step": 1278 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 576.0, "epoch": 0.383623275344931, "grad_norm": 4.175628185272217, "learning_rate": 6.791986363989637e-06, "logp/chosen": -296.0, "logp/rejected": -255.0, "loss": 0.4630126953125, "reward/accuracy": 0.8125, "reward/chosen": -1.3046875, "reward/margin": 1.4921875, "reward/rejected": -2.796875, "step": 1279 }, { "approx. KL/chosen": 72.0, "approx. KL/rejected": 656.0, "epoch": 0.38392321535692864, "grad_norm": 4.013622283935547, "learning_rate": 6.78758710770397e-06, "logp/chosen": -312.0, "logp/rejected": -288.0, "loss": 0.33172607421875, "reward/accuracy": 0.875, "reward/chosen": -0.9375, "reward/margin": 2.015625, "reward/rejected": -2.953125, "step": 1280 }, { "approx. KL/chosen": 113.0, "approx. KL/rejected": 568.0, "epoch": 0.3842231553689262, "grad_norm": 3.522439956665039, "learning_rate": 6.7831862642033886e-06, "logp/chosen": -290.0, "logp/rejected": -286.0, "loss": 0.318878173828125, "reward/accuracy": 0.875, "reward/chosen": -0.9375, "reward/margin": 1.9921875, "reward/rejected": -2.9375, "step": 1281 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 486.0, "epoch": 0.38452309538092383, "grad_norm": 4.740118980407715, "learning_rate": 6.7787838373954415e-06, "logp/chosen": -372.0, "logp/rejected": -312.0, "loss": 0.73919677734375, "reward/accuracy": 0.5, "reward/chosen": -1.5859375, "reward/margin": 0.765625, "reward/rejected": -2.359375, "step": 1282 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 780.0, "epoch": 0.3848230353929214, "grad_norm": 5.357082366943359, "learning_rate": 6.7743798311890816e-06, "logp/chosen": -288.0, "logp/rejected": -322.0, "loss": 0.5509033203125, "reward/accuracy": 0.75, "reward/chosen": -1.0703125, "reward/margin": 1.96875, "reward/rejected": -3.03125, "step": 1283 }, { "approx. KL/chosen": 60.0, "approx. KL/rejected": 304.0, "epoch": 0.385122975404919, "grad_norm": 3.3571889400482178, "learning_rate": 6.7699742494946686e-06, "logp/chosen": -288.0, "logp/rejected": -262.0, "loss": 0.4066162109375, "reward/accuracy": 0.75, "reward/chosen": -0.61328125, "reward/margin": 1.21875, "reward/rejected": -1.8359375, "step": 1284 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 652.0, "epoch": 0.3854229154169166, "grad_norm": 4.883366584777832, "learning_rate": 6.7655670962239565e-06, "logp/chosen": -384.0, "logp/rejected": -362.0, "loss": 0.290374755859375, "reward/accuracy": 0.9375, "reward/chosen": -1.203125, "reward/margin": 2.0, "reward/rejected": -3.203125, "step": 1285 }, { "approx. KL/chosen": 213.0, "approx. KL/rejected": 988.0, "epoch": 0.3857228554289142, "grad_norm": 4.766938209533691, "learning_rate": 6.761158375290099e-06, "logp/chosen": -332.0, "logp/rejected": -636.0, "loss": 0.43902587890625, "reward/accuracy": 0.8125, "reward/chosen": -1.34375, "reward/margin": 2.28125, "reward/rejected": -3.640625, "step": 1286 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 1272.0, "epoch": 0.38602279544091184, "grad_norm": 2.7294297218322754, "learning_rate": 6.756748090607637e-06, "logp/chosen": -418.0, "logp/rejected": -366.0, "loss": 0.23712539672851562, "reward/accuracy": 0.875, "reward/chosen": -1.3828125, "reward/margin": 3.21875, "reward/rejected": -4.59375, "step": 1287 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 972.0, "epoch": 0.3863227354529094, "grad_norm": 3.3294665813446045, "learning_rate": 6.752336246092505e-06, "logp/chosen": -292.0, "logp/rejected": -326.0, "loss": 0.417266845703125, "reward/accuracy": 0.8125, "reward/chosen": -1.8046875, "reward/margin": 1.9609375, "reward/rejected": -3.765625, "step": 1288 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 544.0, "epoch": 0.38662267546490703, "grad_norm": 3.4000189304351807, "learning_rate": 6.74792284566202e-06, "logp/chosen": -320.0, "logp/rejected": -344.0, "loss": 0.382171630859375, "reward/accuracy": 0.8125, "reward/chosen": -1.3671875, "reward/margin": 1.2265625, "reward/rejected": -2.59375, "step": 1289 }, { "approx. KL/chosen": 308.0, "approx. KL/rejected": 760.0, "epoch": 0.3869226154769046, "grad_norm": 8.112146377563477, "learning_rate": 6.743507893234878e-06, "logp/chosen": -336.0, "logp/rejected": -288.0, "loss": 0.45758056640625, "reward/accuracy": 0.75, "reward/chosen": -2.0625, "reward/margin": 1.375, "reward/rejected": -3.4375, "step": 1290 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 434.0, "epoch": 0.3872225554889022, "grad_norm": 3.370961904525757, "learning_rate": 6.7390913927311585e-06, "logp/chosen": -314.0, "logp/rejected": -358.0, "loss": 0.437225341796875, "reward/accuracy": 0.6875, "reward/chosen": -1.125, "reward/margin": 1.234375, "reward/rejected": -2.359375, "step": 1291 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 1288.0, "epoch": 0.38752249550089984, "grad_norm": 2.6555323600769043, "learning_rate": 6.734673348072309e-06, "logp/chosen": -390.0, "logp/rejected": -444.0, "loss": 0.2659759521484375, "reward/accuracy": 0.9375, "reward/chosen": -1.375, "reward/margin": 2.671875, "reward/rejected": -4.0625, "step": 1292 }, { "approx. KL/chosen": 243.0, "approx. KL/rejected": 684.0, "epoch": 0.3878224355128974, "grad_norm": 3.9826431274414062, "learning_rate": 6.730253763181156e-06, "logp/chosen": -422.0, "logp/rejected": -378.0, "loss": 0.38338470458984375, "reward/accuracy": 0.6875, "reward/chosen": -1.4140625, "reward/margin": 1.9453125, "reward/rejected": -3.359375, "step": 1293 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 620.0, "epoch": 0.38812237552489504, "grad_norm": 5.7368316650390625, "learning_rate": 6.725832641981888e-06, "logp/chosen": -336.0, "logp/rejected": -342.0, "loss": 0.68450927734375, "reward/accuracy": 0.5625, "reward/chosen": -1.7109375, "reward/margin": 0.8984375, "reward/rejected": -2.609375, "step": 1294 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 580.0, "epoch": 0.3884223155368926, "grad_norm": 3.7574329376220703, "learning_rate": 6.721409988400057e-06, "logp/chosen": -426.0, "logp/rejected": -430.0, "loss": 0.4107666015625, "reward/accuracy": 0.875, "reward/chosen": -1.375, "reward/margin": 1.484375, "reward/rejected": -2.859375, "step": 1295 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 540.0, "epoch": 0.3887222555488902, "grad_norm": 4.703813552856445, "learning_rate": 6.716985806362581e-06, "logp/chosen": -233.0, "logp/rejected": -221.0, "loss": 0.567626953125, "reward/accuracy": 0.5625, "reward/chosen": -1.59375, "reward/margin": 1.0390625, "reward/rejected": -2.640625, "step": 1296 }, { "approx. KL/chosen": 142.0, "approx. KL/rejected": 916.0, "epoch": 0.3890221955608878, "grad_norm": 2.6021692752838135, "learning_rate": 6.712560099797727e-06, "logp/chosen": -202.0, "logp/rejected": -233.0, "loss": 0.203125, "reward/accuracy": 0.9375, "reward/chosen": -1.296875, "reward/margin": 2.546875, "reward/rejected": -3.84375, "step": 1297 }, { "approx. KL/chosen": 402.0, "approx. KL/rejected": 470.0, "epoch": 0.3893221355728854, "grad_norm": 5.8376359939575195, "learning_rate": 6.708132872635124e-06, "logp/chosen": -408.0, "logp/rejected": -388.0, "loss": 0.8729248046875, "reward/accuracy": 0.5625, "reward/chosen": -2.03125, "reward/margin": 0.69921875, "reward/rejected": -2.734375, "step": 1298 }, { "approx. KL/chosen": 105.5, "approx. KL/rejected": 764.0, "epoch": 0.38962207558488304, "grad_norm": 3.6625664234161377, "learning_rate": 6.703704128805745e-06, "logp/chosen": -298.0, "logp/rejected": -316.0, "loss": 0.4213733673095703, "reward/accuracy": 0.8125, "reward/chosen": -1.21875, "reward/margin": 1.875, "reward/rejected": -3.109375, "step": 1299 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 426.0, "epoch": 0.3899220155968806, "grad_norm": 3.3288204669952393, "learning_rate": 6.699273872241912e-06, "logp/chosen": -358.0, "logp/rejected": -284.0, "loss": 0.45208740234375, "reward/accuracy": 0.8125, "reward/chosen": -1.3125, "reward/margin": 1.1953125, "reward/rejected": -2.5, "step": 1300 }, { "approx. KL/chosen": 172.0, "approx. KL/rejected": 446.0, "epoch": 0.39022195560887823, "grad_norm": 4.351337432861328, "learning_rate": 6.694842106877295e-06, "logp/chosen": -274.0, "logp/rejected": -400.0, "loss": 0.415802001953125, "reward/accuracy": 0.75, "reward/chosen": -1.375, "reward/margin": 1.296875, "reward/rejected": -2.671875, "step": 1301 }, { "approx. KL/chosen": 106.5, "approx. KL/rejected": 260.0, "epoch": 0.3905218956208758, "grad_norm": 5.091721057891846, "learning_rate": 6.690408836646892e-06, "logp/chosen": -235.0, "logp/rejected": -233.0, "loss": 0.7738037109375, "reward/accuracy": 0.625, "reward/chosen": -0.8515625, "reward/margin": 0.7890625, "reward/rejected": -1.640625, "step": 1302 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 1024.0, "epoch": 0.3908218356328734, "grad_norm": 3.6119635105133057, "learning_rate": 6.68597406548705e-06, "logp/chosen": -492.0, "logp/rejected": -390.0, "loss": 0.33544921875, "reward/accuracy": 0.8125, "reward/chosen": -1.234375, "reward/margin": 2.515625, "reward/rejected": -3.75, "step": 1303 }, { "approx. KL/chosen": 406.0, "approx. KL/rejected": 792.0, "epoch": 0.39112177564487105, "grad_norm": 4.5775628089904785, "learning_rate": 6.681537797335439e-06, "logp/chosen": -368.0, "logp/rejected": -306.0, "loss": 0.490203857421875, "reward/accuracy": 0.75, "reward/chosen": -1.875, "reward/margin": 1.6015625, "reward/rejected": -3.46875, "step": 1304 }, { "approx. KL/chosen": 314.0, "approx. KL/rejected": 644.0, "epoch": 0.3914217156568686, "grad_norm": 8.424474716186523, "learning_rate": 6.677100036131062e-06, "logp/chosen": -326.0, "logp/rejected": -260.0, "loss": 0.67291259765625, "reward/accuracy": 0.5625, "reward/chosen": -2.109375, "reward/margin": 0.83203125, "reward/rejected": -2.9375, "step": 1305 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 1004.0, "epoch": 0.39172165566886624, "grad_norm": 4.405363082885742, "learning_rate": 6.672660785814251e-06, "logp/chosen": -233.0, "logp/rejected": -332.0, "loss": 0.38909912109375, "reward/accuracy": 0.8125, "reward/chosen": -1.4921875, "reward/margin": 2.25, "reward/rejected": -3.75, "step": 1306 }, { "approx. KL/chosen": 255.0, "approx. KL/rejected": 402.0, "epoch": 0.3920215956808638, "grad_norm": 3.6584742069244385, "learning_rate": 6.6682200503266545e-06, "logp/chosen": -256.0, "logp/rejected": -219.0, "loss": 0.5107421875, "reward/accuracy": 0.75, "reward/chosen": -1.6328125, "reward/margin": 0.90234375, "reward/rejected": -2.53125, "step": 1307 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 544.0, "epoch": 0.39232153569286143, "grad_norm": 3.8786520957946777, "learning_rate": 6.663777833611243e-06, "logp/chosen": -245.0, "logp/rejected": -338.0, "loss": 0.35662841796875, "reward/accuracy": 0.875, "reward/chosen": -1.4609375, "reward/margin": 1.28125, "reward/rejected": -2.734375, "step": 1308 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 716.0, "epoch": 0.392621475704859, "grad_norm": 3.6171982288360596, "learning_rate": 6.659334139612299e-06, "logp/chosen": -314.0, "logp/rejected": -310.0, "loss": 0.520355224609375, "reward/accuracy": 0.75, "reward/chosen": -1.5390625, "reward/margin": 1.7265625, "reward/rejected": -3.265625, "step": 1309 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 476.0, "epoch": 0.3929214157168566, "grad_norm": 2.4717302322387695, "learning_rate": 6.654888972275423e-06, "logp/chosen": -233.0, "logp/rejected": -268.0, "loss": 0.339111328125, "reward/accuracy": 0.875, "reward/chosen": -1.1875, "reward/margin": 1.359375, "reward/rejected": -2.546875, "step": 1310 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 544.0, "epoch": 0.39322135572885425, "grad_norm": 3.088326930999756, "learning_rate": 6.650442335547516e-06, "logp/chosen": -254.0, "logp/rejected": -280.0, "loss": 0.34881591796875, "reward/accuracy": 0.75, "reward/chosen": -1.1953125, "reward/margin": 1.578125, "reward/rejected": -2.78125, "step": 1311 }, { "approx. KL/chosen": 185.0, "approx. KL/rejected": 652.0, "epoch": 0.3935212957408518, "grad_norm": 3.559748411178589, "learning_rate": 6.645994233376788e-06, "logp/chosen": -314.0, "logp/rejected": -288.0, "loss": 0.38494873046875, "reward/accuracy": 0.8125, "reward/chosen": -1.546875, "reward/margin": 1.578125, "reward/rejected": -3.125, "step": 1312 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 576.0, "epoch": 0.39382123575284944, "grad_norm": 4.730678558349609, "learning_rate": 6.641544669712752e-06, "logp/chosen": -384.0, "logp/rejected": -450.0, "loss": 0.54229736328125, "reward/accuracy": 0.75, "reward/chosen": -1.40625, "reward/margin": 1.2421875, "reward/rejected": -2.65625, "step": 1313 }, { "approx. KL/chosen": 180.0, "approx. KL/rejected": 466.0, "epoch": 0.394121175764847, "grad_norm": 4.991402626037598, "learning_rate": 6.637093648506212e-06, "logp/chosen": -232.0, "logp/rejected": -310.0, "loss": 0.6141357421875, "reward/accuracy": 0.75, "reward/chosen": -1.5546875, "reward/margin": 0.9765625, "reward/rejected": -2.53125, "step": 1314 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 620.0, "epoch": 0.39442111577684463, "grad_norm": 2.847857713699341, "learning_rate": 6.632641173709276e-06, "logp/chosen": -253.0, "logp/rejected": -286.0, "loss": 0.272064208984375, "reward/accuracy": 0.875, "reward/chosen": -1.2734375, "reward/margin": 1.8515625, "reward/rejected": -3.125, "step": 1315 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 512.0, "epoch": 0.39472105578884226, "grad_norm": 2.886124610900879, "learning_rate": 6.628187249275331e-06, "logp/chosen": -410.0, "logp/rejected": -370.0, "loss": 0.412811279296875, "reward/accuracy": 0.75, "reward/chosen": -1.375, "reward/margin": 1.359375, "reward/rejected": -2.734375, "step": 1316 }, { "approx. KL/chosen": 83.0, "approx. KL/rejected": 508.0, "epoch": 0.3950209958008398, "grad_norm": 3.8561952114105225, "learning_rate": 6.623731879159061e-06, "logp/chosen": -300.0, "logp/rejected": -278.0, "loss": 0.43878173828125, "reward/accuracy": 0.75, "reward/chosen": -0.9453125, "reward/margin": 1.75, "reward/rejected": -2.6875, "step": 1317 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 246.0, "epoch": 0.39532093581283745, "grad_norm": 6.119197368621826, "learning_rate": 6.619275067316429e-06, "logp/chosen": -236.0, "logp/rejected": -282.0, "loss": 0.692138671875, "reward/accuracy": 0.5625, "reward/chosen": -1.3671875, "reward/margin": 0.44140625, "reward/rejected": -1.8046875, "step": 1318 }, { "approx. KL/chosen": 123.0, "approx. KL/rejected": 1048.0, "epoch": 0.395620875824835, "grad_norm": 3.1951591968536377, "learning_rate": 6.614816817704679e-06, "logp/chosen": -358.0, "logp/rejected": -480.0, "loss": 0.30560302734375, "reward/accuracy": 0.875, "reward/chosen": -1.2578125, "reward/margin": 2.5625, "reward/rejected": -3.828125, "step": 1319 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 616.0, "epoch": 0.39592081583683264, "grad_norm": 3.4971439838409424, "learning_rate": 6.610357134282332e-06, "logp/chosen": -342.0, "logp/rejected": -322.0, "loss": 0.38946533203125, "reward/accuracy": 0.9375, "reward/chosen": -1.21875, "reward/margin": 1.7265625, "reward/rejected": -2.9375, "step": 1320 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 692.0, "epoch": 0.3962207558488302, "grad_norm": 4.969843864440918, "learning_rate": 6.6058960210091806e-06, "logp/chosen": -318.0, "logp/rejected": -300.0, "loss": 0.36376953125, "reward/accuracy": 0.8125, "reward/chosen": -1.3359375, "reward/margin": 2.078125, "reward/rejected": -3.421875, "step": 1321 }, { "approx. KL/chosen": 234.0, "approx. KL/rejected": 412.0, "epoch": 0.39652069586082783, "grad_norm": 5.795616149902344, "learning_rate": 6.601433481846289e-06, "logp/chosen": -308.0, "logp/rejected": -302.0, "loss": 0.7265625, "reward/accuracy": 0.625, "reward/chosen": -1.6328125, "reward/margin": 0.62890625, "reward/rejected": -2.265625, "step": 1322 }, { "approx. KL/chosen": 150.0, "approx. KL/rejected": 796.0, "epoch": 0.39682063587282546, "grad_norm": 3.9628231525421143, "learning_rate": 6.596969520755985e-06, "logp/chosen": -412.0, "logp/rejected": -340.0, "loss": 0.436309814453125, "reward/accuracy": 0.75, "reward/chosen": -1.2578125, "reward/margin": 2.265625, "reward/rejected": -3.515625, "step": 1323 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 636.0, "epoch": 0.397120575884823, "grad_norm": 4.136422157287598, "learning_rate": 6.59250414170186e-06, "logp/chosen": -348.0, "logp/rejected": -312.0, "loss": 0.4517822265625, "reward/accuracy": 0.75, "reward/chosen": -1.7734375, "reward/margin": 1.2109375, "reward/rejected": -2.984375, "step": 1324 }, { "approx. KL/chosen": 138.0, "approx. KL/rejected": 768.0, "epoch": 0.39742051589682065, "grad_norm": 3.911156177520752, "learning_rate": 6.588037348648768e-06, "logp/chosen": -338.0, "logp/rejected": -360.0, "loss": 0.47540283203125, "reward/accuracy": 0.6875, "reward/chosen": -1.2890625, "reward/margin": 1.9375, "reward/rejected": -3.234375, "step": 1325 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 920.0, "epoch": 0.3977204559088182, "grad_norm": 2.3645005226135254, "learning_rate": 6.583569145562813e-06, "logp/chosen": -268.0, "logp/rejected": -378.0, "loss": 0.1826171875, "reward/accuracy": 0.9375, "reward/chosen": -1.375, "reward/margin": 2.59375, "reward/rejected": -3.96875, "step": 1326 }, { "approx. KL/chosen": 308.0, "approx. KL/rejected": 700.0, "epoch": 0.39802039592081584, "grad_norm": 6.724916934967041, "learning_rate": 6.579099536411353e-06, "logp/chosen": -310.0, "logp/rejected": -350.0, "loss": 0.4727783203125, "reward/accuracy": 0.8125, "reward/chosen": -1.671875, "reward/margin": 1.578125, "reward/rejected": -3.25, "step": 1327 }, { "approx. KL/chosen": 102.5, "approx. KL/rejected": 486.0, "epoch": 0.39832033593281346, "grad_norm": 4.311374187469482, "learning_rate": 6.574628525162996e-06, "logp/chosen": -448.0, "logp/rejected": -344.0, "loss": 0.31842041015625, "reward/accuracy": 0.8125, "reward/chosen": -1.1015625, "reward/margin": 1.7734375, "reward/rejected": -2.875, "step": 1328 }, { "approx. KL/chosen": 87.0, "approx. KL/rejected": 668.0, "epoch": 0.39862027594481103, "grad_norm": 3.6015965938568115, "learning_rate": 6.570156115787593e-06, "logp/chosen": -328.0, "logp/rejected": -340.0, "loss": 0.34841156005859375, "reward/accuracy": 0.8125, "reward/chosen": -1.03125, "reward/margin": 2.15625, "reward/rejected": -3.1875, "step": 1329 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 676.0, "epoch": 0.39892021595680865, "grad_norm": 3.166166305541992, "learning_rate": 6.565682312256238e-06, "logp/chosen": -322.0, "logp/rejected": -342.0, "loss": 0.3797607421875, "reward/accuracy": 0.8125, "reward/chosen": -1.375, "reward/margin": 1.6484375, "reward/rejected": -3.03125, "step": 1330 }, { "approx. KL/chosen": 358.0, "approx. KL/rejected": 524.0, "epoch": 0.3992201559688062, "grad_norm": 6.605685234069824, "learning_rate": 6.5612071185412606e-06, "logp/chosen": -264.0, "logp/rejected": -253.0, "loss": 0.5538330078125, "reward/accuracy": 0.75, "reward/chosen": -1.859375, "reward/margin": 0.9296875, "reward/rejected": -2.78125, "step": 1331 }, { "approx. KL/chosen": 243.0, "approx. KL/rejected": 596.0, "epoch": 0.39952009598080385, "grad_norm": 4.144354820251465, "learning_rate": 6.556730538616228e-06, "logp/chosen": -376.0, "logp/rejected": -314.0, "loss": 0.45263671875, "reward/accuracy": 0.6875, "reward/chosen": -1.671875, "reward/margin": 1.1953125, "reward/rejected": -2.859375, "step": 1332 }, { "approx. KL/chosen": 304.0, "approx. KL/rejected": 756.0, "epoch": 0.3998200359928014, "grad_norm": 3.7334089279174805, "learning_rate": 6.552252576455936e-06, "logp/chosen": -406.0, "logp/rejected": -338.0, "loss": 0.36383056640625, "reward/accuracy": 0.8125, "reward/chosen": -1.75, "reward/margin": 1.609375, "reward/rejected": -3.359375, "step": 1333 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 880.0, "epoch": 0.40011997600479904, "grad_norm": 3.275937557220459, "learning_rate": 6.547773236036406e-06, "logp/chosen": -306.0, "logp/rejected": -300.0, "loss": 0.375244140625, "reward/accuracy": 0.8125, "reward/chosen": -1.53125, "reward/margin": 1.8359375, "reward/rejected": -3.359375, "step": 1334 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 474.0, "epoch": 0.40041991601679666, "grad_norm": 6.953616142272949, "learning_rate": 6.543292521334886e-06, "logp/chosen": -284.0, "logp/rejected": -262.0, "loss": 0.77899169921875, "reward/accuracy": 0.75, "reward/chosen": -1.4296875, "reward/margin": 1.03125, "reward/rejected": -2.453125, "step": 1335 }, { "approx. KL/chosen": 394.0, "approx. KL/rejected": 660.0, "epoch": 0.40071985602879423, "grad_norm": 6.8483476638793945, "learning_rate": 6.538810436329844e-06, "logp/chosen": -452.0, "logp/rejected": -348.0, "loss": 0.7138671875, "reward/accuracy": 0.6875, "reward/chosen": -1.8359375, "reward/margin": 1.2265625, "reward/rejected": -3.0625, "step": 1336 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 382.0, "epoch": 0.40101979604079185, "grad_norm": 4.455861568450928, "learning_rate": 6.5343269850009675e-06, "logp/chosen": -452.0, "logp/rejected": -322.0, "loss": 0.6246337890625, "reward/accuracy": 0.4375, "reward/chosen": -1.3828125, "reward/margin": 0.65234375, "reward/rejected": -2.03125, "step": 1337 }, { "approx. KL/chosen": 85.0, "approx. KL/rejected": 852.0, "epoch": 0.4013197360527894, "grad_norm": 3.037095069885254, "learning_rate": 6.529842171329147e-06, "logp/chosen": -248.0, "logp/rejected": -372.0, "loss": 0.41943359375, "reward/accuracy": 0.6875, "reward/chosen": -0.9296875, "reward/margin": 2.25, "reward/rejected": -3.171875, "step": 1338 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 1168.0, "epoch": 0.40161967606478705, "grad_norm": 3.703430414199829, "learning_rate": 6.525355999296497e-06, "logp/chosen": -256.0, "logp/rejected": -302.0, "loss": 0.2631378173828125, "reward/accuracy": 0.875, "reward/chosen": -1.3203125, "reward/margin": 2.953125, "reward/rejected": -4.28125, "step": 1339 }, { "approx. KL/chosen": 71.0, "approx. KL/rejected": 912.0, "epoch": 0.40191961607678467, "grad_norm": 3.200831890106201, "learning_rate": 6.520868472886324e-06, "logp/chosen": -249.0, "logp/rejected": -346.0, "loss": 0.470733642578125, "reward/accuracy": 0.8125, "reward/chosen": -0.80078125, "reward/margin": 2.1875, "reward/rejected": -3.0, "step": 1340 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 494.0, "epoch": 0.40221955608878224, "grad_norm": 3.7251644134521484, "learning_rate": 6.5163795960831465e-06, "logp/chosen": -348.0, "logp/rejected": -282.0, "loss": 0.403411865234375, "reward/accuracy": 0.875, "reward/chosen": -0.953125, "reward/margin": 1.46875, "reward/rejected": -2.421875, "step": 1341 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 756.0, "epoch": 0.40251949610077986, "grad_norm": 4.32686710357666, "learning_rate": 6.5118893728726805e-06, "logp/chosen": -308.0, "logp/rejected": -490.0, "loss": 0.4862556457519531, "reward/accuracy": 0.6875, "reward/chosen": -1.7890625, "reward/margin": 1.6015625, "reward/rejected": -3.390625, "step": 1342 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 506.0, "epoch": 0.40281943611277743, "grad_norm": 4.431898593902588, "learning_rate": 6.5073978072418355e-06, "logp/chosen": -370.0, "logp/rejected": -340.0, "loss": 0.57470703125, "reward/accuracy": 0.8125, "reward/chosen": -1.21875, "reward/margin": 1.4296875, "reward/rejected": -2.640625, "step": 1343 }, { "approx. KL/chosen": 150.0, "approx. KL/rejected": 524.0, "epoch": 0.40311937612477505, "grad_norm": 2.848728895187378, "learning_rate": 6.502904903178712e-06, "logp/chosen": -242.0, "logp/rejected": -243.0, "loss": 0.33154296875, "reward/accuracy": 0.875, "reward/chosen": -1.21875, "reward/margin": 1.7109375, "reward/rejected": -2.9375, "step": 1344 }, { "approx. KL/chosen": 97.5, "approx. KL/rejected": 628.0, "epoch": 0.4034193161367726, "grad_norm": 3.0424914360046387, "learning_rate": 6.498410664672602e-06, "logp/chosen": -338.0, "logp/rejected": -396.0, "loss": 0.3829345703125, "reward/accuracy": 0.8125, "reward/chosen": -1.234375, "reward/margin": 1.6875, "reward/rejected": -2.921875, "step": 1345 }, { "approx. KL/chosen": 406.0, "approx. KL/rejected": 450.0, "epoch": 0.40371925614877024, "grad_norm": 6.378050804138184, "learning_rate": 6.493915095713981e-06, "logp/chosen": -294.0, "logp/rejected": -358.0, "loss": 0.67010498046875, "reward/accuracy": 0.75, "reward/chosen": -1.5859375, "reward/margin": 0.9375, "reward/rejected": -2.515625, "step": 1346 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 792.0, "epoch": 0.40401919616076787, "grad_norm": 4.451230049133301, "learning_rate": 6.489418200294504e-06, "logp/chosen": -444.0, "logp/rejected": -392.0, "loss": 0.502166748046875, "reward/accuracy": 0.8125, "reward/chosen": -1.2421875, "reward/margin": 2.015625, "reward/rejected": -3.25, "step": 1347 }, { "approx. KL/chosen": 138.0, "approx. KL/rejected": 444.0, "epoch": 0.40431913617276544, "grad_norm": 4.481479644775391, "learning_rate": 6.484919982407007e-06, "logp/chosen": -368.0, "logp/rejected": -296.0, "loss": 0.45709228515625, "reward/accuracy": 0.6875, "reward/chosen": -1.0078125, "reward/margin": 1.2421875, "reward/rejected": -2.25, "step": 1348 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 510.0, "epoch": 0.40461907618476306, "grad_norm": 5.495540142059326, "learning_rate": 6.480420446045499e-06, "logp/chosen": -368.0, "logp/rejected": -320.0, "loss": 0.633087158203125, "reward/accuracy": 0.6875, "reward/chosen": -1.7421875, "reward/margin": 0.95703125, "reward/rejected": -2.703125, "step": 1349 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 672.0, "epoch": 0.40491901619676063, "grad_norm": 5.031222820281982, "learning_rate": 6.475919595205159e-06, "logp/chosen": -364.0, "logp/rejected": -444.0, "loss": 0.5723419189453125, "reward/accuracy": 0.8125, "reward/chosen": -1.4921875, "reward/margin": 1.671875, "reward/rejected": -3.15625, "step": 1350 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 664.0, "epoch": 0.40521895620875825, "grad_norm": 3.4726126194000244, "learning_rate": 6.471417433882334e-06, "logp/chosen": -334.0, "logp/rejected": -396.0, "loss": 0.335784912109375, "reward/accuracy": 0.8125, "reward/chosen": -1.4296875, "reward/margin": 1.9609375, "reward/rejected": -3.390625, "step": 1351 }, { "approx. KL/chosen": 255.0, "approx. KL/rejected": 508.0, "epoch": 0.4055188962207559, "grad_norm": 6.412424564361572, "learning_rate": 6.466913966074533e-06, "logp/chosen": -380.0, "logp/rejected": -404.0, "loss": 0.81781005859375, "reward/accuracy": 0.5625, "reward/chosen": -1.5390625, "reward/margin": 1.046875, "reward/rejected": -2.578125, "step": 1352 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 420.0, "epoch": 0.40581883623275344, "grad_norm": 5.091787338256836, "learning_rate": 6.462409195780425e-06, "logp/chosen": -420.0, "logp/rejected": -400.0, "loss": 0.80499267578125, "reward/accuracy": 0.4375, "reward/chosen": -1.46875, "reward/margin": 0.6171875, "reward/rejected": -2.078125, "step": 1353 }, { "approx. KL/chosen": 75.0, "approx. KL/rejected": 490.0, "epoch": 0.40611877624475107, "grad_norm": 1.8811018466949463, "learning_rate": 6.457903126999843e-06, "logp/chosen": -304.0, "logp/rejected": -310.0, "loss": 0.27105712890625, "reward/accuracy": 0.9375, "reward/chosen": -0.82421875, "reward/margin": 1.8046875, "reward/rejected": -2.625, "step": 1354 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 776.0, "epoch": 0.40641871625674864, "grad_norm": 2.7277112007141113, "learning_rate": 6.453395763733758e-06, "logp/chosen": -266.0, "logp/rejected": -376.0, "loss": 0.2890472412109375, "reward/accuracy": 0.8125, "reward/chosen": -0.97265625, "reward/margin": 2.375, "reward/rejected": -3.34375, "step": 1355 }, { "approx. KL/chosen": 57.25, "approx. KL/rejected": 796.0, "epoch": 0.40671865626874626, "grad_norm": 1.7812730073928833, "learning_rate": 6.4488871099843056e-06, "logp/chosen": -338.0, "logp/rejected": -324.0, "loss": 0.19760417938232422, "reward/accuracy": 1.0, "reward/chosen": -0.6015625, "reward/margin": 2.4375, "reward/rejected": -3.03125, "step": 1356 }, { "approx. KL/chosen": 98.0, "approx. KL/rejected": 720.0, "epoch": 0.4070185962807438, "grad_norm": 2.6966865062713623, "learning_rate": 6.4443771697547565e-06, "logp/chosen": -211.0, "logp/rejected": -322.0, "loss": 0.284820556640625, "reward/accuracy": 0.875, "reward/chosen": -0.9453125, "reward/margin": 2.21875, "reward/rejected": -3.15625, "step": 1357 }, { "approx. KL/chosen": 133.0, "approx. KL/rejected": 608.0, "epoch": 0.40731853629274145, "grad_norm": 2.976689577102661, "learning_rate": 6.4398659470495305e-06, "logp/chosen": -398.0, "logp/rejected": -342.0, "loss": 0.264129638671875, "reward/accuracy": 0.875, "reward/chosen": -0.7109375, "reward/margin": 2.140625, "reward/rejected": -2.859375, "step": 1358 }, { "approx. KL/chosen": 215.0, "approx. KL/rejected": 462.0, "epoch": 0.4076184763047391, "grad_norm": 3.541783571243286, "learning_rate": 6.435353445874182e-06, "logp/chosen": -292.0, "logp/rejected": -354.0, "loss": 0.39581298828125, "reward/accuracy": 0.8125, "reward/chosen": -1.234375, "reward/margin": 1.140625, "reward/rejected": -2.375, "step": 1359 }, { "approx. KL/chosen": 94.5, "approx. KL/rejected": 820.0, "epoch": 0.40791841631673664, "grad_norm": 2.9521751403808594, "learning_rate": 6.430839670235403e-06, "logp/chosen": -276.0, "logp/rejected": -392.0, "loss": 0.373931884765625, "reward/accuracy": 0.8125, "reward/chosen": -1.1875, "reward/margin": 2.203125, "reward/rejected": -3.390625, "step": 1360 }, { "approx. KL/chosen": 150.0, "approx. KL/rejected": 584.0, "epoch": 0.40821835632873427, "grad_norm": 4.815142631530762, "learning_rate": 6.426324624141015e-06, "logp/chosen": -374.0, "logp/rejected": -448.0, "loss": 0.45562744140625, "reward/accuracy": 0.75, "reward/chosen": -1.3046875, "reward/margin": 1.59375, "reward/rejected": -2.90625, "step": 1361 }, { "approx. KL/chosen": 93.5, "approx. KL/rejected": 376.0, "epoch": 0.40851829634073183, "grad_norm": 3.1799192428588867, "learning_rate": 6.421808311599969e-06, "logp/chosen": -304.0, "logp/rejected": -330.0, "loss": 0.3369140625, "reward/accuracy": 0.875, "reward/chosen": -0.77734375, "reward/margin": 1.5, "reward/rejected": -2.28125, "step": 1362 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 648.0, "epoch": 0.40881823635272946, "grad_norm": 3.844607353210449, "learning_rate": 6.417290736622341e-06, "logp/chosen": -318.0, "logp/rejected": -350.0, "loss": 0.4105224609375, "reward/accuracy": 0.75, "reward/chosen": -1.15625, "reward/margin": 1.8046875, "reward/rejected": -2.953125, "step": 1363 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 788.0, "epoch": 0.4091181763647271, "grad_norm": 5.824639320373535, "learning_rate": 6.412771903219326e-06, "logp/chosen": -332.0, "logp/rejected": -370.0, "loss": 0.524139404296875, "reward/accuracy": 0.75, "reward/chosen": -1.546875, "reward/margin": 1.984375, "reward/rejected": -3.53125, "step": 1364 }, { "approx. KL/chosen": 92.5, "approx. KL/rejected": 564.0, "epoch": 0.40941811637672465, "grad_norm": 3.3852908611297607, "learning_rate": 6.408251815403237e-06, "logp/chosen": -288.0, "logp/rejected": -268.0, "loss": 0.36266326904296875, "reward/accuracy": 0.75, "reward/chosen": -0.84375, "reward/margin": 1.828125, "reward/rejected": -2.671875, "step": 1365 }, { "approx. KL/chosen": 428.0, "approx. KL/rejected": 816.0, "epoch": 0.4097180563887223, "grad_norm": 4.552982330322266, "learning_rate": 6.403730477187503e-06, "logp/chosen": -296.0, "logp/rejected": -316.0, "loss": 0.470947265625, "reward/accuracy": 0.8125, "reward/chosen": -2.296875, "reward/margin": 1.375, "reward/rejected": -3.671875, "step": 1366 }, { "approx. KL/chosen": 117.0, "approx. KL/rejected": 956.0, "epoch": 0.41001799640071984, "grad_norm": 2.910486936569214, "learning_rate": 6.39920789258666e-06, "logp/chosen": -358.0, "logp/rejected": -320.0, "loss": 0.330596923828125, "reward/accuracy": 0.8125, "reward/chosen": -1.046875, "reward/margin": 2.53125, "reward/rejected": -3.578125, "step": 1367 }, { "approx. KL/chosen": 334.0, "approx. KL/rejected": 1392.0, "epoch": 0.41031793641271747, "grad_norm": 5.695405006408691, "learning_rate": 6.394684065616354e-06, "logp/chosen": -422.0, "logp/rejected": -560.0, "loss": 0.39609527587890625, "reward/accuracy": 0.6875, "reward/chosen": -2.203125, "reward/margin": 2.5625, "reward/rejected": -4.75, "step": 1368 }, { "approx. KL/chosen": 292.0, "approx. KL/rejected": 1184.0, "epoch": 0.41061787642471503, "grad_norm": 5.388107776641846, "learning_rate": 6.390159000293329e-06, "logp/chosen": -240.0, "logp/rejected": -332.0, "loss": 0.5254898071289062, "reward/accuracy": 0.75, "reward/chosen": -1.59375, "reward/margin": 2.453125, "reward/rejected": -4.0625, "step": 1369 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 940.0, "epoch": 0.41091781643671266, "grad_norm": 3.68940806388855, "learning_rate": 6.385632700635436e-06, "logp/chosen": -243.0, "logp/rejected": -243.0, "loss": 0.3570556640625, "reward/accuracy": 0.8125, "reward/chosen": -1.8984375, "reward/margin": 1.9375, "reward/rejected": -3.84375, "step": 1370 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 892.0, "epoch": 0.4112177564487103, "grad_norm": 4.577264308929443, "learning_rate": 6.381105170661614e-06, "logp/chosen": -247.0, "logp/rejected": -286.0, "loss": 0.4248046875, "reward/accuracy": 0.8125, "reward/chosen": -1.71875, "reward/margin": 1.703125, "reward/rejected": -3.421875, "step": 1371 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 1168.0, "epoch": 0.41151769646070785, "grad_norm": 3.735253095626831, "learning_rate": 6.3765764143918995e-06, "logp/chosen": -266.0, "logp/rejected": -356.0, "loss": 0.49689483642578125, "reward/accuracy": 0.6875, "reward/chosen": -2.1875, "reward/margin": 1.6875, "reward/rejected": -3.875, "step": 1372 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 620.0, "epoch": 0.4118176364727055, "grad_norm": 5.074016571044922, "learning_rate": 6.3720464358474185e-06, "logp/chosen": -328.0, "logp/rejected": -296.0, "loss": 0.61102294921875, "reward/accuracy": 0.625, "reward/chosen": -2.078125, "reward/margin": 1.015625, "reward/rejected": -3.078125, "step": 1373 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 1016.0, "epoch": 0.41211757648470304, "grad_norm": 4.640308380126953, "learning_rate": 6.367515239050377e-06, "logp/chosen": -368.0, "logp/rejected": -336.0, "loss": 0.5035362243652344, "reward/accuracy": 0.6875, "reward/chosen": -1.859375, "reward/margin": 2.09375, "reward/rejected": -3.9375, "step": 1374 }, { "approx. KL/chosen": 604.0, "approx. KL/rejected": 1176.0, "epoch": 0.41241751649670066, "grad_norm": 5.838356971740723, "learning_rate": 6.362982828024069e-06, "logp/chosen": -356.0, "logp/rejected": -296.0, "loss": 0.63232421875, "reward/accuracy": 0.625, "reward/chosen": -2.765625, "reward/margin": 1.171875, "reward/rejected": -3.9375, "step": 1375 }, { "approx. KL/chosen": 398.0, "approx. KL/rejected": 764.0, "epoch": 0.41271745650869823, "grad_norm": 4.558561325073242, "learning_rate": 6.358449206792862e-06, "logp/chosen": -434.0, "logp/rejected": -388.0, "loss": 0.59014892578125, "reward/accuracy": 0.6875, "reward/chosen": -2.15625, "reward/margin": 0.88671875, "reward/rejected": -3.046875, "step": 1376 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 536.0, "epoch": 0.41301739652069586, "grad_norm": 3.4474196434020996, "learning_rate": 6.3539143793822e-06, "logp/chosen": -392.0, "logp/rejected": -476.0, "loss": 0.39056396484375, "reward/accuracy": 0.75, "reward/chosen": -1.296875, "reward/margin": 1.5703125, "reward/rejected": -2.859375, "step": 1377 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 984.0, "epoch": 0.4133173365326935, "grad_norm": 5.172198295593262, "learning_rate": 6.349378349818598e-06, "logp/chosen": -350.0, "logp/rejected": -296.0, "loss": 0.5118408203125, "reward/accuracy": 0.8125, "reward/chosen": -2.03125, "reward/margin": 1.9140625, "reward/rejected": -3.953125, "step": 1378 }, { "approx. KL/chosen": 248.0, "approx. KL/rejected": 992.0, "epoch": 0.41361727654469105, "grad_norm": 3.5458219051361084, "learning_rate": 6.344841122129637e-06, "logp/chosen": -316.0, "logp/rejected": -338.0, "loss": 0.3860206604003906, "reward/accuracy": 0.75, "reward/chosen": -1.6640625, "reward/margin": 2.359375, "reward/rejected": -4.03125, "step": 1379 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 680.0, "epoch": 0.41391721655668867, "grad_norm": 5.639711380004883, "learning_rate": 6.340302700343963e-06, "logp/chosen": -286.0, "logp/rejected": -216.0, "loss": 0.58135986328125, "reward/accuracy": 0.75, "reward/chosen": -1.4453125, "reward/margin": 1.609375, "reward/rejected": -3.0625, "step": 1380 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 604.0, "epoch": 0.41421715656868624, "grad_norm": 3.202127456665039, "learning_rate": 6.335763088491284e-06, "logp/chosen": -338.0, "logp/rejected": -274.0, "loss": 0.33319091796875, "reward/accuracy": 0.875, "reward/chosen": -1.625, "reward/margin": 1.4453125, "reward/rejected": -3.078125, "step": 1381 }, { "approx. KL/chosen": 185.0, "approx. KL/rejected": 504.0, "epoch": 0.41451709658068386, "grad_norm": 4.365208148956299, "learning_rate": 6.331222290602361e-06, "logp/chosen": -328.0, "logp/rejected": -300.0, "loss": 0.558837890625, "reward/accuracy": 0.75, "reward/chosen": -1.2265625, "reward/margin": 1.640625, "reward/rejected": -2.859375, "step": 1382 }, { "approx. KL/chosen": 221.0, "approx. KL/rejected": 1328.0, "epoch": 0.4148170365926815, "grad_norm": 2.8689920902252197, "learning_rate": 6.326680310709011e-06, "logp/chosen": -366.0, "logp/rejected": -314.0, "loss": 0.16991424560546875, "reward/accuracy": 0.875, "reward/chosen": -1.5859375, "reward/margin": 3.3125, "reward/rejected": -4.875, "step": 1383 }, { "approx. KL/chosen": 398.0, "approx. KL/rejected": 572.0, "epoch": 0.41511697660467906, "grad_norm": 4.9957075119018555, "learning_rate": 6.322137152844099e-06, "logp/chosen": -418.0, "logp/rejected": -398.0, "loss": 0.78155517578125, "reward/accuracy": 0.75, "reward/chosen": -1.8125, "reward/margin": 1.03125, "reward/rejected": -2.84375, "step": 1384 }, { "approx. KL/chosen": 426.0, "approx. KL/rejected": 672.0, "epoch": 0.4154169166166767, "grad_norm": 6.194642066955566, "learning_rate": 6.317592821041536e-06, "logp/chosen": -368.0, "logp/rejected": -406.0, "loss": 0.651763916015625, "reward/accuracy": 0.6875, "reward/chosen": -2.546875, "reward/margin": 0.81640625, "reward/rejected": -3.359375, "step": 1385 }, { "approx. KL/chosen": 350.0, "approx. KL/rejected": 552.0, "epoch": 0.41571685662867425, "grad_norm": 5.604451656341553, "learning_rate": 6.313047319336276e-06, "logp/chosen": -434.0, "logp/rejected": -358.0, "loss": 0.633331298828125, "reward/accuracy": 0.625, "reward/chosen": -1.625, "reward/margin": 1.1015625, "reward/rejected": -2.734375, "step": 1386 }, { "approx. KL/chosen": 364.0, "approx. KL/rejected": 1016.0, "epoch": 0.41601679664067187, "grad_norm": 3.9829440116882324, "learning_rate": 6.308500651764311e-06, "logp/chosen": -464.0, "logp/rejected": -430.0, "loss": 0.40304994583129883, "reward/accuracy": 0.8125, "reward/chosen": -2.140625, "reward/margin": 1.75, "reward/rejected": -3.890625, "step": 1387 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 732.0, "epoch": 0.41631673665266944, "grad_norm": 3.620687961578369, "learning_rate": 6.303952822362671e-06, "logp/chosen": -380.0, "logp/rejected": -366.0, "loss": 0.38299560546875, "reward/accuracy": 0.8125, "reward/chosen": -1.6171875, "reward/margin": 1.796875, "reward/rejected": -3.40625, "step": 1388 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 860.0, "epoch": 0.41661667666466706, "grad_norm": 4.16035270690918, "learning_rate": 6.299403835169411e-06, "logp/chosen": -394.0, "logp/rejected": -374.0, "loss": 0.412109375, "reward/accuracy": 0.75, "reward/chosen": -1.9140625, "reward/margin": 1.5703125, "reward/rejected": -3.484375, "step": 1389 }, { "approx. KL/chosen": 334.0, "approx. KL/rejected": 1024.0, "epoch": 0.4169166166766647, "grad_norm": 3.5927693843841553, "learning_rate": 6.294853694223621e-06, "logp/chosen": -340.0, "logp/rejected": -370.0, "loss": 0.47406005859375, "reward/accuracy": 0.75, "reward/chosen": -2.328125, "reward/margin": 1.671875, "reward/rejected": -3.984375, "step": 1390 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 848.0, "epoch": 0.41721655668866225, "grad_norm": 5.322573661804199, "learning_rate": 6.290302403565414e-06, "logp/chosen": -208.0, "logp/rejected": -338.0, "loss": 0.48931884765625, "reward/accuracy": 0.75, "reward/chosen": -1.9453125, "reward/margin": 1.578125, "reward/rejected": -3.53125, "step": 1391 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 1232.0, "epoch": 0.4175164967006599, "grad_norm": 3.2183032035827637, "learning_rate": 6.285749967235919e-06, "logp/chosen": -408.0, "logp/rejected": -420.0, "loss": 0.26256561279296875, "reward/accuracy": 0.875, "reward/chosen": -1.703125, "reward/margin": 2.515625, "reward/rejected": -4.21875, "step": 1392 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 450.0, "epoch": 0.41781643671265745, "grad_norm": 4.160252094268799, "learning_rate": 6.2811963892772885e-06, "logp/chosen": -278.0, "logp/rejected": -380.0, "loss": 0.529998779296875, "reward/accuracy": 0.8125, "reward/chosen": -1.3125, "reward/margin": 1.25, "reward/rejected": -2.5625, "step": 1393 }, { "approx. KL/chosen": 127.5, "approx. KL/rejected": 644.0, "epoch": 0.41811637672465507, "grad_norm": 4.799356937408447, "learning_rate": 6.276641673732685e-06, "logp/chosen": -266.0, "logp/rejected": -284.0, "loss": 0.555816650390625, "reward/accuracy": 0.6875, "reward/chosen": -1.3125, "reward/margin": 1.421875, "reward/rejected": -2.734375, "step": 1394 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 796.0, "epoch": 0.4184163167366527, "grad_norm": 4.888111114501953, "learning_rate": 6.2720858246462835e-06, "logp/chosen": -338.0, "logp/rejected": -392.0, "loss": 0.3830108642578125, "reward/accuracy": 0.8125, "reward/chosen": -1.5234375, "reward/margin": 1.921875, "reward/rejected": -3.453125, "step": 1395 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 772.0, "epoch": 0.41871625674865026, "grad_norm": 2.9988794326782227, "learning_rate": 6.26752884606326e-06, "logp/chosen": -344.0, "logp/rejected": -352.0, "loss": 0.280059814453125, "reward/accuracy": 0.875, "reward/chosen": -1.484375, "reward/margin": 2.078125, "reward/rejected": -3.5625, "step": 1396 }, { "approx. KL/chosen": 252.0, "approx. KL/rejected": 1216.0, "epoch": 0.4190161967606479, "grad_norm": 4.982690811157227, "learning_rate": 6.262970742029803e-06, "logp/chosen": -268.0, "logp/rejected": -334.0, "loss": 0.47119140625, "reward/accuracy": 0.75, "reward/chosen": -1.6796875, "reward/margin": 2.3125, "reward/rejected": -3.984375, "step": 1397 }, { "approx. KL/chosen": 126.5, "approx. KL/rejected": 768.0, "epoch": 0.41931613677264545, "grad_norm": 2.6346230506896973, "learning_rate": 6.258411516593089e-06, "logp/chosen": -362.0, "logp/rejected": -360.0, "loss": 0.2467193603515625, "reward/accuracy": 0.875, "reward/chosen": -1.1484375, "reward/margin": 2.1875, "reward/rejected": -3.328125, "step": 1398 }, { "approx. KL/chosen": 390.0, "approx. KL/rejected": 868.0, "epoch": 0.4196160767846431, "grad_norm": 7.196276664733887, "learning_rate": 6.2538511738012984e-06, "logp/chosen": -356.0, "logp/rejected": -362.0, "loss": 0.7490386962890625, "reward/accuracy": 0.8125, "reward/chosen": -2.078125, "reward/margin": 1.484375, "reward/rejected": -3.5625, "step": 1399 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 536.0, "epoch": 0.41991601679664065, "grad_norm": 4.998885631561279, "learning_rate": 6.2492897177036035e-06, "logp/chosen": -418.0, "logp/rejected": -332.0, "loss": 0.6669921875, "reward/accuracy": 0.6875, "reward/chosen": -1.109375, "reward/margin": 1.3671875, "reward/rejected": -2.46875, "step": 1400 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 624.0, "epoch": 0.42021595680863827, "grad_norm": 3.228022813796997, "learning_rate": 6.24472715235016e-06, "logp/chosen": -260.0, "logp/rejected": -302.0, "loss": 0.4281005859375, "reward/accuracy": 0.8125, "reward/chosen": -1.953125, "reward/margin": 1.140625, "reward/rejected": -3.09375, "step": 1401 }, { "approx. KL/chosen": 223.0, "approx. KL/rejected": 544.0, "epoch": 0.4205158968206359, "grad_norm": 4.224113464355469, "learning_rate": 6.240163481792112e-06, "logp/chosen": -280.0, "logp/rejected": -244.0, "loss": 0.33349609375, "reward/accuracy": 0.875, "reward/chosen": -1.640625, "reward/margin": 1.4296875, "reward/rejected": -3.0625, "step": 1402 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 900.0, "epoch": 0.42081583683263346, "grad_norm": 2.9909493923187256, "learning_rate": 6.235598710081584e-06, "logp/chosen": -386.0, "logp/rejected": -394.0, "loss": 0.3408660888671875, "reward/accuracy": 0.9375, "reward/chosen": -1.4296875, "reward/margin": 2.015625, "reward/rejected": -3.453125, "step": 1403 }, { "approx. KL/chosen": 378.0, "approx. KL/rejected": 620.0, "epoch": 0.4211157768446311, "grad_norm": 5.880988121032715, "learning_rate": 6.231032841271679e-06, "logp/chosen": -252.0, "logp/rejected": -292.0, "loss": 0.6466064453125, "reward/accuracy": 0.6875, "reward/chosen": -1.984375, "reward/margin": 1.0859375, "reward/rejected": -3.078125, "step": 1404 }, { "approx. KL/chosen": 316.0, "approx. KL/rejected": 1040.0, "epoch": 0.42141571685662865, "grad_norm": 4.098703384399414, "learning_rate": 6.226465879416473e-06, "logp/chosen": -442.0, "logp/rejected": -360.0, "loss": 0.3671722412109375, "reward/accuracy": 0.875, "reward/chosen": -1.890625, "reward/margin": 2.0, "reward/rejected": -3.890625, "step": 1405 }, { "approx. KL/chosen": 412.0, "approx. KL/rejected": 996.0, "epoch": 0.4217156568686263, "grad_norm": 5.295916557312012, "learning_rate": 6.221897828571015e-06, "logp/chosen": -228.0, "logp/rejected": -318.0, "loss": 0.5044937133789062, "reward/accuracy": 0.75, "reward/chosen": -2.421875, "reward/margin": 1.671875, "reward/rejected": -4.09375, "step": 1406 }, { "approx. KL/chosen": 245.0, "approx. KL/rejected": 470.0, "epoch": 0.4220155968806239, "grad_norm": 3.7748517990112305, "learning_rate": 6.217328692791317e-06, "logp/chosen": -302.0, "logp/rejected": -252.0, "loss": 0.500732421875, "reward/accuracy": 0.6875, "reward/chosen": -1.890625, "reward/margin": 0.88671875, "reward/rejected": -2.78125, "step": 1407 }, { "approx. KL/chosen": 249.0, "approx. KL/rejected": 504.0, "epoch": 0.42231553689262147, "grad_norm": 4.256992340087891, "learning_rate": 6.212758476134355e-06, "logp/chosen": -242.0, "logp/rejected": -276.0, "loss": 0.6148681640625, "reward/accuracy": 0.6875, "reward/chosen": -1.8671875, "reward/margin": 0.8359375, "reward/rejected": -2.703125, "step": 1408 }, { "approx. KL/chosen": 247.0, "approx. KL/rejected": 648.0, "epoch": 0.4226154769046191, "grad_norm": 4.635004043579102, "learning_rate": 6.2081871826580705e-06, "logp/chosen": -382.0, "logp/rejected": -324.0, "loss": 0.60736083984375, "reward/accuracy": 0.6875, "reward/chosen": -1.78125, "reward/margin": 1.21875, "reward/rejected": -3.0, "step": 1409 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 600.0, "epoch": 0.42291541691661666, "grad_norm": 4.1995344161987305, "learning_rate": 6.203614816421353e-06, "logp/chosen": -396.0, "logp/rejected": -386.0, "loss": 0.5357666015625, "reward/accuracy": 0.6875, "reward/chosen": -1.734375, "reward/margin": 1.1796875, "reward/rejected": -2.90625, "step": 1410 }, { "approx. KL/chosen": 241.0, "approx. KL/rejected": 536.0, "epoch": 0.4232153569286143, "grad_norm": 3.74129319190979, "learning_rate": 6.199041381484051e-06, "logp/chosen": -336.0, "logp/rejected": -454.0, "loss": 0.5009765625, "reward/accuracy": 0.75, "reward/chosen": -2.0, "reward/margin": 1.0078125, "reward/rejected": -3.015625, "step": 1411 }, { "approx. KL/chosen": 304.0, "approx. KL/rejected": 688.0, "epoch": 0.42351529694061185, "grad_norm": 4.0626139640808105, "learning_rate": 6.1944668819069565e-06, "logp/chosen": -492.0, "logp/rejected": -316.0, "loss": 0.522613525390625, "reward/accuracy": 0.8125, "reward/chosen": -1.578125, "reward/margin": 1.71875, "reward/rejected": -3.296875, "step": 1412 }, { "approx. KL/chosen": 255.0, "approx. KL/rejected": 1120.0, "epoch": 0.4238152369526095, "grad_norm": 4.890384197235107, "learning_rate": 6.189891321751811e-06, "logp/chosen": -342.0, "logp/rejected": -326.0, "loss": 0.462860107421875, "reward/accuracy": 0.6875, "reward/chosen": -1.7578125, "reward/margin": 2.015625, "reward/rejected": -3.765625, "step": 1413 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 784.0, "epoch": 0.4241151769646071, "grad_norm": 3.816864490509033, "learning_rate": 6.185314705081296e-06, "logp/chosen": -328.0, "logp/rejected": -350.0, "loss": 0.5560302734375, "reward/accuracy": 0.6875, "reward/chosen": -1.65625, "reward/margin": 1.375, "reward/rejected": -3.03125, "step": 1414 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 432.0, "epoch": 0.42441511697660467, "grad_norm": 3.499535322189331, "learning_rate": 6.18073703595903e-06, "logp/chosen": -390.0, "logp/rejected": -328.0, "loss": 0.458984375, "reward/accuracy": 0.8125, "reward/chosen": -1.1953125, "reward/margin": 1.40625, "reward/rejected": -2.59375, "step": 1415 }, { "approx. KL/chosen": 544.0, "approx. KL/rejected": 1256.0, "epoch": 0.4247150569886023, "grad_norm": 6.160314083099365, "learning_rate": 6.1761583184495696e-06, "logp/chosen": -404.0, "logp/rejected": -408.0, "loss": 0.5300521850585938, "reward/accuracy": 0.8125, "reward/chosen": -2.578125, "reward/margin": 1.7109375, "reward/rejected": -4.28125, "step": 1416 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 732.0, "epoch": 0.42501499700059986, "grad_norm": 3.278592109680176, "learning_rate": 6.171578556618398e-06, "logp/chosen": -227.0, "logp/rejected": -268.0, "loss": 0.350830078125, "reward/accuracy": 0.8125, "reward/chosen": -1.5078125, "reward/margin": 2.03125, "reward/rejected": -3.53125, "step": 1417 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 468.0, "epoch": 0.4253149370125975, "grad_norm": 3.246107339859009, "learning_rate": 6.166997754531927e-06, "logp/chosen": -370.0, "logp/rejected": -340.0, "loss": 0.36944580078125, "reward/accuracy": 0.75, "reward/chosen": -1.578125, "reward/margin": 1.125, "reward/rejected": -2.703125, "step": 1418 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 752.0, "epoch": 0.4256148770245951, "grad_norm": 2.7047886848449707, "learning_rate": 6.162415916257495e-06, "logp/chosen": -342.0, "logp/rejected": -544.0, "loss": 0.25830078125, "reward/accuracy": 0.8125, "reward/chosen": -1.3359375, "reward/margin": 2.015625, "reward/rejected": -3.34375, "step": 1419 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 764.0, "epoch": 0.4259148170365927, "grad_norm": 3.595883369445801, "learning_rate": 6.157833045863355e-06, "logp/chosen": -364.0, "logp/rejected": -330.0, "loss": 0.394439697265625, "reward/accuracy": 0.75, "reward/chosen": -1.6015625, "reward/margin": 1.8515625, "reward/rejected": -3.453125, "step": 1420 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 940.0, "epoch": 0.4262147570485903, "grad_norm": 2.848191261291504, "learning_rate": 6.153249147418683e-06, "logp/chosen": -444.0, "logp/rejected": -368.0, "loss": 0.296234130859375, "reward/accuracy": 0.8125, "reward/chosen": -1.6171875, "reward/margin": 2.09375, "reward/rejected": -3.71875, "step": 1421 }, { "approx. KL/chosen": 123.5, "approx. KL/rejected": 644.0, "epoch": 0.42651469706058787, "grad_norm": 4.46319055557251, "learning_rate": 6.148664224993561e-06, "logp/chosen": -358.0, "logp/rejected": -390.0, "loss": 0.43743896484375, "reward/accuracy": 0.75, "reward/chosen": -1.1015625, "reward/margin": 1.6796875, "reward/rejected": -2.78125, "step": 1422 }, { "approx. KL/chosen": 318.0, "approx. KL/rejected": 1040.0, "epoch": 0.4268146370725855, "grad_norm": 3.4947493076324463, "learning_rate": 6.144078282658987e-06, "logp/chosen": -528.0, "logp/rejected": -268.0, "loss": 0.3462371826171875, "reward/accuracy": 0.875, "reward/chosen": -1.671875, "reward/margin": 2.296875, "reward/rejected": -3.96875, "step": 1423 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 620.0, "epoch": 0.42711457708458306, "grad_norm": 3.857025384902954, "learning_rate": 6.139491324486857e-06, "logp/chosen": -237.0, "logp/rejected": -390.0, "loss": 0.4393310546875, "reward/accuracy": 0.75, "reward/chosen": -1.734375, "reward/margin": 1.328125, "reward/rejected": -3.0625, "step": 1424 }, { "approx. KL/chosen": 90.5, "approx. KL/rejected": 732.0, "epoch": 0.4274145170965807, "grad_norm": 3.2440123558044434, "learning_rate": 6.134903354549977e-06, "logp/chosen": -284.0, "logp/rejected": -330.0, "loss": 0.355743408203125, "reward/accuracy": 0.875, "reward/chosen": -0.85546875, "reward/margin": 2.59375, "reward/rejected": -3.453125, "step": 1425 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 428.0, "epoch": 0.4277144571085783, "grad_norm": 3.515322208404541, "learning_rate": 6.1303143769220445e-06, "logp/chosen": -328.0, "logp/rejected": -338.0, "loss": 0.323883056640625, "reward/accuracy": 0.8125, "reward/chosen": -1.171875, "reward/margin": 1.4453125, "reward/rejected": -2.609375, "step": 1426 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 812.0, "epoch": 0.4280143971205759, "grad_norm": 3.8540053367614746, "learning_rate": 6.1257243956776545e-06, "logp/chosen": -362.0, "logp/rejected": -354.0, "loss": 0.322021484375, "reward/accuracy": 0.875, "reward/chosen": -2.171875, "reward/margin": 1.484375, "reward/rejected": -3.65625, "step": 1427 }, { "approx. KL/chosen": 209.0, "approx. KL/rejected": 840.0, "epoch": 0.4283143371325735, "grad_norm": 3.8982748985290527, "learning_rate": 6.121133414892295e-06, "logp/chosen": -430.0, "logp/rejected": -474.0, "loss": 0.321502685546875, "reward/accuracy": 0.875, "reward/chosen": -1.578125, "reward/margin": 1.9140625, "reward/rejected": -3.484375, "step": 1428 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 676.0, "epoch": 0.42861427714457107, "grad_norm": 4.249454021453857, "learning_rate": 6.116541438642339e-06, "logp/chosen": -378.0, "logp/rejected": -266.0, "loss": 0.31939697265625, "reward/accuracy": 0.875, "reward/chosen": -1.5390625, "reward/margin": 1.640625, "reward/rejected": -3.1875, "step": 1429 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 1648.0, "epoch": 0.4289142171565687, "grad_norm": 2.6773366928100586, "learning_rate": 6.111948471005046e-06, "logp/chosen": -338.0, "logp/rejected": -364.0, "loss": 0.2572288513183594, "reward/accuracy": 0.875, "reward/chosen": -1.328125, "reward/margin": 3.328125, "reward/rejected": -4.65625, "step": 1430 }, { "approx. KL/chosen": 340.0, "approx. KL/rejected": 604.0, "epoch": 0.4292141571685663, "grad_norm": 4.683799743652344, "learning_rate": 6.10735451605855e-06, "logp/chosen": -298.0, "logp/rejected": -338.0, "loss": 0.486328125, "reward/accuracy": 0.75, "reward/chosen": -2.0, "reward/margin": 1.046875, "reward/rejected": -3.046875, "step": 1431 }, { "approx. KL/chosen": 372.0, "approx. KL/rejected": 972.0, "epoch": 0.4295140971805639, "grad_norm": 3.3389246463775635, "learning_rate": 6.1027595778818685e-06, "logp/chosen": -396.0, "logp/rejected": -414.0, "loss": 0.2896728515625, "reward/accuracy": 0.8125, "reward/chosen": -2.0, "reward/margin": 1.9921875, "reward/rejected": -3.984375, "step": 1432 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 1032.0, "epoch": 0.4298140371925615, "grad_norm": 3.442439079284668, "learning_rate": 6.098163660554889e-06, "logp/chosen": -229.0, "logp/rejected": -326.0, "loss": 0.398162841796875, "reward/accuracy": 0.875, "reward/chosen": -2.046875, "reward/margin": 2.015625, "reward/rejected": -4.0625, "step": 1433 }, { "approx. KL/chosen": 314.0, "approx. KL/rejected": 544.0, "epoch": 0.4301139772045591, "grad_norm": 4.9104204177856445, "learning_rate": 6.0935667681583665e-06, "logp/chosen": -536.0, "logp/rejected": -400.0, "loss": 0.7177734375, "reward/accuracy": 0.6875, "reward/chosen": -1.9921875, "reward/margin": 0.890625, "reward/rejected": -2.875, "step": 1434 }, { "approx. KL/chosen": 292.0, "approx. KL/rejected": 872.0, "epoch": 0.4304139172165567, "grad_norm": 3.2513158321380615, "learning_rate": 6.088968904773925e-06, "logp/chosen": -253.0, "logp/rejected": -316.0, "loss": 0.347808837890625, "reward/accuracy": 0.875, "reward/chosen": -2.15625, "reward/margin": 1.6328125, "reward/rejected": -3.796875, "step": 1435 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 1016.0, "epoch": 0.43071385722855426, "grad_norm": 4.9494948387146, "learning_rate": 6.084370074484051e-06, "logp/chosen": -360.0, "logp/rejected": -354.0, "loss": 0.487030029296875, "reward/accuracy": 0.75, "reward/chosen": -1.65625, "reward/margin": 1.9765625, "reward/rejected": -3.625, "step": 1436 }, { "approx. KL/chosen": 332.0, "approx. KL/rejected": 912.0, "epoch": 0.4310137972405519, "grad_norm": 6.401081085205078, "learning_rate": 6.0797702813720845e-06, "logp/chosen": -316.0, "logp/rejected": -362.0, "loss": 0.7630386352539062, "reward/accuracy": 0.6875, "reward/chosen": -2.03125, "reward/margin": 1.578125, "reward/rejected": -3.609375, "step": 1437 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 1040.0, "epoch": 0.4313137372525495, "grad_norm": 2.234605073928833, "learning_rate": 6.075169529522226e-06, "logp/chosen": -302.0, "logp/rejected": -322.0, "loss": 0.28955078125, "reward/accuracy": 0.875, "reward/chosen": -2.015625, "reward/margin": 2.109375, "reward/rejected": -4.125, "step": 1438 }, { "approx. KL/chosen": 454.0, "approx. KL/rejected": 1120.0, "epoch": 0.4316136772645471, "grad_norm": 3.653881072998047, "learning_rate": 6.070567823019524e-06, "logp/chosen": -314.0, "logp/rejected": -342.0, "loss": 0.5028076171875, "reward/accuracy": 0.75, "reward/chosen": -2.59375, "reward/margin": 1.6953125, "reward/rejected": -4.28125, "step": 1439 }, { "approx. KL/chosen": 492.0, "approx. KL/rejected": 832.0, "epoch": 0.4319136172765447, "grad_norm": 5.18904447555542, "learning_rate": 6.065965165949879e-06, "logp/chosen": -253.0, "logp/rejected": -324.0, "loss": 0.61981201171875, "reward/accuracy": 0.625, "reward/chosen": -2.640625, "reward/margin": 0.8671875, "reward/rejected": -3.5, "step": 1440 }, { "approx. KL/chosen": 352.0, "approx. KL/rejected": 1456.0, "epoch": 0.43221355728854227, "grad_norm": 3.4689223766326904, "learning_rate": 6.061361562400027e-06, "logp/chosen": -330.0, "logp/rejected": -354.0, "loss": 0.14209747314453125, "reward/accuracy": 0.9375, "reward/chosen": -2.265625, "reward/margin": 2.9375, "reward/rejected": -5.1875, "step": 1441 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 1248.0, "epoch": 0.4325134973005399, "grad_norm": 3.9662997722625732, "learning_rate": 6.056757016457553e-06, "logp/chosen": -330.0, "logp/rejected": -358.0, "loss": 0.34954833984375, "reward/accuracy": 0.875, "reward/chosen": -1.3984375, "reward/margin": 2.96875, "reward/rejected": -4.375, "step": 1442 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 836.0, "epoch": 0.4328134373125375, "grad_norm": 4.0960588455200195, "learning_rate": 6.052151532210875e-06, "logp/chosen": -416.0, "logp/rejected": -408.0, "loss": 0.475341796875, "reward/accuracy": 0.625, "reward/chosen": -2.03125, "reward/margin": 1.4609375, "reward/rejected": -3.484375, "step": 1443 }, { "approx. KL/chosen": 386.0, "approx. KL/rejected": 804.0, "epoch": 0.4331133773245351, "grad_norm": 5.565241813659668, "learning_rate": 6.047545113749243e-06, "logp/chosen": -239.0, "logp/rejected": -278.0, "loss": 0.449951171875, "reward/accuracy": 0.75, "reward/chosen": -2.234375, "reward/margin": 1.546875, "reward/rejected": -3.78125, "step": 1444 }, { "approx. KL/chosen": 330.0, "approx. KL/rejected": 552.0, "epoch": 0.4334133173365327, "grad_norm": 4.289402484893799, "learning_rate": 6.042937765162741e-06, "logp/chosen": -314.0, "logp/rejected": -286.0, "loss": 0.636962890625, "reward/accuracy": 0.75, "reward/chosen": -1.859375, "reward/margin": 1.1484375, "reward/rejected": -3.0, "step": 1445 }, { "approx. KL/chosen": 434.0, "approx. KL/rejected": 1704.0, "epoch": 0.4337132573485303, "grad_norm": 2.39575457572937, "learning_rate": 6.038329490542272e-06, "logp/chosen": -322.0, "logp/rejected": -344.0, "loss": 0.20977783203125, "reward/accuracy": 0.9375, "reward/chosen": -2.65625, "reward/margin": 2.5625, "reward/rejected": -5.21875, "step": 1446 }, { "approx. KL/chosen": 620.0, "approx. KL/rejected": 1296.0, "epoch": 0.4340131973605279, "grad_norm": 5.557381629943848, "learning_rate": 6.033720293979569e-06, "logp/chosen": -352.0, "logp/rejected": -364.0, "loss": 0.4302978515625, "reward/accuracy": 0.75, "reward/chosen": -2.984375, "reward/margin": 1.7421875, "reward/rejected": -4.71875, "step": 1447 }, { "approx. KL/chosen": 376.0, "approx. KL/rejected": 1200.0, "epoch": 0.43431313737252547, "grad_norm": 4.075446605682373, "learning_rate": 6.029110179567177e-06, "logp/chosen": -282.0, "logp/rejected": -356.0, "loss": 0.3333892822265625, "reward/accuracy": 0.75, "reward/chosen": -2.265625, "reward/margin": 2.203125, "reward/rejected": -4.46875, "step": 1448 }, { "approx. KL/chosen": 213.0, "approx. KL/rejected": 648.0, "epoch": 0.4346130773845231, "grad_norm": 4.928713798522949, "learning_rate": 6.02449915139846e-06, "logp/chosen": -340.0, "logp/rejected": -364.0, "loss": 0.6531982421875, "reward/accuracy": 0.625, "reward/chosen": -1.8203125, "reward/margin": 1.171875, "reward/rejected": -3.0, "step": 1449 }, { "approx. KL/chosen": 430.0, "approx. KL/rejected": 1256.0, "epoch": 0.4349130173965207, "grad_norm": 4.877151966094971, "learning_rate": 6.01988721356759e-06, "logp/chosen": -328.0, "logp/rejected": -326.0, "loss": 0.3737335205078125, "reward/accuracy": 0.8125, "reward/chosen": -2.234375, "reward/margin": 2.296875, "reward/rejected": -4.53125, "step": 1450 }, { "approx. KL/chosen": 222.0, "approx. KL/rejected": 784.0, "epoch": 0.4352129574085183, "grad_norm": 4.402005195617676, "learning_rate": 6.015274370169551e-06, "logp/chosen": -328.0, "logp/rejected": -352.0, "loss": 0.32415771484375, "reward/accuracy": 0.8125, "reward/chosen": -1.5859375, "reward/margin": 1.9921875, "reward/rejected": -3.578125, "step": 1451 }, { "approx. KL/chosen": 396.0, "approx. KL/rejected": 1488.0, "epoch": 0.4355128974205159, "grad_norm": 3.27535343170166, "learning_rate": 6.010660625300129e-06, "logp/chosen": -444.0, "logp/rejected": -386.0, "loss": 0.2941741943359375, "reward/accuracy": 0.8125, "reward/chosen": -2.40625, "reward/margin": 2.6875, "reward/rejected": -5.09375, "step": 1452 }, { "approx. KL/chosen": 240.0, "approx. KL/rejected": 768.0, "epoch": 0.4358128374325135, "grad_norm": 3.662647008895874, "learning_rate": 6.006045983055908e-06, "logp/chosen": -280.0, "logp/rejected": -302.0, "loss": 0.45462799072265625, "reward/accuracy": 0.625, "reward/chosen": -1.890625, "reward/margin": 1.4140625, "reward/rejected": -3.3125, "step": 1453 }, { "approx. KL/chosen": 664.0, "approx. KL/rejected": 980.0, "epoch": 0.4361127774445111, "grad_norm": 7.564298629760742, "learning_rate": 6.001430447534274e-06, "logp/chosen": -276.0, "logp/rejected": -298.0, "loss": 0.96502685546875, "reward/accuracy": 0.5625, "reward/chosen": -3.125, "reward/margin": 0.921875, "reward/rejected": -4.03125, "step": 1454 }, { "approx. KL/chosen": 253.0, "approx. KL/rejected": 724.0, "epoch": 0.4364127174565087, "grad_norm": 4.221399307250977, "learning_rate": 5.996814022833399e-06, "logp/chosen": -376.0, "logp/rejected": -390.0, "loss": 0.5030517578125, "reward/accuracy": 0.625, "reward/chosen": -2.0625, "reward/margin": 1.234375, "reward/rejected": -3.296875, "step": 1455 }, { "approx. KL/chosen": 496.0, "approx. KL/rejected": 1152.0, "epoch": 0.4367126574685063, "grad_norm": 8.211048126220703, "learning_rate": 5.992196713052253e-06, "logp/chosen": -322.0, "logp/rejected": -380.0, "loss": 0.85333251953125, "reward/accuracy": 0.5625, "reward/chosen": -2.5625, "reward/margin": 1.4140625, "reward/rejected": -3.96875, "step": 1456 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 1032.0, "epoch": 0.4370125974805039, "grad_norm": 2.809603452682495, "learning_rate": 5.987578522290585e-06, "logp/chosen": -382.0, "logp/rejected": -430.0, "loss": 0.216796875, "reward/accuracy": 0.9375, "reward/chosen": -2.015625, "reward/margin": 2.140625, "reward/rejected": -4.15625, "step": 1457 }, { "approx. KL/chosen": 556.0, "approx. KL/rejected": 584.0, "epoch": 0.4373125374925015, "grad_norm": 7.071138381958008, "learning_rate": 5.982959454648929e-06, "logp/chosen": -422.0, "logp/rejected": -332.0, "loss": 0.77294921875, "reward/accuracy": 0.625, "reward/chosen": -2.65625, "reward/margin": 0.474609375, "reward/rejected": -3.125, "step": 1458 }, { "approx. KL/chosen": 366.0, "approx. KL/rejected": 1048.0, "epoch": 0.4376124775044991, "grad_norm": 6.462491989135742, "learning_rate": 5.978339514228598e-06, "logp/chosen": -290.0, "logp/rejected": -314.0, "loss": 0.58856201171875, "reward/accuracy": 0.75, "reward/chosen": -2.375, "reward/margin": 1.609375, "reward/rejected": -3.984375, "step": 1459 }, { "approx. KL/chosen": 290.0, "approx. KL/rejected": 1264.0, "epoch": 0.4379124175164967, "grad_norm": 3.703524589538574, "learning_rate": 5.973718705131678e-06, "logp/chosen": -452.0, "logp/rejected": -380.0, "loss": 0.241455078125, "reward/accuracy": 0.875, "reward/chosen": -2.03125, "reward/margin": 2.5625, "reward/rejected": -4.59375, "step": 1460 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 1152.0, "epoch": 0.4382123575284943, "grad_norm": 2.6271746158599854, "learning_rate": 5.96909703146103e-06, "logp/chosen": -232.0, "logp/rejected": -340.0, "loss": 0.2823944091796875, "reward/accuracy": 0.8125, "reward/chosen": -1.9140625, "reward/margin": 2.453125, "reward/rejected": -4.375, "step": 1461 }, { "approx. KL/chosen": 364.0, "approx. KL/rejected": 1104.0, "epoch": 0.4385122975404919, "grad_norm": 3.5543487071990967, "learning_rate": 5.964474497320277e-06, "logp/chosen": -380.0, "logp/rejected": -324.0, "loss": 0.3871612548828125, "reward/accuracy": 0.8125, "reward/chosen": -2.15625, "reward/margin": 2.328125, "reward/rejected": -4.46875, "step": 1462 }, { "approx. KL/chosen": 251.0, "approx. KL/rejected": 612.0, "epoch": 0.4388122375524895, "grad_norm": 4.761712074279785, "learning_rate": 5.959851106813812e-06, "logp/chosen": -490.0, "logp/rejected": -406.0, "loss": 0.5869140625, "reward/accuracy": 0.6875, "reward/chosen": -1.8828125, "reward/margin": 1.2265625, "reward/rejected": -3.109375, "step": 1463 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 836.0, "epoch": 0.4391121775644871, "grad_norm": 3.3652257919311523, "learning_rate": 5.955226864046784e-06, "logp/chosen": -306.0, "logp/rejected": -390.0, "loss": 0.364715576171875, "reward/accuracy": 0.8125, "reward/chosen": -1.90625, "reward/margin": 1.7890625, "reward/rejected": -3.6875, "step": 1464 }, { "approx. KL/chosen": 314.0, "approx. KL/rejected": 1336.0, "epoch": 0.4394121175764847, "grad_norm": 3.8817081451416016, "learning_rate": 5.950601773125102e-06, "logp/chosen": -380.0, "logp/rejected": -352.0, "loss": 0.344146728515625, "reward/accuracy": 0.6875, "reward/chosen": -2.140625, "reward/margin": 2.4375, "reward/rejected": -4.59375, "step": 1465 }, { "approx. KL/chosen": 384.0, "approx. KL/rejected": 1088.0, "epoch": 0.4397120575884823, "grad_norm": 4.753509521484375, "learning_rate": 5.945975838155426e-06, "logp/chosen": -294.0, "logp/rejected": -374.0, "loss": 0.4500579833984375, "reward/accuracy": 0.8125, "reward/chosen": -2.40625, "reward/margin": 1.8984375, "reward/rejected": -4.3125, "step": 1466 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 1048.0, "epoch": 0.44001199760047993, "grad_norm": 4.0422043800354, "learning_rate": 5.9413490632451635e-06, "logp/chosen": -424.0, "logp/rejected": -410.0, "loss": 0.289398193359375, "reward/accuracy": 0.8125, "reward/chosen": -2.1875, "reward/margin": 2.15625, "reward/rejected": -4.34375, "step": 1467 }, { "approx. KL/chosen": 388.0, "approx. KL/rejected": 692.0, "epoch": 0.4403119376124775, "grad_norm": 5.032476425170898, "learning_rate": 5.936721452502473e-06, "logp/chosen": -342.0, "logp/rejected": -346.0, "loss": 0.753662109375, "reward/accuracy": 0.5, "reward/chosen": -2.515625, "reward/margin": 0.76171875, "reward/rejected": -3.28125, "step": 1468 }, { "approx. KL/chosen": 206.0, "approx. KL/rejected": 1224.0, "epoch": 0.4406118776244751, "grad_norm": 2.780186176300049, "learning_rate": 5.9320930100362515e-06, "logp/chosen": -348.0, "logp/rejected": -340.0, "loss": 0.2167339324951172, "reward/accuracy": 0.875, "reward/chosen": -1.609375, "reward/margin": 2.765625, "reward/rejected": -4.375, "step": 1469 }, { "approx. KL/chosen": 322.0, "approx. KL/rejected": 748.0, "epoch": 0.4409118176364727, "grad_norm": 4.089969635009766, "learning_rate": 5.927463739956135e-06, "logp/chosen": -296.0, "logp/rejected": -372.0, "loss": 0.523681640625, "reward/accuracy": 0.5625, "reward/chosen": -2.046875, "reward/margin": 1.1875, "reward/rejected": -3.234375, "step": 1470 }, { "approx. KL/chosen": 127.0, "approx. KL/rejected": 1120.0, "epoch": 0.4412117576484703, "grad_norm": 2.1578078269958496, "learning_rate": 5.922833646372495e-06, "logp/chosen": -270.0, "logp/rejected": -292.0, "loss": 0.17987442016601562, "reward/accuracy": 0.9375, "reward/chosen": -1.21875, "reward/margin": 3.109375, "reward/rejected": -4.3125, "step": 1471 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 776.0, "epoch": 0.4415116976604679, "grad_norm": 3.9583635330200195, "learning_rate": 5.918202733396434e-06, "logp/chosen": -444.0, "logp/rejected": -496.0, "loss": 0.5457763671875, "reward/accuracy": 0.75, "reward/chosen": -1.671875, "reward/margin": 1.4453125, "reward/rejected": -3.109375, "step": 1472 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 972.0, "epoch": 0.4418116376724655, "grad_norm": 4.29280948638916, "learning_rate": 5.9135710051397835e-06, "logp/chosen": -298.0, "logp/rejected": -322.0, "loss": 0.4719886779785156, "reward/accuracy": 0.625, "reward/chosen": -1.828125, "reward/margin": 2.03125, "reward/rejected": -3.859375, "step": 1473 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 1048.0, "epoch": 0.44211157768446313, "grad_norm": 4.049343109130859, "learning_rate": 5.908938465715093e-06, "logp/chosen": -384.0, "logp/rejected": -388.0, "loss": 0.62640380859375, "reward/accuracy": 0.8125, "reward/chosen": -2.21875, "reward/margin": 1.7734375, "reward/rejected": -4.0, "step": 1474 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 932.0, "epoch": 0.4424115176964607, "grad_norm": 4.595029830932617, "learning_rate": 5.9043051192356395e-06, "logp/chosen": -328.0, "logp/rejected": -362.0, "loss": 0.7166748046875, "reward/accuracy": 0.625, "reward/chosen": -1.96875, "reward/margin": 1.390625, "reward/rejected": -3.359375, "step": 1475 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 860.0, "epoch": 0.4427114577084583, "grad_norm": 4.896424770355225, "learning_rate": 5.8996709698154145e-06, "logp/chosen": -340.0, "logp/rejected": -342.0, "loss": 0.5359954833984375, "reward/accuracy": 0.625, "reward/chosen": -2.25, "reward/margin": 1.40625, "reward/rejected": -3.65625, "step": 1476 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 904.0, "epoch": 0.4430113977204559, "grad_norm": 3.254565477371216, "learning_rate": 5.8950360215691185e-06, "logp/chosen": -316.0, "logp/rejected": -316.0, "loss": 0.33612060546875, "reward/accuracy": 0.875, "reward/chosen": -1.8671875, "reward/margin": 1.859375, "reward/rejected": -3.71875, "step": 1477 }, { "approx. KL/chosen": 252.0, "approx. KL/rejected": 520.0, "epoch": 0.4433113377324535, "grad_norm": 4.453820705413818, "learning_rate": 5.8904002786121676e-06, "logp/chosen": -328.0, "logp/rejected": -440.0, "loss": 0.5474853515625, "reward/accuracy": 0.6875, "reward/chosen": -2.109375, "reward/margin": 0.76171875, "reward/rejected": -2.875, "step": 1478 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 704.0, "epoch": 0.4436112777444511, "grad_norm": 3.758568048477173, "learning_rate": 5.885763745060678e-06, "logp/chosen": -258.0, "logp/rejected": -338.0, "loss": 0.455810546875, "reward/accuracy": 0.6875, "reward/chosen": -1.609375, "reward/margin": 1.578125, "reward/rejected": -3.1875, "step": 1479 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 764.0, "epoch": 0.4439112177564487, "grad_norm": 5.987349033355713, "learning_rate": 5.8811264250314714e-06, "logp/chosen": -294.0, "logp/rejected": -380.0, "loss": 0.61572265625, "reward/accuracy": 0.5, "reward/chosen": -1.953125, "reward/margin": 1.3046875, "reward/rejected": -3.265625, "step": 1480 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 772.0, "epoch": 0.44421115776844633, "grad_norm": 3.8177268505096436, "learning_rate": 5.876488322642066e-06, "logp/chosen": -412.0, "logp/rejected": -356.0, "loss": 0.41785430908203125, "reward/accuracy": 0.75, "reward/chosen": -1.65625, "reward/margin": 1.703125, "reward/rejected": -3.359375, "step": 1481 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 482.0, "epoch": 0.4445110977804439, "grad_norm": 4.042351722717285, "learning_rate": 5.871849442010675e-06, "logp/chosen": -358.0, "logp/rejected": -356.0, "loss": 0.5079345703125, "reward/accuracy": 0.8125, "reward/chosen": -1.3984375, "reward/margin": 1.2734375, "reward/rejected": -2.671875, "step": 1482 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 1004.0, "epoch": 0.4448110377924415, "grad_norm": 2.1447103023529053, "learning_rate": 5.867209787256205e-06, "logp/chosen": -258.0, "logp/rejected": -348.0, "loss": 0.181915283203125, "reward/accuracy": 1.0, "reward/chosen": -1.6953125, "reward/margin": 2.515625, "reward/rejected": -4.21875, "step": 1483 }, { "approx. KL/chosen": 316.0, "approx. KL/rejected": 840.0, "epoch": 0.4451109778044391, "grad_norm": 5.486515522003174, "learning_rate": 5.862569362498245e-06, "logp/chosen": -225.0, "logp/rejected": -308.0, "loss": 0.8184814453125, "reward/accuracy": 0.75, "reward/chosen": -2.203125, "reward/margin": 0.9453125, "reward/rejected": -3.140625, "step": 1484 }, { "approx. KL/chosen": 133.0, "approx. KL/rejected": 580.0, "epoch": 0.4454109178164367, "grad_norm": 4.432990074157715, "learning_rate": 5.857928171857074e-06, "logp/chosen": -416.0, "logp/rejected": -392.0, "loss": 0.48592376708984375, "reward/accuracy": 0.8125, "reward/chosen": -1.1640625, "reward/margin": 1.5, "reward/rejected": -2.671875, "step": 1485 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 1192.0, "epoch": 0.44571085782843434, "grad_norm": 3.9186315536499023, "learning_rate": 5.8532862194536425e-06, "logp/chosen": -268.0, "logp/rejected": -504.0, "loss": 0.46875, "reward/accuracy": 0.6875, "reward/chosen": -1.9296875, "reward/margin": 2.40625, "reward/rejected": -4.34375, "step": 1486 }, { "approx. KL/chosen": 394.0, "approx. KL/rejected": 728.0, "epoch": 0.4460107978404319, "grad_norm": 6.36501407623291, "learning_rate": 5.8486435094095866e-06, "logp/chosen": -278.0, "logp/rejected": -296.0, "loss": 0.5179443359375, "reward/accuracy": 0.625, "reward/chosen": -2.421875, "reward/margin": 0.87109375, "reward/rejected": -3.296875, "step": 1487 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 792.0, "epoch": 0.44631073785242953, "grad_norm": 4.170838356018066, "learning_rate": 5.844000045847209e-06, "logp/chosen": -240.0, "logp/rejected": -272.0, "loss": 0.52764892578125, "reward/accuracy": 0.75, "reward/chosen": -1.65625, "reward/margin": 1.5625, "reward/rejected": -3.21875, "step": 1488 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 844.0, "epoch": 0.4466106778644271, "grad_norm": 2.7420337200164795, "learning_rate": 5.839355832889484e-06, "logp/chosen": -330.0, "logp/rejected": -380.0, "loss": 0.3577880859375, "reward/accuracy": 0.6875, "reward/chosen": -1.3125, "reward/margin": 2.078125, "reward/rejected": -3.390625, "step": 1489 }, { "approx. KL/chosen": 476.0, "approx. KL/rejected": 836.0, "epoch": 0.4469106178764247, "grad_norm": 5.790197372436523, "learning_rate": 5.834710874660051e-06, "logp/chosen": -304.0, "logp/rejected": -392.0, "loss": 0.819305419921875, "reward/accuracy": 0.625, "reward/chosen": -2.3125, "reward/margin": 1.0234375, "reward/rejected": -3.34375, "step": 1490 }, { "approx. KL/chosen": 358.0, "approx. KL/rejected": 1400.0, "epoch": 0.4472105578884223, "grad_norm": 6.1874775886535645, "learning_rate": 5.8300651752832096e-06, "logp/chosen": -314.0, "logp/rejected": -464.0, "loss": 0.5550193786621094, "reward/accuracy": 0.625, "reward/chosen": -2.140625, "reward/margin": 2.265625, "reward/rejected": -4.40625, "step": 1491 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 548.0, "epoch": 0.4475104979004199, "grad_norm": 5.210012912750244, "learning_rate": 5.825418738883921e-06, "logp/chosen": -370.0, "logp/rejected": -270.0, "loss": 0.64703369140625, "reward/accuracy": 0.6875, "reward/chosen": -1.84375, "reward/margin": 1.0, "reward/rejected": -2.84375, "step": 1492 }, { "approx. KL/chosen": 90.5, "approx. KL/rejected": 640.0, "epoch": 0.44781043791241754, "grad_norm": 3.500736713409424, "learning_rate": 5.820771569587795e-06, "logp/chosen": -352.0, "logp/rejected": -398.0, "loss": 0.3236236572265625, "reward/accuracy": 0.8125, "reward/chosen": -1.0390625, "reward/margin": 2.0625, "reward/rejected": -3.109375, "step": 1493 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 820.0, "epoch": 0.4481103779244151, "grad_norm": 7.519548416137695, "learning_rate": 5.816123671521099e-06, "logp/chosen": -396.0, "logp/rejected": -418.0, "loss": 0.62481689453125, "reward/accuracy": 0.75, "reward/chosen": -1.9140625, "reward/margin": 1.734375, "reward/rejected": -3.65625, "step": 1494 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 592.0, "epoch": 0.44841031793641273, "grad_norm": 4.764280796051025, "learning_rate": 5.811475048810744e-06, "logp/chosen": -298.0, "logp/rejected": -314.0, "loss": 0.490509033203125, "reward/accuracy": 0.625, "reward/chosen": -1.6640625, "reward/margin": 1.4140625, "reward/rejected": -3.078125, "step": 1495 }, { "approx. KL/chosen": 324.0, "approx. KL/rejected": 812.0, "epoch": 0.4487102579484103, "grad_norm": 5.3020219802856445, "learning_rate": 5.806825705584281e-06, "logp/chosen": -238.0, "logp/rejected": -346.0, "loss": 0.596527099609375, "reward/accuracy": 0.75, "reward/chosen": -2.015625, "reward/margin": 1.5859375, "reward/rejected": -3.59375, "step": 1496 }, { "approx. KL/chosen": 430.0, "approx. KL/rejected": 936.0, "epoch": 0.4490101979604079, "grad_norm": 3.63726806640625, "learning_rate": 5.80217564596991e-06, "logp/chosen": -380.0, "logp/rejected": -438.0, "loss": 0.35400390625, "reward/accuracy": 0.8125, "reward/chosen": -2.046875, "reward/margin": 1.6015625, "reward/rejected": -3.65625, "step": 1497 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 512.0, "epoch": 0.44931013797240554, "grad_norm": 5.366282939910889, "learning_rate": 5.797524874096455e-06, "logp/chosen": -346.0, "logp/rejected": -334.0, "loss": 0.69287109375, "reward/accuracy": 0.625, "reward/chosen": -1.78125, "reward/margin": 0.921875, "reward/rejected": -2.703125, "step": 1498 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 1904.0, "epoch": 0.4496100779844031, "grad_norm": 4.24479341506958, "learning_rate": 5.792873394093383e-06, "logp/chosen": -200.0, "logp/rejected": -308.0, "loss": 0.541412353515625, "reward/accuracy": 0.8125, "reward/chosen": -1.9140625, "reward/margin": 2.828125, "reward/rejected": -4.75, "step": 1499 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 724.0, "epoch": 0.44991001799640074, "grad_norm": 6.524581432342529, "learning_rate": 5.788221210090785e-06, "logp/chosen": -314.0, "logp/rejected": -332.0, "loss": 0.6137237548828125, "reward/accuracy": 0.5625, "reward/chosen": -2.015625, "reward/margin": 1.2265625, "reward/rejected": -3.25, "step": 1500 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 652.0, "epoch": 0.4502099580083983, "grad_norm": 3.0660464763641357, "learning_rate": 5.783568326219376e-06, "logp/chosen": -253.0, "logp/rejected": -384.0, "loss": 0.43660736083984375, "reward/accuracy": 0.6875, "reward/chosen": -1.09375, "reward/margin": 1.875, "reward/rejected": -2.96875, "step": 1501 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 640.0, "epoch": 0.4505098980203959, "grad_norm": 4.000159740447998, "learning_rate": 5.778914746610497e-06, "logp/chosen": -360.0, "logp/rejected": -448.0, "loss": 0.429443359375, "reward/accuracy": 0.6875, "reward/chosen": -1.9140625, "reward/margin": 1.1875, "reward/rejected": -3.09375, "step": 1502 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 470.0, "epoch": 0.4508098380323935, "grad_norm": 4.1148271560668945, "learning_rate": 5.7742604753961e-06, "logp/chosen": -300.0, "logp/rejected": -408.0, "loss": 0.42926025390625, "reward/accuracy": 0.9375, "reward/chosen": -1.25, "reward/margin": 1.4921875, "reward/rejected": -2.734375, "step": 1503 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 888.0, "epoch": 0.4511097780443911, "grad_norm": 3.2177560329437256, "learning_rate": 5.769605516708758e-06, "logp/chosen": -210.0, "logp/rejected": -404.0, "loss": 0.268035888671875, "reward/accuracy": 0.875, "reward/chosen": -1.40625, "reward/margin": 2.09375, "reward/rejected": -3.5, "step": 1504 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 620.0, "epoch": 0.45140971805638874, "grad_norm": 2.9530842304229736, "learning_rate": 5.764949874681651e-06, "logp/chosen": -304.0, "logp/rejected": -298.0, "loss": 0.43206787109375, "reward/accuracy": 0.8125, "reward/chosen": -1.6796875, "reward/margin": 1.421875, "reward/rejected": -3.09375, "step": 1505 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 960.0, "epoch": 0.4517096580683863, "grad_norm": 3.4845869541168213, "learning_rate": 5.760293553448564e-06, "logp/chosen": -290.0, "logp/rejected": -280.0, "loss": 0.430419921875, "reward/accuracy": 0.8125, "reward/chosen": -1.6875, "reward/margin": 1.8125, "reward/rejected": -3.5, "step": 1506 }, { "approx. KL/chosen": 106.5, "approx. KL/rejected": 326.0, "epoch": 0.45200959808038393, "grad_norm": 4.00917387008667, "learning_rate": 5.75563655714389e-06, "logp/chosen": -408.0, "logp/rejected": -358.0, "loss": 0.48876953125, "reward/accuracy": 0.75, "reward/chosen": -1.25, "reward/margin": 0.87890625, "reward/rejected": -2.125, "step": 1507 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 264.0, "epoch": 0.4523095380923815, "grad_norm": 4.262923717498779, "learning_rate": 5.750978889902616e-06, "logp/chosen": -328.0, "logp/rejected": -340.0, "loss": 0.595458984375, "reward/accuracy": 0.5625, "reward/chosen": -1.3984375, "reward/margin": 0.61328125, "reward/rejected": -2.015625, "step": 1508 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 560.0, "epoch": 0.4526094781043791, "grad_norm": 5.440158367156982, "learning_rate": 5.7463205558603285e-06, "logp/chosen": -382.0, "logp/rejected": -342.0, "loss": 0.5089111328125, "reward/accuracy": 0.6875, "reward/chosen": -1.359375, "reward/margin": 1.265625, "reward/rejected": -2.625, "step": 1509 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 560.0, "epoch": 0.45290941811637675, "grad_norm": 4.459059238433838, "learning_rate": 5.741661559153204e-06, "logp/chosen": -312.0, "logp/rejected": -366.0, "loss": 0.55682373046875, "reward/accuracy": 0.8125, "reward/chosen": -1.5234375, "reward/margin": 1.1015625, "reward/rejected": -2.625, "step": 1510 }, { "approx. KL/chosen": 142.0, "approx. KL/rejected": 640.0, "epoch": 0.4532093581283743, "grad_norm": 2.5184829235076904, "learning_rate": 5.737001903918009e-06, "logp/chosen": -448.0, "logp/rejected": -364.0, "loss": 0.193267822265625, "reward/accuracy": 0.9375, "reward/chosen": -0.90234375, "reward/margin": 2.484375, "reward/rejected": -3.390625, "step": 1511 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 732.0, "epoch": 0.45350929814037194, "grad_norm": 4.052749156951904, "learning_rate": 5.732341594292094e-06, "logp/chosen": -214.0, "logp/rejected": -300.0, "loss": 0.5106353759765625, "reward/accuracy": 0.6875, "reward/chosen": -1.359375, "reward/margin": 1.8515625, "reward/rejected": -3.21875, "step": 1512 }, { "approx. KL/chosen": 123.0, "approx. KL/rejected": 580.0, "epoch": 0.4538092381523695, "grad_norm": 2.7172398567199707, "learning_rate": 5.72768063441339e-06, "logp/chosen": -372.0, "logp/rejected": -334.0, "loss": 0.2971038818359375, "reward/accuracy": 0.8125, "reward/chosen": -1.0546875, "reward/margin": 1.9921875, "reward/rejected": -3.046875, "step": 1513 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 608.0, "epoch": 0.45410917816436713, "grad_norm": 4.812954425811768, "learning_rate": 5.723019028420409e-06, "logp/chosen": -288.0, "logp/rejected": -256.0, "loss": 0.470703125, "reward/accuracy": 0.8125, "reward/chosen": -1.3515625, "reward/margin": 1.4375, "reward/rejected": -2.78125, "step": 1514 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 500.0, "epoch": 0.4544091181763647, "grad_norm": 3.6400928497314453, "learning_rate": 5.718356780452228e-06, "logp/chosen": -253.0, "logp/rejected": -370.0, "loss": 0.427978515625, "reward/accuracy": 0.875, "reward/chosen": -1.421875, "reward/margin": 1.3671875, "reward/rejected": -2.78125, "step": 1515 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 498.0, "epoch": 0.4547090581883623, "grad_norm": 4.213711261749268, "learning_rate": 5.713693894648502e-06, "logp/chosen": -384.0, "logp/rejected": -310.0, "loss": 0.431884765625, "reward/accuracy": 0.8125, "reward/chosen": -1.1015625, "reward/margin": 1.4140625, "reward/rejected": -2.515625, "step": 1516 }, { "approx. KL/chosen": 108.5, "approx. KL/rejected": 372.0, "epoch": 0.45500899820035995, "grad_norm": 3.3163340091705322, "learning_rate": 5.70903037514945e-06, "logp/chosen": -496.0, "logp/rejected": -422.0, "loss": 0.3616943359375, "reward/accuracy": 0.875, "reward/chosen": -0.73046875, "reward/margin": 1.7109375, "reward/rejected": -2.4375, "step": 1517 }, { "approx. KL/chosen": 133.0, "approx. KL/rejected": 450.0, "epoch": 0.4553089382123575, "grad_norm": 3.3190224170684814, "learning_rate": 5.7043662260958545e-06, "logp/chosen": -282.0, "logp/rejected": -310.0, "loss": 0.433349609375, "reward/accuracy": 0.75, "reward/chosen": -1.234375, "reward/margin": 1.3828125, "reward/rejected": -2.609375, "step": 1518 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 692.0, "epoch": 0.45560887822435514, "grad_norm": 3.0059876441955566, "learning_rate": 5.699701451629055e-06, "logp/chosen": -304.0, "logp/rejected": -247.0, "loss": 0.3380393981933594, "reward/accuracy": 0.875, "reward/chosen": -1.4375, "reward/margin": 1.90625, "reward/rejected": -3.34375, "step": 1519 }, { "approx. KL/chosen": 94.5, "approx. KL/rejected": 520.0, "epoch": 0.4559088182363527, "grad_norm": 1.889092206954956, "learning_rate": 5.695036055890947e-06, "logp/chosen": -340.0, "logp/rejected": -314.0, "loss": 0.18359375, "reward/accuracy": 1.0, "reward/chosen": -0.80078125, "reward/margin": 1.9765625, "reward/rejected": -2.78125, "step": 1520 }, { "approx. KL/chosen": 392.0, "approx. KL/rejected": 524.0, "epoch": 0.45620875824835033, "grad_norm": 7.556766510009766, "learning_rate": 5.690370043023979e-06, "logp/chosen": -332.0, "logp/rejected": -312.0, "loss": 0.76123046875, "reward/accuracy": 0.6875, "reward/chosen": -2.109375, "reward/margin": 0.51171875, "reward/rejected": -2.625, "step": 1521 }, { "approx. KL/chosen": 110.5, "approx. KL/rejected": 680.0, "epoch": 0.45650869826034796, "grad_norm": 3.526629686355591, "learning_rate": 5.685703417171144e-06, "logp/chosen": -368.0, "logp/rejected": -370.0, "loss": 0.45050048828125, "reward/accuracy": 0.6875, "reward/chosen": -1.1015625, "reward/margin": 1.78125, "reward/rejected": -2.890625, "step": 1522 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 564.0, "epoch": 0.4568086382723455, "grad_norm": 4.386633396148682, "learning_rate": 5.681036182475986e-06, "logp/chosen": -278.0, "logp/rejected": -298.0, "loss": 0.4835205078125, "reward/accuracy": 0.75, "reward/chosen": -1.2421875, "reward/margin": 1.4921875, "reward/rejected": -2.734375, "step": 1523 }, { "approx. KL/chosen": 238.0, "approx. KL/rejected": 280.0, "epoch": 0.45710857828434315, "grad_norm": 7.052920818328857, "learning_rate": 5.676368343082582e-06, "logp/chosen": -227.0, "logp/rejected": -246.0, "loss": 0.852294921875, "reward/accuracy": 0.5, "reward/chosen": -1.6875, "reward/margin": 0.302734375, "reward/rejected": -1.9921875, "step": 1524 }, { "approx. KL/chosen": 112.0, "approx. KL/rejected": 420.0, "epoch": 0.4574085182963407, "grad_norm": 2.9433298110961914, "learning_rate": 5.67169990313555e-06, "logp/chosen": -246.0, "logp/rejected": -193.0, "loss": 0.4356689453125, "reward/accuracy": 0.8125, "reward/chosen": -0.92578125, "reward/margin": 1.3671875, "reward/rejected": -2.296875, "step": 1525 }, { "approx. KL/chosen": 123.5, "approx. KL/rejected": 744.0, "epoch": 0.45770845830833834, "grad_norm": 3.7915236949920654, "learning_rate": 5.667030866780041e-06, "logp/chosen": -170.0, "logp/rejected": -268.0, "loss": 0.282989501953125, "reward/accuracy": 0.75, "reward/chosen": -1.1484375, "reward/margin": 2.21875, "reward/rejected": -3.375, "step": 1526 }, { "approx. KL/chosen": 225.0, "approx. KL/rejected": 604.0, "epoch": 0.4580083983203359, "grad_norm": 3.5452558994293213, "learning_rate": 5.6623612381617346e-06, "logp/chosen": -356.0, "logp/rejected": -320.0, "loss": 0.56005859375, "reward/accuracy": 0.6875, "reward/chosen": -1.6796875, "reward/margin": 1.234375, "reward/rejected": -2.921875, "step": 1527 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 380.0, "epoch": 0.45830833833233353, "grad_norm": 5.776950836181641, "learning_rate": 5.657691021426836e-06, "logp/chosen": -266.0, "logp/rejected": -314.0, "loss": 0.572479248046875, "reward/accuracy": 0.6875, "reward/chosen": -1.3359375, "reward/margin": 0.87109375, "reward/rejected": -2.203125, "step": 1528 }, { "approx. KL/chosen": 137.0, "approx. KL/rejected": 688.0, "epoch": 0.45860827834433116, "grad_norm": 4.193502426147461, "learning_rate": 5.653020220722075e-06, "logp/chosen": -304.0, "logp/rejected": -384.0, "loss": 0.4410400390625, "reward/accuracy": 0.75, "reward/chosen": -1.3125, "reward/margin": 1.609375, "reward/rejected": -2.921875, "step": 1529 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 696.0, "epoch": 0.4589082183563287, "grad_norm": 7.611412525177002, "learning_rate": 5.648348840194696e-06, "logp/chosen": -251.0, "logp/rejected": -338.0, "loss": 0.462799072265625, "reward/accuracy": 0.8125, "reward/chosen": -1.3828125, "reward/margin": 1.96875, "reward/rejected": -3.359375, "step": 1530 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 696.0, "epoch": 0.45920815836832635, "grad_norm": 3.7177939414978027, "learning_rate": 5.6436768839924625e-06, "logp/chosen": -250.0, "logp/rejected": -280.0, "loss": 0.356231689453125, "reward/accuracy": 0.8125, "reward/chosen": -1.578125, "reward/margin": 1.6953125, "reward/rejected": -3.265625, "step": 1531 }, { "approx. KL/chosen": 212.0, "approx. KL/rejected": 820.0, "epoch": 0.4595080983803239, "grad_norm": 4.53534460067749, "learning_rate": 5.639004356263646e-06, "logp/chosen": -310.0, "logp/rejected": -360.0, "loss": 0.5540771484375, "reward/accuracy": 0.625, "reward/chosen": -1.640625, "reward/margin": 1.5078125, "reward/rejected": -3.140625, "step": 1532 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 876.0, "epoch": 0.45980803839232154, "grad_norm": 3.580339193344116, "learning_rate": 5.6343312611570266e-06, "logp/chosen": -274.0, "logp/rejected": -470.0, "loss": 0.3291015625, "reward/accuracy": 0.8125, "reward/chosen": -1.6875, "reward/margin": 1.984375, "reward/rejected": -3.671875, "step": 1533 }, { "approx. KL/chosen": 123.0, "approx. KL/rejected": 744.0, "epoch": 0.46010797840431916, "grad_norm": 3.1534953117370605, "learning_rate": 5.629657602821888e-06, "logp/chosen": -209.0, "logp/rejected": -274.0, "loss": 0.25479888916015625, "reward/accuracy": 0.875, "reward/chosen": -1.015625, "reward/margin": 2.578125, "reward/rejected": -3.59375, "step": 1534 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 462.0, "epoch": 0.46040791841631673, "grad_norm": 4.865708351135254, "learning_rate": 5.624983385408014e-06, "logp/chosen": -318.0, "logp/rejected": -282.0, "loss": 0.7164306640625, "reward/accuracy": 0.625, "reward/chosen": -1.65625, "reward/margin": 0.95703125, "reward/rejected": -2.609375, "step": 1535 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 436.0, "epoch": 0.46070785842831435, "grad_norm": 3.8951358795166016, "learning_rate": 5.620308613065686e-06, "logp/chosen": -268.0, "logp/rejected": -318.0, "loss": 0.51763916015625, "reward/accuracy": 0.75, "reward/chosen": -1.125, "reward/margin": 1.40625, "reward/rejected": -2.53125, "step": 1536 }, { "approx. KL/chosen": 246.0, "approx. KL/rejected": 604.0, "epoch": 0.4610077984403119, "grad_norm": 3.800938606262207, "learning_rate": 5.615633289945676e-06, "logp/chosen": -348.0, "logp/rejected": -306.0, "loss": 0.505859375, "reward/accuracy": 0.6875, "reward/chosen": -1.4140625, "reward/margin": 1.3125, "reward/rejected": -2.71875, "step": 1537 }, { "approx. KL/chosen": 92.5, "approx. KL/rejected": 772.0, "epoch": 0.46130773845230955, "grad_norm": 2.5837905406951904, "learning_rate": 5.610957420199247e-06, "logp/chosen": -364.0, "logp/rejected": -444.0, "loss": 0.181396484375, "reward/accuracy": 0.9375, "reward/chosen": -0.89453125, "reward/margin": 2.5, "reward/rejected": -3.390625, "step": 1538 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 812.0, "epoch": 0.4616076784643071, "grad_norm": 2.2275986671447754, "learning_rate": 5.606281007978145e-06, "logp/chosen": -324.0, "logp/rejected": -286.0, "loss": 0.2786865234375, "reward/accuracy": 0.8125, "reward/chosen": -1.6015625, "reward/margin": 1.84375, "reward/rejected": -3.4375, "step": 1539 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 852.0, "epoch": 0.46190761847630474, "grad_norm": 3.795747756958008, "learning_rate": 5.6016040574345996e-06, "logp/chosen": -322.0, "logp/rejected": -362.0, "loss": 0.502777099609375, "reward/accuracy": 0.6875, "reward/chosen": -1.5390625, "reward/margin": 1.453125, "reward/rejected": -3.0, "step": 1540 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 984.0, "epoch": 0.46220755848830236, "grad_norm": 2.9652788639068604, "learning_rate": 5.596926572721319e-06, "logp/chosen": -296.0, "logp/rejected": -364.0, "loss": 0.3282470703125, "reward/accuracy": 0.8125, "reward/chosen": -1.5859375, "reward/margin": 2.109375, "reward/rejected": -3.703125, "step": 1541 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 832.0, "epoch": 0.46250749850029993, "grad_norm": 2.709434747695923, "learning_rate": 5.592248557991483e-06, "logp/chosen": -255.0, "logp/rejected": -280.0, "loss": 0.23681640625, "reward/accuracy": 0.875, "reward/chosen": -1.78125, "reward/margin": 1.984375, "reward/rejected": -3.765625, "step": 1542 }, { "approx. KL/chosen": 348.0, "approx. KL/rejected": 912.0, "epoch": 0.46280743851229755, "grad_norm": 4.747045516967773, "learning_rate": 5.5875700173987455e-06, "logp/chosen": -296.0, "logp/rejected": -328.0, "loss": 0.632537841796875, "reward/accuracy": 0.625, "reward/chosen": -1.9765625, "reward/margin": 1.65625, "reward/rejected": -3.625, "step": 1543 }, { "approx. KL/chosen": 175.0, "approx. KL/rejected": 904.0, "epoch": 0.4631073785242951, "grad_norm": 3.7777228355407715, "learning_rate": 5.582890955097223e-06, "logp/chosen": -342.0, "logp/rejected": -380.0, "loss": 0.346282958984375, "reward/accuracy": 0.75, "reward/chosen": -1.3203125, "reward/margin": 2.53125, "reward/rejected": -3.84375, "step": 1544 }, { "approx. KL/chosen": 344.0, "approx. KL/rejected": 620.0, "epoch": 0.46340731853629274, "grad_norm": 4.973840713500977, "learning_rate": 5.5782113752415015e-06, "logp/chosen": -274.0, "logp/rejected": -362.0, "loss": 0.47796630859375, "reward/accuracy": 0.75, "reward/chosen": -2.109375, "reward/margin": 1.1796875, "reward/rejected": -3.28125, "step": 1545 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 720.0, "epoch": 0.46370725854829037, "grad_norm": 4.008981227874756, "learning_rate": 5.5735312819866176e-06, "logp/chosen": -298.0, "logp/rejected": -294.0, "loss": 0.5289306640625, "reward/accuracy": 0.6875, "reward/chosen": -1.6640625, "reward/margin": 1.5390625, "reward/rejected": -3.203125, "step": 1546 }, { "approx. KL/chosen": 106.0, "approx. KL/rejected": 652.0, "epoch": 0.46400719856028794, "grad_norm": 3.000020980834961, "learning_rate": 5.568850679488071e-06, "logp/chosen": -262.0, "logp/rejected": -278.0, "loss": 0.402191162109375, "reward/accuracy": 0.8125, "reward/chosen": -1.0546875, "reward/margin": 1.921875, "reward/rejected": -2.96875, "step": 1547 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 1296.0, "epoch": 0.46430713857228556, "grad_norm": 5.400137424468994, "learning_rate": 5.564169571901814e-06, "logp/chosen": -270.0, "logp/rejected": -376.0, "loss": 0.4027557373046875, "reward/accuracy": 0.875, "reward/chosen": -1.953125, "reward/margin": 2.421875, "reward/rejected": -4.375, "step": 1548 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 398.0, "epoch": 0.46460707858428313, "grad_norm": 5.128246307373047, "learning_rate": 5.559487963384242e-06, "logp/chosen": -274.0, "logp/rejected": -306.0, "loss": 0.6529083251953125, "reward/accuracy": 0.5625, "reward/chosen": -1.34375, "reward/margin": 0.87890625, "reward/rejected": -2.21875, "step": 1549 }, { "approx. KL/chosen": 141.0, "approx. KL/rejected": 896.0, "epoch": 0.46490701859628075, "grad_norm": 4.455383777618408, "learning_rate": 5.554805858092198e-06, "logp/chosen": -430.0, "logp/rejected": -392.0, "loss": 0.344573974609375, "reward/accuracy": 0.875, "reward/chosen": -1.2265625, "reward/margin": 2.484375, "reward/rejected": -3.71875, "step": 1550 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 548.0, "epoch": 0.4652069586082783, "grad_norm": 4.019834518432617, "learning_rate": 5.550123260182967e-06, "logp/chosen": -354.0, "logp/rejected": -328.0, "loss": 0.5667724609375, "reward/accuracy": 0.75, "reward/chosen": -1.7265625, "reward/margin": 0.94921875, "reward/rejected": -2.671875, "step": 1551 }, { "approx. KL/chosen": 123.0, "approx. KL/rejected": 600.0, "epoch": 0.46550689862027594, "grad_norm": 2.6464614868164062, "learning_rate": 5.545440173814269e-06, "logp/chosen": -382.0, "logp/rejected": -390.0, "loss": 0.22772216796875, "reward/accuracy": 0.9375, "reward/chosen": -1.1015625, "reward/margin": 2.03125, "reward/rejected": -3.125, "step": 1552 }, { "approx. KL/chosen": 251.0, "approx. KL/rejected": 358.0, "epoch": 0.46580683863227357, "grad_norm": 5.746090412139893, "learning_rate": 5.5407566031442605e-06, "logp/chosen": -398.0, "logp/rejected": -394.0, "loss": 0.88262939453125, "reward/accuracy": 0.625, "reward/chosen": -1.6171875, "reward/margin": 0.404296875, "reward/rejected": -2.015625, "step": 1553 }, { "approx. KL/chosen": 121.5, "approx. KL/rejected": 648.0, "epoch": 0.46610677864427114, "grad_norm": 2.9511115550994873, "learning_rate": 5.536072552331526e-06, "logp/chosen": -370.0, "logp/rejected": -402.0, "loss": 0.2899627685546875, "reward/accuracy": 1.0, "reward/chosen": -1.1171875, "reward/margin": 1.7890625, "reward/rejected": -2.90625, "step": 1554 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 592.0, "epoch": 0.46640671865626876, "grad_norm": 3.1018128395080566, "learning_rate": 5.531388025535079e-06, "logp/chosen": -308.0, "logp/rejected": -352.0, "loss": 0.404052734375, "reward/accuracy": 0.75, "reward/chosen": -1.5390625, "reward/margin": 1.328125, "reward/rejected": -2.859375, "step": 1555 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 656.0, "epoch": 0.4667066586682663, "grad_norm": 3.6582248210906982, "learning_rate": 5.526703026914351e-06, "logp/chosen": -336.0, "logp/rejected": -300.0, "loss": 0.43975830078125, "reward/accuracy": 0.6875, "reward/chosen": -1.546875, "reward/margin": 1.5234375, "reward/rejected": -3.078125, "step": 1556 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 512.0, "epoch": 0.46700659868026395, "grad_norm": 3.430124282836914, "learning_rate": 5.5220175606291955e-06, "logp/chosen": -328.0, "logp/rejected": -386.0, "loss": 0.4158935546875, "reward/accuracy": 0.8125, "reward/chosen": -1.6015625, "reward/margin": 1.125, "reward/rejected": -2.734375, "step": 1557 }, { "approx. KL/chosen": 126.0, "approx. KL/rejected": 588.0, "epoch": 0.4673065386922616, "grad_norm": 3.7798447608947754, "learning_rate": 5.517331630839882e-06, "logp/chosen": -272.0, "logp/rejected": -256.0, "loss": 0.3475379943847656, "reward/accuracy": 0.8125, "reward/chosen": -1.0625, "reward/margin": 1.984375, "reward/rejected": -3.046875, "step": 1558 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 836.0, "epoch": 0.46760647870425914, "grad_norm": 3.4258859157562256, "learning_rate": 5.51264524170709e-06, "logp/chosen": -346.0, "logp/rejected": -362.0, "loss": 0.393218994140625, "reward/accuracy": 0.8125, "reward/chosen": -1.328125, "reward/margin": 2.0, "reward/rejected": -3.328125, "step": 1559 }, { "approx. KL/chosen": 332.0, "approx. KL/rejected": 536.0, "epoch": 0.46790641871625677, "grad_norm": 5.375891208648682, "learning_rate": 5.5079583973919046e-06, "logp/chosen": -330.0, "logp/rejected": -318.0, "loss": 0.69696044921875, "reward/accuracy": 0.5625, "reward/chosen": -1.8515625, "reward/margin": 0.83203125, "reward/rejected": -2.671875, "step": 1560 }, { "approx. KL/chosen": 338.0, "approx. KL/rejected": 1056.0, "epoch": 0.46820635872825433, "grad_norm": 4.05057954788208, "learning_rate": 5.503271102055822e-06, "logp/chosen": -288.0, "logp/rejected": -384.0, "loss": 0.411285400390625, "reward/accuracy": 0.6875, "reward/chosen": -1.8203125, "reward/margin": 2.0, "reward/rejected": -3.8125, "step": 1561 }, { "approx. KL/chosen": 180.0, "approx. KL/rejected": 904.0, "epoch": 0.46850629874025196, "grad_norm": 2.990084171295166, "learning_rate": 5.498583359860732e-06, "logp/chosen": -199.0, "logp/rejected": -300.0, "loss": 0.275848388671875, "reward/accuracy": 0.875, "reward/chosen": -1.5078125, "reward/margin": 2.203125, "reward/rejected": -3.703125, "step": 1562 }, { "approx. KL/chosen": 45.25, "approx. KL/rejected": 482.0, "epoch": 0.4688062387522495, "grad_norm": 3.144700527191162, "learning_rate": 5.493895174968923e-06, "logp/chosen": -300.0, "logp/rejected": -326.0, "loss": 0.3332672119140625, "reward/accuracy": 0.8125, "reward/chosen": -0.59375, "reward/margin": 1.9921875, "reward/rejected": -2.578125, "step": 1563 }, { "approx. KL/chosen": 117.5, "approx. KL/rejected": 430.0, "epoch": 0.46910617876424715, "grad_norm": 4.571965217590332, "learning_rate": 5.489206551543079e-06, "logp/chosen": -394.0, "logp/rejected": -262.0, "loss": 0.4019775390625, "reward/accuracy": 0.8125, "reward/chosen": -1.0703125, "reward/margin": 1.4375, "reward/rejected": -2.5, "step": 1564 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 764.0, "epoch": 0.4694061187762448, "grad_norm": 2.7631165981292725, "learning_rate": 5.484517493746271e-06, "logp/chosen": -284.0, "logp/rejected": -296.0, "loss": 0.29742431640625, "reward/accuracy": 0.875, "reward/chosen": -1.625, "reward/margin": 1.90625, "reward/rejected": -3.53125, "step": 1565 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 464.0, "epoch": 0.46970605878824234, "grad_norm": 5.851556301116943, "learning_rate": 5.479828005741956e-06, "logp/chosen": -334.0, "logp/rejected": -318.0, "loss": 0.5433349609375, "reward/accuracy": 0.6875, "reward/chosen": -1.640625, "reward/margin": 0.99609375, "reward/rejected": -2.640625, "step": 1566 }, { "approx. KL/chosen": 119.0, "approx. KL/rejected": 398.0, "epoch": 0.47000599880023997, "grad_norm": 3.6024229526519775, "learning_rate": 5.475138091693974e-06, "logp/chosen": -374.0, "logp/rejected": -268.0, "loss": 0.40667724609375, "reward/accuracy": 0.8125, "reward/chosen": -1.1875, "reward/margin": 1.3125, "reward/rejected": -2.5, "step": 1567 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 660.0, "epoch": 0.47030593881223753, "grad_norm": 6.094951629638672, "learning_rate": 5.470447755766542e-06, "logp/chosen": -300.0, "logp/rejected": -418.0, "loss": 0.49253082275390625, "reward/accuracy": 0.6875, "reward/chosen": -1.6796875, "reward/margin": 1.5078125, "reward/rejected": -3.1875, "step": 1568 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 848.0, "epoch": 0.47060587882423516, "grad_norm": 3.2658865451812744, "learning_rate": 5.46575700212425e-06, "logp/chosen": -348.0, "logp/rejected": -414.0, "loss": 0.267059326171875, "reward/accuracy": 0.9375, "reward/chosen": -1.609375, "reward/margin": 1.9296875, "reward/rejected": -3.546875, "step": 1569 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 828.0, "epoch": 0.4709058188362327, "grad_norm": 2.9829485416412354, "learning_rate": 5.461065834932064e-06, "logp/chosen": -212.0, "logp/rejected": -276.0, "loss": 0.37091064453125, "reward/accuracy": 0.8125, "reward/chosen": -1.7109375, "reward/margin": 2.0625, "reward/rejected": -3.765625, "step": 1570 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 804.0, "epoch": 0.47120575884823035, "grad_norm": 4.107412815093994, "learning_rate": 5.456374258355313e-06, "logp/chosen": -316.0, "logp/rejected": -420.0, "loss": 0.4589996337890625, "reward/accuracy": 0.75, "reward/chosen": -1.203125, "reward/margin": 1.90625, "reward/rejected": -3.109375, "step": 1571 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 756.0, "epoch": 0.471505698860228, "grad_norm": 2.2509005069732666, "learning_rate": 5.45168227655969e-06, "logp/chosen": -251.0, "logp/rejected": -300.0, "loss": 0.20880126953125, "reward/accuracy": 1.0, "reward/chosen": -1.3046875, "reward/margin": 2.015625, "reward/rejected": -3.3125, "step": 1572 }, { "approx. KL/chosen": 392.0, "approx. KL/rejected": 952.0, "epoch": 0.47180563887222554, "grad_norm": 4.119349002838135, "learning_rate": 5.446989893711251e-06, "logp/chosen": -342.0, "logp/rejected": -342.0, "loss": 0.42340087890625, "reward/accuracy": 0.75, "reward/chosen": -1.9375, "reward/margin": 1.78125, "reward/rejected": -3.71875, "step": 1573 }, { "approx. KL/chosen": 148.0, "approx. KL/rejected": 604.0, "epoch": 0.47210557888422316, "grad_norm": 3.79744291305542, "learning_rate": 5.442297113976402e-06, "logp/chosen": -368.0, "logp/rejected": -390.0, "loss": 0.451904296875, "reward/accuracy": 0.75, "reward/chosen": -1.2578125, "reward/margin": 1.7421875, "reward/rejected": -3.0, "step": 1574 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 572.0, "epoch": 0.47240551889622073, "grad_norm": 4.291261672973633, "learning_rate": 5.437603941521909e-06, "logp/chosen": -272.0, "logp/rejected": -276.0, "loss": 0.436737060546875, "reward/accuracy": 0.75, "reward/chosen": -1.46875, "reward/margin": 1.578125, "reward/rejected": -3.046875, "step": 1575 }, { "approx. KL/chosen": 124.5, "approx. KL/rejected": 378.0, "epoch": 0.47270545890821836, "grad_norm": 3.2794413566589355, "learning_rate": 5.4329103805148775e-06, "logp/chosen": -358.0, "logp/rejected": -374.0, "loss": 0.4393310546875, "reward/accuracy": 0.75, "reward/chosen": -1.140625, "reward/margin": 1.1328125, "reward/rejected": -2.28125, "step": 1576 }, { "approx. KL/chosen": 388.0, "approx. KL/rejected": 748.0, "epoch": 0.473005398920216, "grad_norm": 5.198923587799072, "learning_rate": 5.4282164351227685e-06, "logp/chosen": -358.0, "logp/rejected": -382.0, "loss": 0.4619140625, "reward/accuracy": 0.6875, "reward/chosen": -1.8203125, "reward/margin": 1.484375, "reward/rejected": -3.296875, "step": 1577 }, { "approx. KL/chosen": 696.0, "approx. KL/rejected": 896.0, "epoch": 0.47330533893221355, "grad_norm": 8.095059394836426, "learning_rate": 5.423522109513376e-06, "logp/chosen": -336.0, "logp/rejected": -294.0, "loss": 1.136932373046875, "reward/accuracy": 0.75, "reward/chosen": -2.140625, "reward/margin": 1.609375, "reward/rejected": -3.75, "step": 1578 }, { "approx. KL/chosen": 222.0, "approx. KL/rejected": 380.0, "epoch": 0.47360527894421117, "grad_norm": 5.9383320808410645, "learning_rate": 5.418827407854834e-06, "logp/chosen": -384.0, "logp/rejected": -378.0, "loss": 0.7503662109375, "reward/accuracy": 0.5625, "reward/chosen": -1.8046875, "reward/margin": 0.484375, "reward/rejected": -2.296875, "step": 1579 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 568.0, "epoch": 0.47390521895620874, "grad_norm": 5.139063358306885, "learning_rate": 5.414132334315614e-06, "logp/chosen": -302.0, "logp/rejected": -312.0, "loss": 0.70245361328125, "reward/accuracy": 0.4375, "reward/chosen": -1.90625, "reward/margin": 0.76953125, "reward/rejected": -2.671875, "step": 1580 }, { "approx. KL/chosen": 386.0, "approx. KL/rejected": 696.0, "epoch": 0.47420515896820636, "grad_norm": 5.810208320617676, "learning_rate": 5.409436893064511e-06, "logp/chosen": -338.0, "logp/rejected": -382.0, "loss": 0.51739501953125, "reward/accuracy": 0.75, "reward/chosen": -2.265625, "reward/margin": 1.1328125, "reward/rejected": -3.40625, "step": 1581 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 532.0, "epoch": 0.47450509898020393, "grad_norm": 4.754612922668457, "learning_rate": 5.404741088270654e-06, "logp/chosen": -249.0, "logp/rejected": -332.0, "loss": 0.623046875, "reward/accuracy": 0.6875, "reward/chosen": -1.5390625, "reward/margin": 1.2734375, "reward/rejected": -2.8125, "step": 1582 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 672.0, "epoch": 0.47480503899220156, "grad_norm": 3.716033935546875, "learning_rate": 5.400044924103488e-06, "logp/chosen": -370.0, "logp/rejected": -354.0, "loss": 0.539794921875, "reward/accuracy": 0.6875, "reward/chosen": -1.515625, "reward/margin": 1.6171875, "reward/rejected": -3.140625, "step": 1583 }, { "approx. KL/chosen": 187.0, "approx. KL/rejected": 648.0, "epoch": 0.4751049790041992, "grad_norm": 3.567739486694336, "learning_rate": 5.395348404732782e-06, "logp/chosen": -312.0, "logp/rejected": -276.0, "loss": 0.309326171875, "reward/accuracy": 0.8125, "reward/chosen": -1.59375, "reward/margin": 1.6640625, "reward/rejected": -3.25, "step": 1584 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 712.0, "epoch": 0.47540491901619675, "grad_norm": 4.4309492111206055, "learning_rate": 5.390651534328616e-06, "logp/chosen": -215.0, "logp/rejected": -310.0, "loss": 0.4581298828125, "reward/accuracy": 0.6875, "reward/chosen": -1.53125, "reward/margin": 1.71875, "reward/rejected": -3.25, "step": 1585 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 808.0, "epoch": 0.47570485902819437, "grad_norm": 4.67612886428833, "learning_rate": 5.385954317061386e-06, "logp/chosen": -352.0, "logp/rejected": -316.0, "loss": 0.36065673828125, "reward/accuracy": 0.8125, "reward/chosen": -1.359375, "reward/margin": 2.15625, "reward/rejected": -3.515625, "step": 1586 }, { "approx. KL/chosen": 177.0, "approx. KL/rejected": 692.0, "epoch": 0.47600479904019194, "grad_norm": 4.002135276794434, "learning_rate": 5.381256757101794e-06, "logp/chosen": -390.0, "logp/rejected": -396.0, "loss": 0.3921966552734375, "reward/accuracy": 0.75, "reward/chosen": -1.046875, "reward/margin": 1.71875, "reward/rejected": -2.765625, "step": 1587 }, { "approx. KL/chosen": 93.5, "approx. KL/rejected": 548.0, "epoch": 0.47630473905218956, "grad_norm": 3.571847438812256, "learning_rate": 5.376558858620844e-06, "logp/chosen": -486.0, "logp/rejected": -396.0, "loss": 0.3924560546875, "reward/accuracy": 0.6875, "reward/chosen": -0.765625, "reward/margin": 2.015625, "reward/rejected": -2.78125, "step": 1588 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 528.0, "epoch": 0.4766046790641872, "grad_norm": 4.974588871002197, "learning_rate": 5.371860625789847e-06, "logp/chosen": -384.0, "logp/rejected": -332.0, "loss": 0.41009521484375, "reward/accuracy": 0.75, "reward/chosen": -1.1953125, "reward/margin": 1.609375, "reward/rejected": -2.796875, "step": 1589 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 956.0, "epoch": 0.47690461907618475, "grad_norm": 2.4178545475006104, "learning_rate": 5.367162062780404e-06, "logp/chosen": -270.0, "logp/rejected": -418.0, "loss": 0.179290771484375, "reward/accuracy": 0.875, "reward/chosen": -1.171875, "reward/margin": 2.78125, "reward/rejected": -3.953125, "step": 1590 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 644.0, "epoch": 0.4772045590881824, "grad_norm": 2.5032103061676025, "learning_rate": 5.362463173764415e-06, "logp/chosen": -402.0, "logp/rejected": -270.0, "loss": 0.3546142578125, "reward/accuracy": 0.9375, "reward/chosen": -1.1484375, "reward/margin": 2.125, "reward/rejected": -3.28125, "step": 1591 }, { "approx. KL/chosen": 103.5, "approx. KL/rejected": 552.0, "epoch": 0.47750449910017995, "grad_norm": 2.5108284950256348, "learning_rate": 5.357763962914062e-06, "logp/chosen": -446.0, "logp/rejected": -360.0, "loss": 0.290252685546875, "reward/accuracy": 0.875, "reward/chosen": -0.6171875, "reward/margin": 2.15625, "reward/rejected": -2.78125, "step": 1592 }, { "approx. KL/chosen": 348.0, "approx. KL/rejected": 740.0, "epoch": 0.47780443911217757, "grad_norm": 4.5316901206970215, "learning_rate": 5.35306443440182e-06, "logp/chosen": -328.0, "logp/rejected": -288.0, "loss": 0.61187744140625, "reward/accuracy": 0.625, "reward/chosen": -2.046875, "reward/margin": 1.0859375, "reward/rejected": -3.125, "step": 1593 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 350.0, "epoch": 0.47810437912417514, "grad_norm": 4.502467155456543, "learning_rate": 5.348364592400442e-06, "logp/chosen": -278.0, "logp/rejected": -264.0, "loss": 0.5537109375, "reward/accuracy": 0.625, "reward/chosen": -1.3203125, "reward/margin": 0.6484375, "reward/rejected": -1.96875, "step": 1594 }, { "approx. KL/chosen": 124.5, "approx. KL/rejected": 480.0, "epoch": 0.47840431913617276, "grad_norm": 4.418668270111084, "learning_rate": 5.343664441082964e-06, "logp/chosen": -338.0, "logp/rejected": -378.0, "loss": 0.323150634765625, "reward/accuracy": 0.8125, "reward/chosen": -1.0234375, "reward/margin": 1.609375, "reward/rejected": -2.625, "step": 1595 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 430.0, "epoch": 0.4787042591481704, "grad_norm": 4.632256984710693, "learning_rate": 5.3389639846226885e-06, "logp/chosen": -316.0, "logp/rejected": -340.0, "loss": 0.4065284729003906, "reward/accuracy": 0.875, "reward/chosen": -1.1171875, "reward/margin": 1.4921875, "reward/rejected": -2.609375, "step": 1596 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 628.0, "epoch": 0.47900419916016795, "grad_norm": 4.645503997802734, "learning_rate": 5.334263227193196e-06, "logp/chosen": -304.0, "logp/rejected": -292.0, "loss": 0.6239013671875, "reward/accuracy": 0.625, "reward/chosen": -1.6640625, "reward/margin": 0.984375, "reward/rejected": -2.65625, "step": 1597 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 400.0, "epoch": 0.4793041391721656, "grad_norm": 5.590077877044678, "learning_rate": 5.3295621729683325e-06, "logp/chosen": -396.0, "logp/rejected": -356.0, "loss": 0.802947998046875, "reward/accuracy": 0.6875, "reward/chosen": -1.484375, "reward/margin": 0.875, "reward/rejected": -2.359375, "step": 1598 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 420.0, "epoch": 0.47960407918416315, "grad_norm": 5.379322528839111, "learning_rate": 5.324860826122205e-06, "logp/chosen": -312.0, "logp/rejected": -356.0, "loss": 0.52838134765625, "reward/accuracy": 0.75, "reward/chosen": -1.203125, "reward/margin": 1.265625, "reward/rejected": -2.46875, "step": 1599 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 752.0, "epoch": 0.47990401919616077, "grad_norm": 3.050180673599243, "learning_rate": 5.320159190829186e-06, "logp/chosen": -532.0, "logp/rejected": -420.0, "loss": 0.2191314697265625, "reward/accuracy": 0.875, "reward/chosen": -0.859375, "reward/margin": 2.515625, "reward/rejected": -3.375, "step": 1600 }, { "approx. KL/chosen": 498.0, "approx. KL/rejected": 736.0, "epoch": 0.4802039592081584, "grad_norm": 6.518435001373291, "learning_rate": 5.315457271263896e-06, "logp/chosen": -356.0, "logp/rejected": -420.0, "loss": 0.78125, "reward/accuracy": 0.5625, "reward/chosen": -2.40625, "reward/margin": 0.482421875, "reward/rejected": -2.890625, "step": 1601 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 712.0, "epoch": 0.48050389922015596, "grad_norm": 3.6931850910186768, "learning_rate": 5.310755071601215e-06, "logp/chosen": -286.0, "logp/rejected": -304.0, "loss": 0.35137939453125, "reward/accuracy": 0.875, "reward/chosen": -1.453125, "reward/margin": 1.828125, "reward/rejected": -3.28125, "step": 1602 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 532.0, "epoch": 0.4808038392321536, "grad_norm": 3.931149959564209, "learning_rate": 5.306052596016269e-06, "logp/chosen": -388.0, "logp/rejected": -384.0, "loss": 0.32916259765625, "reward/accuracy": 0.8125, "reward/chosen": -1.046875, "reward/margin": 1.890625, "reward/rejected": -2.9375, "step": 1603 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 704.0, "epoch": 0.48110377924415115, "grad_norm": 2.7150702476501465, "learning_rate": 5.301349848684427e-06, "logp/chosen": -348.0, "logp/rejected": -278.0, "loss": 0.314666748046875, "reward/accuracy": 0.875, "reward/chosen": -1.4453125, "reward/margin": 1.9296875, "reward/rejected": -3.375, "step": 1604 }, { "approx. KL/chosen": 70.0, "approx. KL/rejected": 372.0, "epoch": 0.4814037192561488, "grad_norm": 2.970515012741089, "learning_rate": 5.296646833781305e-06, "logp/chosen": -342.0, "logp/rejected": -318.0, "loss": 0.3100433349609375, "reward/accuracy": 0.875, "reward/chosen": -0.6171875, "reward/margin": 1.8359375, "reward/rejected": -2.453125, "step": 1605 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 932.0, "epoch": 0.48170365926814634, "grad_norm": 4.95019006729126, "learning_rate": 5.2919435554827485e-06, "logp/chosen": -488.0, "logp/rejected": -350.0, "loss": 0.4017486572265625, "reward/accuracy": 0.8125, "reward/chosen": -1.921875, "reward/margin": 1.8125, "reward/rejected": -3.734375, "step": 1606 }, { "approx. KL/chosen": 406.0, "approx. KL/rejected": 548.0, "epoch": 0.48200359928014397, "grad_norm": 5.533806324005127, "learning_rate": 5.287240017964844e-06, "logp/chosen": -330.0, "logp/rejected": -326.0, "loss": 0.63818359375, "reward/accuracy": 0.6875, "reward/chosen": -2.265625, "reward/margin": 0.7421875, "reward/rejected": -3.0, "step": 1607 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 496.0, "epoch": 0.4823035392921416, "grad_norm": 5.14366340637207, "learning_rate": 5.282536225403904e-06, "logp/chosen": -302.0, "logp/rejected": -380.0, "loss": 0.60662841796875, "reward/accuracy": 0.75, "reward/chosen": -1.3125, "reward/margin": 1.359375, "reward/rejected": -2.671875, "step": 1608 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 780.0, "epoch": 0.48260347930413916, "grad_norm": 5.7192511558532715, "learning_rate": 5.27783218197647e-06, "logp/chosen": -292.0, "logp/rejected": -266.0, "loss": 0.82666015625, "reward/accuracy": 0.6875, "reward/chosen": -1.7109375, "reward/margin": 1.1875, "reward/rejected": -2.90625, "step": 1609 }, { "approx. KL/chosen": 460.0, "approx. KL/rejected": 460.0, "epoch": 0.4829034193161368, "grad_norm": 6.1455254554748535, "learning_rate": 5.273127891859304e-06, "logp/chosen": -268.0, "logp/rejected": -282.0, "loss": 0.888427734375, "reward/accuracy": 0.5625, "reward/chosen": -2.296875, "reward/margin": 0.3046875, "reward/rejected": -2.609375, "step": 1610 }, { "approx. KL/chosen": 254.0, "approx. KL/rejected": 704.0, "epoch": 0.48320335932813435, "grad_norm": 9.484704971313477, "learning_rate": 5.26842335922939e-06, "logp/chosen": -460.0, "logp/rejected": -370.0, "loss": 0.4642333984375, "reward/accuracy": 0.625, "reward/chosen": -1.890625, "reward/margin": 1.390625, "reward/rejected": -3.28125, "step": 1611 }, { "approx. KL/chosen": 249.0, "approx. KL/rejected": 932.0, "epoch": 0.483503299340132, "grad_norm": 3.5124382972717285, "learning_rate": 5.263718588263924e-06, "logp/chosen": -294.0, "logp/rejected": -308.0, "loss": 0.453125, "reward/accuracy": 0.875, "reward/chosen": -1.765625, "reward/margin": 1.84375, "reward/rejected": -3.609375, "step": 1612 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 836.0, "epoch": 0.4838032393521296, "grad_norm": 3.5882537364959717, "learning_rate": 5.259013583140316e-06, "logp/chosen": -324.0, "logp/rejected": -384.0, "loss": 0.4718170166015625, "reward/accuracy": 0.6875, "reward/chosen": -1.6015625, "reward/margin": 2.03125, "reward/rejected": -3.640625, "step": 1613 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 812.0, "epoch": 0.48410317936412717, "grad_norm": 3.8783838748931885, "learning_rate": 5.25430834803618e-06, "logp/chosen": -404.0, "logp/rejected": -298.0, "loss": 0.3438720703125, "reward/accuracy": 0.9375, "reward/chosen": -1.6796875, "reward/margin": 1.875, "reward/rejected": -3.5625, "step": 1614 }, { "approx. KL/chosen": 233.0, "approx. KL/rejected": 580.0, "epoch": 0.4844031193761248, "grad_norm": 3.410367727279663, "learning_rate": 5.2496028871293424e-06, "logp/chosen": -370.0, "logp/rejected": -304.0, "loss": 0.3492431640625, "reward/accuracy": 0.875, "reward/chosen": -1.640625, "reward/margin": 1.5546875, "reward/rejected": -3.1875, "step": 1615 }, { "approx. KL/chosen": 248.0, "approx. KL/rejected": 572.0, "epoch": 0.48470305938812236, "grad_norm": 3.10982346534729, "learning_rate": 5.244897204597821e-06, "logp/chosen": -524.0, "logp/rejected": -422.0, "loss": 0.3206787109375, "reward/accuracy": 0.8125, "reward/chosen": -1.8984375, "reward/margin": 1.25, "reward/rejected": -3.15625, "step": 1616 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 560.0, "epoch": 0.48500299940012, "grad_norm": 4.706833839416504, "learning_rate": 5.240191304619836e-06, "logp/chosen": -292.0, "logp/rejected": -416.0, "loss": 0.5091552734375, "reward/accuracy": 0.8125, "reward/chosen": -1.625, "reward/margin": 1.390625, "reward/rejected": -3.015625, "step": 1617 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 748.0, "epoch": 0.48530293941211755, "grad_norm": 5.199319839477539, "learning_rate": 5.235485191373797e-06, "logp/chosen": -442.0, "logp/rejected": -336.0, "loss": 0.4764404296875, "reward/accuracy": 0.6875, "reward/chosen": -2.0625, "reward/margin": 1.296875, "reward/rejected": -3.359375, "step": 1618 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 620.0, "epoch": 0.4856028794241152, "grad_norm": 3.299220085144043, "learning_rate": 5.230778869038307e-06, "logp/chosen": -358.0, "logp/rejected": -278.0, "loss": 0.36505126953125, "reward/accuracy": 0.875, "reward/chosen": -1.453125, "reward/margin": 1.828125, "reward/rejected": -3.28125, "step": 1619 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 446.0, "epoch": 0.4859028194361128, "grad_norm": 3.627156972885132, "learning_rate": 5.226072341792153e-06, "logp/chosen": -296.0, "logp/rejected": -334.0, "loss": 0.486328125, "reward/accuracy": 0.75, "reward/chosen": -1.6484375, "reward/margin": 1.125, "reward/rejected": -2.765625, "step": 1620 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 668.0, "epoch": 0.48620275944811037, "grad_norm": 3.751380681991577, "learning_rate": 5.221365613814301e-06, "logp/chosen": -276.0, "logp/rejected": -304.0, "loss": 0.556427001953125, "reward/accuracy": 0.625, "reward/chosen": -1.6640625, "reward/margin": 1.359375, "reward/rejected": -3.03125, "step": 1621 }, { "approx. KL/chosen": 222.0, "approx. KL/rejected": 688.0, "epoch": 0.486502699460108, "grad_norm": 2.9306817054748535, "learning_rate": 5.2166586892839e-06, "logp/chosen": -344.0, "logp/rejected": -344.0, "loss": 0.2830810546875, "reward/accuracy": 0.9375, "reward/chosen": -1.265625, "reward/margin": 2.0625, "reward/rejected": -3.328125, "step": 1622 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 768.0, "epoch": 0.48680263947210556, "grad_norm": 5.335057735443115, "learning_rate": 5.211951572380269e-06, "logp/chosen": -524.0, "logp/rejected": -370.0, "loss": 0.4291534423828125, "reward/accuracy": 0.8125, "reward/chosen": -1.578125, "reward/margin": 2.09375, "reward/rejected": -3.671875, "step": 1623 }, { "approx. KL/chosen": 316.0, "approx. KL/rejected": 572.0, "epoch": 0.4871025794841032, "grad_norm": 4.4131364822387695, "learning_rate": 5.2072442672829036e-06, "logp/chosen": -193.0, "logp/rejected": -206.0, "loss": 0.43310546875, "reward/accuracy": 0.8125, "reward/chosen": -1.6171875, "reward/margin": 1.078125, "reward/rejected": -2.703125, "step": 1624 }, { "approx. KL/chosen": 118.0, "approx. KL/rejected": 640.0, "epoch": 0.4874025194961008, "grad_norm": 2.159156560897827, "learning_rate": 5.20253677817146e-06, "logp/chosen": -416.0, "logp/rejected": -344.0, "loss": 0.235626220703125, "reward/accuracy": 0.9375, "reward/chosen": -1.3125, "reward/margin": 1.96875, "reward/rejected": -3.28125, "step": 1625 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 808.0, "epoch": 0.4877024595080984, "grad_norm": 5.894382476806641, "learning_rate": 5.197829109225761e-06, "logp/chosen": -450.0, "logp/rejected": -362.0, "loss": 0.37274169921875, "reward/accuracy": 0.8125, "reward/chosen": -1.71875, "reward/margin": 1.8046875, "reward/rejected": -3.53125, "step": 1626 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 1016.0, "epoch": 0.488002399520096, "grad_norm": 3.5813679695129395, "learning_rate": 5.193121264625791e-06, "logp/chosen": -338.0, "logp/rejected": -358.0, "loss": 0.34287261962890625, "reward/accuracy": 0.8125, "reward/chosen": -2.125, "reward/margin": 2.046875, "reward/rejected": -4.1875, "step": 1627 }, { "approx. KL/chosen": 316.0, "approx. KL/rejected": 608.0, "epoch": 0.48830233953209357, "grad_norm": 5.086663246154785, "learning_rate": 5.188413248551684e-06, "logp/chosen": -352.0, "logp/rejected": -326.0, "loss": 0.37176513671875, "reward/accuracy": 0.875, "reward/chosen": -1.8046875, "reward/margin": 1.3984375, "reward/rejected": -3.203125, "step": 1628 }, { "approx. KL/chosen": 484.0, "approx. KL/rejected": 792.0, "epoch": 0.4886022795440912, "grad_norm": 9.759343147277832, "learning_rate": 5.183705065183736e-06, "logp/chosen": -474.0, "logp/rejected": -370.0, "loss": 0.730712890625, "reward/accuracy": 0.75, "reward/chosen": -2.1875, "reward/margin": 1.3125, "reward/rejected": -3.5, "step": 1629 }, { "approx. KL/chosen": 98.0, "approx. KL/rejected": 442.0, "epoch": 0.48890221955608876, "grad_norm": 4.181701183319092, "learning_rate": 5.178996718702383e-06, "logp/chosen": -330.0, "logp/rejected": -374.0, "loss": 0.400848388671875, "reward/accuracy": 0.75, "reward/chosen": -0.98828125, "reward/margin": 1.5234375, "reward/rejected": -2.515625, "step": 1630 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 576.0, "epoch": 0.4892021595680864, "grad_norm": 5.854031562805176, "learning_rate": 5.174288213288208e-06, "logp/chosen": -358.0, "logp/rejected": -280.0, "loss": 0.6956787109375, "reward/accuracy": 0.625, "reward/chosen": -1.875, "reward/margin": 0.88671875, "reward/rejected": -2.765625, "step": 1631 }, { "approx. KL/chosen": 249.0, "approx. KL/rejected": 448.0, "epoch": 0.489502099580084, "grad_norm": 3.986849069595337, "learning_rate": 5.1695795531219375e-06, "logp/chosen": -352.0, "logp/rejected": -350.0, "loss": 0.5528564453125, "reward/accuracy": 0.625, "reward/chosen": -1.6484375, "reward/margin": 0.85546875, "reward/rejected": -2.5, "step": 1632 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 976.0, "epoch": 0.4898020395920816, "grad_norm": 4.336747169494629, "learning_rate": 5.164870742384431e-06, "logp/chosen": -354.0, "logp/rejected": -312.0, "loss": 0.42421722412109375, "reward/accuracy": 0.875, "reward/chosen": -1.8671875, "reward/margin": 2.171875, "reward/rejected": -4.03125, "step": 1633 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 580.0, "epoch": 0.4901019796040792, "grad_norm": 4.281094074249268, "learning_rate": 5.16016178525669e-06, "logp/chosen": -217.0, "logp/rejected": -254.0, "loss": 0.46954345703125, "reward/accuracy": 0.8125, "reward/chosen": -1.5234375, "reward/margin": 1.46875, "reward/rejected": -3.0, "step": 1634 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 688.0, "epoch": 0.49040191961607676, "grad_norm": 3.7327449321746826, "learning_rate": 5.1554526859198365e-06, "logp/chosen": -266.0, "logp/rejected": -350.0, "loss": 0.43145751953125, "reward/accuracy": 0.625, "reward/chosen": -1.6640625, "reward/margin": 1.4765625, "reward/rejected": -3.140625, "step": 1635 }, { "approx. KL/chosen": 177.0, "approx. KL/rejected": 664.0, "epoch": 0.4907018596280744, "grad_norm": 2.6926889419555664, "learning_rate": 5.150743448555123e-06, "logp/chosen": -294.0, "logp/rejected": -320.0, "loss": 0.24827957153320312, "reward/accuracy": 0.8125, "reward/chosen": -0.79296875, "reward/margin": 2.265625, "reward/rejected": -3.0625, "step": 1636 }, { "approx. KL/chosen": 215.0, "approx. KL/rejected": 768.0, "epoch": 0.491001799640072, "grad_norm": 3.6343326568603516, "learning_rate": 5.146034077343925e-06, "logp/chosen": -228.0, "logp/rejected": -350.0, "loss": 0.377410888671875, "reward/accuracy": 0.8125, "reward/chosen": -1.6640625, "reward/margin": 1.734375, "reward/rejected": -3.390625, "step": 1637 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 592.0, "epoch": 0.4913017396520696, "grad_norm": 3.3790502548217773, "learning_rate": 5.141324576467736e-06, "logp/chosen": -272.0, "logp/rejected": -276.0, "loss": 0.46759033203125, "reward/accuracy": 0.75, "reward/chosen": -1.5078125, "reward/margin": 1.4296875, "reward/rejected": -2.9375, "step": 1638 }, { "approx. KL/chosen": 290.0, "approx. KL/rejected": 510.0, "epoch": 0.4916016796640672, "grad_norm": 4.704582691192627, "learning_rate": 5.136614950108167e-06, "logp/chosen": -366.0, "logp/rejected": -392.0, "loss": 0.6224365234375, "reward/accuracy": 0.625, "reward/chosen": -1.7265625, "reward/margin": 1.0078125, "reward/rejected": -2.734375, "step": 1639 }, { "approx. KL/chosen": 175.0, "approx. KL/rejected": 908.0, "epoch": 0.49190161967606477, "grad_norm": 3.2188773155212402, "learning_rate": 5.131905202446935e-06, "logp/chosen": -274.0, "logp/rejected": -344.0, "loss": 0.220550537109375, "reward/accuracy": 0.9375, "reward/chosen": -1.515625, "reward/margin": 2.359375, "reward/rejected": -3.875, "step": 1640 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 612.0, "epoch": 0.4922015596880624, "grad_norm": 2.529719114303589, "learning_rate": 5.12719533766587e-06, "logp/chosen": -350.0, "logp/rejected": -388.0, "loss": 0.236419677734375, "reward/accuracy": 1.0, "reward/chosen": -1.4375, "reward/margin": 1.828125, "reward/rejected": -3.265625, "step": 1641 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 672.0, "epoch": 0.49250149970005996, "grad_norm": 3.492076873779297, "learning_rate": 5.122485359946902e-06, "logp/chosen": -402.0, "logp/rejected": -356.0, "loss": 0.43218994140625, "reward/accuracy": 0.75, "reward/chosen": -1.9609375, "reward/margin": 1.484375, "reward/rejected": -3.453125, "step": 1642 }, { "approx. KL/chosen": 174.0, "approx. KL/rejected": 452.0, "epoch": 0.4928014397120576, "grad_norm": 3.4463422298431396, "learning_rate": 5.117775273472066e-06, "logp/chosen": -322.0, "logp/rejected": -330.0, "loss": 0.466552734375, "reward/accuracy": 0.75, "reward/chosen": -1.6015625, "reward/margin": 0.95703125, "reward/rejected": -2.5625, "step": 1643 }, { "approx. KL/chosen": 480.0, "approx. KL/rejected": 1008.0, "epoch": 0.4931013797240552, "grad_norm": 6.310573101043701, "learning_rate": 5.113065082423487e-06, "logp/chosen": -422.0, "logp/rejected": -314.0, "loss": 0.7426223754882812, "reward/accuracy": 0.75, "reward/chosen": -2.375, "reward/margin": 1.609375, "reward/rejected": -3.984375, "step": 1644 }, { "approx. KL/chosen": 252.0, "approx. KL/rejected": 512.0, "epoch": 0.4934013197360528, "grad_norm": 5.056336879730225, "learning_rate": 5.108354790983391e-06, "logp/chosen": -380.0, "logp/rejected": -360.0, "loss": 0.509857177734375, "reward/accuracy": 0.75, "reward/chosen": -1.4296875, "reward/margin": 1.1953125, "reward/rejected": -2.625, "step": 1645 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 696.0, "epoch": 0.4937012597480504, "grad_norm": 4.319741725921631, "learning_rate": 5.103644403334086e-06, "logp/chosen": -256.0, "logp/rejected": -292.0, "loss": 0.6375732421875, "reward/accuracy": 0.4375, "reward/chosen": -1.65625, "reward/margin": 1.234375, "reward/rejected": -2.890625, "step": 1646 }, { "approx. KL/chosen": 247.0, "approx. KL/rejected": 868.0, "epoch": 0.49400119976004797, "grad_norm": 4.991474151611328, "learning_rate": 5.098933923657968e-06, "logp/chosen": -308.0, "logp/rejected": -320.0, "loss": 0.65106201171875, "reward/accuracy": 0.6875, "reward/chosen": -1.7265625, "reward/margin": 1.484375, "reward/rejected": -3.203125, "step": 1647 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 716.0, "epoch": 0.4943011397720456, "grad_norm": 3.924842357635498, "learning_rate": 5.094223356137517e-06, "logp/chosen": -221.0, "logp/rejected": -256.0, "loss": 0.39972686767578125, "reward/accuracy": 0.8125, "reward/chosen": -1.515625, "reward/margin": 1.8515625, "reward/rejected": -3.359375, "step": 1648 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 1024.0, "epoch": 0.4946010797840432, "grad_norm": 3.486417293548584, "learning_rate": 5.089512704955288e-06, "logp/chosen": -276.0, "logp/rejected": -600.0, "loss": 0.3943328857421875, "reward/accuracy": 0.8125, "reward/chosen": -1.75, "reward/margin": 2.15625, "reward/rejected": -3.90625, "step": 1649 }, { "approx. KL/chosen": 376.0, "approx. KL/rejected": 1024.0, "epoch": 0.4949010197960408, "grad_norm": 4.119996070861816, "learning_rate": 5.084801974293909e-06, "logp/chosen": -446.0, "logp/rejected": -428.0, "loss": 0.38427734375, "reward/accuracy": 0.8125, "reward/chosen": -2.40625, "reward/margin": 1.6953125, "reward/rejected": -4.09375, "step": 1650 }, { "approx. KL/chosen": 233.0, "approx. KL/rejected": 494.0, "epoch": 0.4952009598080384, "grad_norm": 3.4893975257873535, "learning_rate": 5.080091168336086e-06, "logp/chosen": -330.0, "logp/rejected": -310.0, "loss": 0.461090087890625, "reward/accuracy": 0.875, "reward/chosen": -1.21875, "reward/margin": 1.296875, "reward/rejected": -2.515625, "step": 1651 }, { "approx. KL/chosen": 290.0, "approx. KL/rejected": 900.0, "epoch": 0.495500899820036, "grad_norm": 4.692022323608398, "learning_rate": 5.07538029126458e-06, "logp/chosen": -396.0, "logp/rejected": -376.0, "loss": 0.407745361328125, "reward/accuracy": 0.75, "reward/chosen": -1.7421875, "reward/margin": 1.953125, "reward/rejected": -3.6875, "step": 1652 }, { "approx. KL/chosen": 243.0, "approx. KL/rejected": 536.0, "epoch": 0.4958008398320336, "grad_norm": 4.4416656494140625, "learning_rate": 5.070669347262227e-06, "logp/chosen": -312.0, "logp/rejected": -304.0, "loss": 0.716064453125, "reward/accuracy": 0.625, "reward/chosen": -1.7421875, "reward/margin": 1.015625, "reward/rejected": -2.75, "step": 1653 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 580.0, "epoch": 0.49610077984403117, "grad_norm": 4.29437780380249, "learning_rate": 5.065958340511914e-06, "logp/chosen": -306.0, "logp/rejected": -270.0, "loss": 0.452117919921875, "reward/accuracy": 0.8125, "reward/chosen": -1.234375, "reward/margin": 1.671875, "reward/rejected": -2.90625, "step": 1654 }, { "approx. KL/chosen": 212.0, "approx. KL/rejected": 584.0, "epoch": 0.4964007198560288, "grad_norm": 3.291134834289551, "learning_rate": 5.061247275196589e-06, "logp/chosen": -378.0, "logp/rejected": -354.0, "loss": 0.314727783203125, "reward/accuracy": 0.875, "reward/chosen": -1.2890625, "reward/margin": 1.625, "reward/rejected": -2.921875, "step": 1655 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 840.0, "epoch": 0.4967006598680264, "grad_norm": 3.989924907684326, "learning_rate": 5.056536155499249e-06, "logp/chosen": -356.0, "logp/rejected": -410.0, "loss": 0.4281005859375, "reward/accuracy": 0.6875, "reward/chosen": -1.5546875, "reward/margin": 1.796875, "reward/rejected": -3.34375, "step": 1656 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 612.0, "epoch": 0.497000599880024, "grad_norm": 4.856938362121582, "learning_rate": 5.051824985602939e-06, "logp/chosen": -398.0, "logp/rejected": -344.0, "loss": 0.51593017578125, "reward/accuracy": 0.6875, "reward/chosen": -1.21875, "reward/margin": 1.671875, "reward/rejected": -2.890625, "step": 1657 }, { "approx. KL/chosen": 185.0, "approx. KL/rejected": 660.0, "epoch": 0.4973005398920216, "grad_norm": 2.742016315460205, "learning_rate": 5.0471137696907535e-06, "logp/chosen": -308.0, "logp/rejected": -298.0, "loss": 0.3660888671875, "reward/accuracy": 0.875, "reward/chosen": -1.6171875, "reward/margin": 1.6484375, "reward/rejected": -3.265625, "step": 1658 }, { "approx. KL/chosen": 104.0, "approx. KL/rejected": 644.0, "epoch": 0.4976004799040192, "grad_norm": 3.357384443283081, "learning_rate": 5.0424025119458195e-06, "logp/chosen": -360.0, "logp/rejected": -344.0, "loss": 0.408355712890625, "reward/accuracy": 0.6875, "reward/chosen": -1.0703125, "reward/margin": 1.71875, "reward/rejected": -2.796875, "step": 1659 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 556.0, "epoch": 0.4979004199160168, "grad_norm": 4.346411228179932, "learning_rate": 5.0376912165513096e-06, "logp/chosen": -426.0, "logp/rejected": -576.0, "loss": 0.469696044921875, "reward/accuracy": 0.75, "reward/chosen": -1.3046875, "reward/margin": 1.375, "reward/rejected": -2.6875, "step": 1660 }, { "approx. KL/chosen": 101.0, "approx. KL/rejected": 568.0, "epoch": 0.4982003599280144, "grad_norm": 2.488539457321167, "learning_rate": 5.032979887690424e-06, "logp/chosen": -334.0, "logp/rejected": -274.0, "loss": 0.302978515625, "reward/accuracy": 0.875, "reward/chosen": -0.9609375, "reward/margin": 1.8046875, "reward/rejected": -2.765625, "step": 1661 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 704.0, "epoch": 0.498500299940012, "grad_norm": 3.866537094116211, "learning_rate": 5.028268529546393e-06, "logp/chosen": -368.0, "logp/rejected": -424.0, "loss": 0.3177490234375, "reward/accuracy": 0.875, "reward/chosen": -1.203125, "reward/margin": 2.15625, "reward/rejected": -3.359375, "step": 1662 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 716.0, "epoch": 0.4988002399520096, "grad_norm": 4.219998359680176, "learning_rate": 5.023557146302479e-06, "logp/chosen": -588.0, "logp/rejected": -444.0, "loss": 0.509368896484375, "reward/accuracy": 0.6875, "reward/chosen": -1.796875, "reward/margin": 1.4765625, "reward/rejected": -3.265625, "step": 1663 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 420.0, "epoch": 0.4991001799640072, "grad_norm": 3.543034076690674, "learning_rate": 5.018845742141958e-06, "logp/chosen": -426.0, "logp/rejected": -328.0, "loss": 0.463134765625, "reward/accuracy": 0.8125, "reward/chosen": -1.4453125, "reward/margin": 1.1953125, "reward/rejected": -2.640625, "step": 1664 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 676.0, "epoch": 0.4994001199760048, "grad_norm": 5.6100287437438965, "learning_rate": 5.014134321248132e-06, "logp/chosen": -324.0, "logp/rejected": -262.0, "loss": 0.423858642578125, "reward/accuracy": 0.75, "reward/chosen": -1.875, "reward/margin": 1.5703125, "reward/rejected": -3.453125, "step": 1665 }, { "approx. KL/chosen": 148.0, "approx. KL/rejected": 382.0, "epoch": 0.4997000599880024, "grad_norm": 4.584682464599609, "learning_rate": 5.0094228878043115e-06, "logp/chosen": -508.0, "logp/rejected": -392.0, "loss": 0.4681396484375, "reward/accuracy": 0.8125, "reward/chosen": -1.296875, "reward/margin": 1.1484375, "reward/rejected": -2.4375, "step": 1666 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 1224.0, "epoch": 0.5, "grad_norm": 5.388979911804199, "learning_rate": 5.004711445993825e-06, "logp/chosen": -352.0, "logp/rejected": -344.0, "loss": 0.5604696273803711, "reward/accuracy": 0.6875, "reward/chosen": -1.578125, "reward/margin": 1.9765625, "reward/rejected": -3.546875, "step": 1667 }, { "approx. KL/chosen": 330.0, "approx. KL/rejected": 824.0, "epoch": 0.5002999400119976, "grad_norm": 2.9683175086975098, "learning_rate": 5e-06, "logp/chosen": -380.0, "logp/rejected": -424.0, "loss": 0.330718994140625, "reward/accuracy": 0.875, "reward/chosen": -2.125, "reward/margin": 1.5703125, "reward/rejected": -3.6875, "step": 1668 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 1012.0, "epoch": 0.5005998800239952, "grad_norm": 3.6200642585754395, "learning_rate": 4.995288554006177e-06, "logp/chosen": -338.0, "logp/rejected": -330.0, "loss": 0.4037017822265625, "reward/accuracy": 0.875, "reward/chosen": -1.7265625, "reward/margin": 2.171875, "reward/rejected": -3.90625, "step": 1669 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 632.0, "epoch": 0.5008998200359928, "grad_norm": 2.7947840690612793, "learning_rate": 4.9905771121956884e-06, "logp/chosen": -490.0, "logp/rejected": -450.0, "loss": 0.24688720703125, "reward/accuracy": 0.875, "reward/chosen": -1.3828125, "reward/margin": 1.75, "reward/rejected": -3.125, "step": 1670 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 636.0, "epoch": 0.5011997600479904, "grad_norm": 3.9626517295837402, "learning_rate": 4.985865678751871e-06, "logp/chosen": -358.0, "logp/rejected": -326.0, "loss": 0.52032470703125, "reward/accuracy": 0.8125, "reward/chosen": -1.6171875, "reward/margin": 1.3515625, "reward/rejected": -2.96875, "step": 1671 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 720.0, "epoch": 0.501499700059988, "grad_norm": 3.223501682281494, "learning_rate": 4.981154257858043e-06, "logp/chosen": -294.0, "logp/rejected": -384.0, "loss": 0.45086669921875, "reward/accuracy": 0.75, "reward/chosen": -1.1953125, "reward/margin": 1.8515625, "reward/rejected": -3.046875, "step": 1672 }, { "approx. KL/chosen": 213.0, "approx. KL/rejected": 792.0, "epoch": 0.5017996400719856, "grad_norm": 3.8843202590942383, "learning_rate": 4.976442853697522e-06, "logp/chosen": -249.0, "logp/rejected": -278.0, "loss": 0.4452362060546875, "reward/accuracy": 0.6875, "reward/chosen": -1.640625, "reward/margin": 1.8359375, "reward/rejected": -3.46875, "step": 1673 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 856.0, "epoch": 0.5020995800839833, "grad_norm": 2.5331671237945557, "learning_rate": 4.971731470453607e-06, "logp/chosen": -296.0, "logp/rejected": -250.0, "loss": 0.21905517578125, "reward/accuracy": 0.875, "reward/chosen": -1.5703125, "reward/margin": 2.171875, "reward/rejected": -3.734375, "step": 1674 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 494.0, "epoch": 0.5023995200959808, "grad_norm": 5.727078437805176, "learning_rate": 4.967020112309577e-06, "logp/chosen": -286.0, "logp/rejected": -270.0, "loss": 0.54412841796875, "reward/accuracy": 0.8125, "reward/chosen": -1.453125, "reward/margin": 1.2578125, "reward/rejected": -2.703125, "step": 1675 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 736.0, "epoch": 0.5026994601079784, "grad_norm": 5.42697811126709, "learning_rate": 4.962308783448693e-06, "logp/chosen": -394.0, "logp/rejected": -402.0, "loss": 0.669403076171875, "reward/accuracy": 0.625, "reward/chosen": -1.8203125, "reward/margin": 1.2265625, "reward/rejected": -3.046875, "step": 1676 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 568.0, "epoch": 0.502999400119976, "grad_norm": 3.3361592292785645, "learning_rate": 4.957597488054181e-06, "logp/chosen": -234.0, "logp/rejected": -262.0, "loss": 0.36334228515625, "reward/accuracy": 0.875, "reward/chosen": -1.2734375, "reward/margin": 1.6328125, "reward/rejected": -2.90625, "step": 1677 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 612.0, "epoch": 0.5032993401319736, "grad_norm": 4.5871100425720215, "learning_rate": 4.952886230309248e-06, "logp/chosen": -352.0, "logp/rejected": -235.0, "loss": 0.47442626953125, "reward/accuracy": 0.75, "reward/chosen": -1.96875, "reward/margin": 1.1875, "reward/rejected": -3.15625, "step": 1678 }, { "approx. KL/chosen": 141.0, "approx. KL/rejected": 872.0, "epoch": 0.5035992801439712, "grad_norm": 3.7041244506835938, "learning_rate": 4.948175014397061e-06, "logp/chosen": -388.0, "logp/rejected": -548.0, "loss": 0.26727294921875, "reward/accuracy": 0.875, "reward/chosen": -1.0546875, "reward/margin": 2.390625, "reward/rejected": -3.453125, "step": 1679 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 704.0, "epoch": 0.5038992201559688, "grad_norm": 3.265130043029785, "learning_rate": 4.943463844500754e-06, "logp/chosen": -276.0, "logp/rejected": -296.0, "loss": 0.32391357421875, "reward/accuracy": 0.75, "reward/chosen": -1.4609375, "reward/margin": 1.6640625, "reward/rejected": -3.125, "step": 1680 }, { "approx. KL/chosen": 123.5, "approx. KL/rejected": 744.0, "epoch": 0.5041991601679664, "grad_norm": 3.036505699157715, "learning_rate": 4.938752724803413e-06, "logp/chosen": -276.0, "logp/rejected": -418.0, "loss": 0.31273651123046875, "reward/accuracy": 0.875, "reward/chosen": -0.87890625, "reward/margin": 2.265625, "reward/rejected": -3.140625, "step": 1681 }, { "approx. KL/chosen": 187.0, "approx. KL/rejected": 552.0, "epoch": 0.504499100179964, "grad_norm": 4.340166091918945, "learning_rate": 4.934041659488088e-06, "logp/chosen": -432.0, "logp/rejected": -376.0, "loss": 0.4605712890625, "reward/accuracy": 0.6875, "reward/chosen": -1.21875, "reward/margin": 1.640625, "reward/rejected": -2.859375, "step": 1682 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 900.0, "epoch": 0.5047990401919616, "grad_norm": 2.6278154850006104, "learning_rate": 4.929330652737775e-06, "logp/chosen": -358.0, "logp/rejected": -270.0, "loss": 0.2542572021484375, "reward/accuracy": 1.0, "reward/chosen": -1.2890625, "reward/margin": 2.390625, "reward/rejected": -3.671875, "step": 1683 }, { "approx. KL/chosen": 119.5, "approx. KL/rejected": 768.0, "epoch": 0.5050989802039592, "grad_norm": 3.441176414489746, "learning_rate": 4.9246197087354204e-06, "logp/chosen": -380.0, "logp/rejected": -412.0, "loss": 0.30548095703125, "reward/accuracy": 0.875, "reward/chosen": -1.0234375, "reward/margin": 2.25, "reward/rejected": -3.265625, "step": 1684 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 640.0, "epoch": 0.5053989202159568, "grad_norm": 4.566925048828125, "learning_rate": 4.919908831663918e-06, "logp/chosen": -322.0, "logp/rejected": -362.0, "loss": 0.578857421875, "reward/accuracy": 0.6875, "reward/chosen": -1.265625, "reward/margin": 1.4453125, "reward/rejected": -2.71875, "step": 1685 }, { "approx. KL/chosen": 80.5, "approx. KL/rejected": 652.0, "epoch": 0.5056988602279544, "grad_norm": 3.368863582611084, "learning_rate": 4.9151980257060925e-06, "logp/chosen": -432.0, "logp/rejected": -444.0, "loss": 0.247344970703125, "reward/accuracy": 0.9375, "reward/chosen": -0.7265625, "reward/margin": 2.359375, "reward/rejected": -3.09375, "step": 1686 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 728.0, "epoch": 0.505998800239952, "grad_norm": 3.422234535217285, "learning_rate": 4.910487295044714e-06, "logp/chosen": -270.0, "logp/rejected": -282.0, "loss": 0.353271484375, "reward/accuracy": 0.875, "reward/chosen": -1.234375, "reward/margin": 1.8671875, "reward/rejected": -3.109375, "step": 1687 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 544.0, "epoch": 0.5062987402519497, "grad_norm": 3.7525699138641357, "learning_rate": 4.9057766438624834e-06, "logp/chosen": -278.0, "logp/rejected": -253.0, "loss": 0.39892578125, "reward/accuracy": 0.75, "reward/chosen": -1.6953125, "reward/margin": 1.203125, "reward/rejected": -2.890625, "step": 1688 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 668.0, "epoch": 0.5065986802639472, "grad_norm": 4.537018299102783, "learning_rate": 4.901066076342032e-06, "logp/chosen": -350.0, "logp/rejected": -310.0, "loss": 0.5343017578125, "reward/accuracy": 0.6875, "reward/chosen": -1.3203125, "reward/margin": 1.6328125, "reward/rejected": -2.953125, "step": 1689 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 764.0, "epoch": 0.5068986202759448, "grad_norm": 4.487181186676025, "learning_rate": 4.896355596665917e-06, "logp/chosen": -416.0, "logp/rejected": -400.0, "loss": 0.6507720947265625, "reward/accuracy": 0.75, "reward/chosen": -1.2734375, "reward/margin": 1.9296875, "reward/rejected": -3.203125, "step": 1690 }, { "approx. KL/chosen": 120.0, "approx. KL/rejected": 760.0, "epoch": 0.5071985602879424, "grad_norm": 3.6180896759033203, "learning_rate": 4.891645209016611e-06, "logp/chosen": -380.0, "logp/rejected": -348.0, "loss": 0.336639404296875, "reward/accuracy": 0.8125, "reward/chosen": -1.0625, "reward/margin": 2.1875, "reward/rejected": -3.25, "step": 1691 }, { "approx. KL/chosen": 104.5, "approx. KL/rejected": 884.0, "epoch": 0.50749850029994, "grad_norm": 2.7015304565429688, "learning_rate": 4.8869349175765144e-06, "logp/chosen": -348.0, "logp/rejected": -442.0, "loss": 0.31597900390625, "reward/accuracy": 0.8125, "reward/chosen": -1.0234375, "reward/margin": 2.296875, "reward/rejected": -3.328125, "step": 1692 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 1048.0, "epoch": 0.5077984403119377, "grad_norm": 4.874357223510742, "learning_rate": 4.882224726527935e-06, "logp/chosen": -580.0, "logp/rejected": -318.0, "loss": 0.5053138732910156, "reward/accuracy": 0.8125, "reward/chosen": -1.40625, "reward/margin": 2.375, "reward/rejected": -3.78125, "step": 1693 }, { "approx. KL/chosen": 372.0, "approx. KL/rejected": 640.0, "epoch": 0.5080983803239352, "grad_norm": 4.632598400115967, "learning_rate": 4.877514640053098e-06, "logp/chosen": -388.0, "logp/rejected": -344.0, "loss": 0.6480712890625, "reward/accuracy": 0.625, "reward/chosen": -2.1875, "reward/margin": 1.09375, "reward/rejected": -3.28125, "step": 1694 }, { "approx. KL/chosen": 218.0, "approx. KL/rejected": 824.0, "epoch": 0.5083983203359328, "grad_norm": 4.611573696136475, "learning_rate": 4.872804662334133e-06, "logp/chosen": -420.0, "logp/rejected": -408.0, "loss": 0.3938140869140625, "reward/accuracy": 0.875, "reward/chosen": -1.1484375, "reward/margin": 2.515625, "reward/rejected": -3.671875, "step": 1695 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 672.0, "epoch": 0.5086982603479304, "grad_norm": 4.521704196929932, "learning_rate": 4.8680947975530665e-06, "logp/chosen": -358.0, "logp/rejected": -334.0, "loss": 0.5323486328125, "reward/accuracy": 0.75, "reward/chosen": -1.7890625, "reward/margin": 1.3515625, "reward/rejected": -3.140625, "step": 1696 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 520.0, "epoch": 0.508998200359928, "grad_norm": 5.2463860511779785, "learning_rate": 4.863385049891834e-06, "logp/chosen": -366.0, "logp/rejected": -338.0, "loss": 0.5675048828125, "reward/accuracy": 0.8125, "reward/chosen": -1.3515625, "reward/margin": 1.3828125, "reward/rejected": -2.734375, "step": 1697 }, { "approx. KL/chosen": 213.0, "approx. KL/rejected": 536.0, "epoch": 0.5092981403719257, "grad_norm": 4.163737773895264, "learning_rate": 4.858675423532264e-06, "logp/chosen": -276.0, "logp/rejected": -328.0, "loss": 0.4853515625, "reward/accuracy": 0.6875, "reward/chosen": -1.3984375, "reward/margin": 1.375, "reward/rejected": -2.765625, "step": 1698 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 944.0, "epoch": 0.5095980803839232, "grad_norm": 2.967378616333008, "learning_rate": 4.853965922656075e-06, "logp/chosen": -266.0, "logp/rejected": -352.0, "loss": 0.312225341796875, "reward/accuracy": 0.75, "reward/chosen": -1.3203125, "reward/margin": 2.359375, "reward/rejected": -3.6875, "step": 1699 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 952.0, "epoch": 0.5098980203959208, "grad_norm": 3.0663070678710938, "learning_rate": 4.849256551444879e-06, "logp/chosen": -242.0, "logp/rejected": -286.0, "loss": 0.29290771484375, "reward/accuracy": 0.875, "reward/chosen": -2.015625, "reward/margin": 1.953125, "reward/rejected": -3.96875, "step": 1700 }, { "approx. KL/chosen": 172.0, "approx. KL/rejected": 1072.0, "epoch": 0.5101979604079184, "grad_norm": 3.9351646900177, "learning_rate": 4.844547314080166e-06, "logp/chosen": -254.0, "logp/rejected": -348.0, "loss": 0.5046348571777344, "reward/accuracy": 0.6875, "reward/chosen": -1.296875, "reward/margin": 2.40625, "reward/rejected": -3.703125, "step": 1701 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 780.0, "epoch": 0.510497900419916, "grad_norm": 3.569992780685425, "learning_rate": 4.839838214743312e-06, "logp/chosen": -314.0, "logp/rejected": -304.0, "loss": 0.44036865234375, "reward/accuracy": 0.75, "reward/chosen": -1.90625, "reward/margin": 1.609375, "reward/rejected": -3.515625, "step": 1702 }, { "approx. KL/chosen": 193.0, "approx. KL/rejected": 736.0, "epoch": 0.5107978404319136, "grad_norm": 3.4530558586120605, "learning_rate": 4.835129257615569e-06, "logp/chosen": -340.0, "logp/rejected": -308.0, "loss": 0.3397064208984375, "reward/accuracy": 0.875, "reward/chosen": -1.109375, "reward/margin": 2.34375, "reward/rejected": -3.453125, "step": 1703 }, { "approx. KL/chosen": 338.0, "approx. KL/rejected": 976.0, "epoch": 0.5110977804439112, "grad_norm": 3.8156886100769043, "learning_rate": 4.830420446878066e-06, "logp/chosen": -406.0, "logp/rejected": -296.0, "loss": 0.3687744140625, "reward/accuracy": 0.8125, "reward/chosen": -1.796875, "reward/margin": 2.140625, "reward/rejected": -3.9375, "step": 1704 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 560.0, "epoch": 0.5113977204559088, "grad_norm": 5.652719020843506, "learning_rate": 4.825711786711794e-06, "logp/chosen": -406.0, "logp/rejected": -478.0, "loss": 0.77679443359375, "reward/accuracy": 0.625, "reward/chosen": -1.421875, "reward/margin": 1.25, "reward/rejected": -2.671875, "step": 1705 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 684.0, "epoch": 0.5116976604679064, "grad_norm": 4.40544319152832, "learning_rate": 4.821003281297619e-06, "logp/chosen": -192.0, "logp/rejected": -286.0, "loss": 0.62255859375, "reward/accuracy": 0.625, "reward/chosen": -1.7578125, "reward/margin": 1.1328125, "reward/rejected": -2.890625, "step": 1706 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 976.0, "epoch": 0.5119976004799041, "grad_norm": 3.8713181018829346, "learning_rate": 4.816294934816265e-06, "logp/chosen": -426.0, "logp/rejected": -300.0, "loss": 0.3055877685546875, "reward/accuracy": 0.8125, "reward/chosen": -1.734375, "reward/margin": 2.125, "reward/rejected": -3.84375, "step": 1707 }, { "approx. KL/chosen": 400.0, "approx. KL/rejected": 772.0, "epoch": 0.5122975404919016, "grad_norm": 4.435489654541016, "learning_rate": 4.811586751448315e-06, "logp/chosen": -430.0, "logp/rejected": -356.0, "loss": 0.592254638671875, "reward/accuracy": 0.8125, "reward/chosen": -1.8671875, "reward/margin": 1.5078125, "reward/rejected": -3.375, "step": 1708 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 498.0, "epoch": 0.5125974805038992, "grad_norm": 3.3836731910705566, "learning_rate": 4.806878735374212e-06, "logp/chosen": -326.0, "logp/rejected": -418.0, "loss": 0.38641357421875, "reward/accuracy": 0.8125, "reward/chosen": -1.15625, "reward/margin": 1.5703125, "reward/rejected": -2.734375, "step": 1709 }, { "approx. KL/chosen": 215.0, "approx. KL/rejected": 1064.0, "epoch": 0.5128974205158968, "grad_norm": 4.93035364151001, "learning_rate": 4.80217089077424e-06, "logp/chosen": -358.0, "logp/rejected": -388.0, "loss": 0.545562744140625, "reward/accuracy": 0.75, "reward/chosen": -1.65625, "reward/margin": 2.15625, "reward/rejected": -3.8125, "step": 1710 }, { "approx. KL/chosen": 147.0, "approx. KL/rejected": 752.0, "epoch": 0.5131973605278944, "grad_norm": 2.479581117630005, "learning_rate": 4.797463221828542e-06, "logp/chosen": -290.0, "logp/rejected": -368.0, "loss": 0.30718994140625, "reward/accuracy": 0.8125, "reward/chosen": -1.28125, "reward/margin": 2.15625, "reward/rejected": -3.4375, "step": 1711 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 310.0, "epoch": 0.5134973005398921, "grad_norm": 6.266079902648926, "learning_rate": 4.792755732717097e-06, "logp/chosen": -376.0, "logp/rejected": -296.0, "loss": 1.0386962890625, "reward/accuracy": 0.5625, "reward/chosen": -1.421875, "reward/margin": 0.41015625, "reward/rejected": -1.8359375, "step": 1712 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 1016.0, "epoch": 0.5137972405518896, "grad_norm": 4.998201847076416, "learning_rate": 4.78804842761973e-06, "logp/chosen": -338.0, "logp/rejected": -392.0, "loss": 0.5321044921875, "reward/accuracy": 0.75, "reward/chosen": -2.078125, "reward/margin": 1.6484375, "reward/rejected": -3.71875, "step": 1713 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 592.0, "epoch": 0.5140971805638872, "grad_norm": 4.814817428588867, "learning_rate": 4.7833413107161026e-06, "logp/chosen": -348.0, "logp/rejected": -270.0, "loss": 0.4368896484375, "reward/accuracy": 0.75, "reward/chosen": -1.4375, "reward/margin": 1.5, "reward/rejected": -2.9375, "step": 1714 }, { "approx. KL/chosen": 332.0, "approx. KL/rejected": 728.0, "epoch": 0.5143971205758848, "grad_norm": 7.010258197784424, "learning_rate": 4.7786343861857e-06, "logp/chosen": -332.0, "logp/rejected": -384.0, "loss": 0.35516357421875, "reward/accuracy": 0.8125, "reward/chosen": -2.046875, "reward/margin": 1.4296875, "reward/rejected": -3.484375, "step": 1715 }, { "approx. KL/chosen": 103.5, "approx. KL/rejected": 576.0, "epoch": 0.5146970605878824, "grad_norm": 3.1764228343963623, "learning_rate": 4.773927658207849e-06, "logp/chosen": -420.0, "logp/rejected": -372.0, "loss": 0.31982421875, "reward/accuracy": 0.875, "reward/chosen": -0.76171875, "reward/margin": 2.234375, "reward/rejected": -2.984375, "step": 1716 }, { "approx. KL/chosen": 233.0, "approx. KL/rejected": 716.0, "epoch": 0.5149970005998801, "grad_norm": 2.308499574661255, "learning_rate": 4.7692211309616935e-06, "logp/chosen": -376.0, "logp/rejected": -318.0, "loss": 0.26666259765625, "reward/accuracy": 0.9375, "reward/chosen": -1.6875, "reward/margin": 1.640625, "reward/rejected": -3.328125, "step": 1717 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 724.0, "epoch": 0.5152969406118776, "grad_norm": 4.224826335906982, "learning_rate": 4.764514808626203e-06, "logp/chosen": -384.0, "logp/rejected": -370.0, "loss": 0.493408203125, "reward/accuracy": 0.6875, "reward/chosen": -1.2734375, "reward/margin": 1.5859375, "reward/rejected": -2.859375, "step": 1718 }, { "approx. KL/chosen": 124.5, "approx. KL/rejected": 532.0, "epoch": 0.5155968806238752, "grad_norm": 3.147601366043091, "learning_rate": 4.759808695380167e-06, "logp/chosen": -258.0, "logp/rejected": -344.0, "loss": 0.34063720703125, "reward/accuracy": 0.8125, "reward/chosen": -1.046875, "reward/margin": 1.6953125, "reward/rejected": -2.75, "step": 1719 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 560.0, "epoch": 0.5158968206358728, "grad_norm": 5.8040242195129395, "learning_rate": 4.75510279540218e-06, "logp/chosen": -410.0, "logp/rejected": -282.0, "loss": 0.59197998046875, "reward/accuracy": 0.875, "reward/chosen": -1.53125, "reward/margin": 1.375, "reward/rejected": -2.90625, "step": 1720 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 712.0, "epoch": 0.5161967606478705, "grad_norm": 5.601198673248291, "learning_rate": 4.750397112870659e-06, "logp/chosen": -328.0, "logp/rejected": -266.0, "loss": 0.4614715576171875, "reward/accuracy": 0.8125, "reward/chosen": -1.390625, "reward/margin": 1.71875, "reward/rejected": -3.109375, "step": 1721 }, { "approx. KL/chosen": 90.5, "approx. KL/rejected": 756.0, "epoch": 0.5164967006598681, "grad_norm": 3.0163662433624268, "learning_rate": 4.74569165196382e-06, "logp/chosen": -376.0, "logp/rejected": -402.0, "loss": 0.3122138977050781, "reward/accuracy": 0.875, "reward/chosen": -0.96875, "reward/margin": 2.359375, "reward/rejected": -3.328125, "step": 1722 }, { "approx. KL/chosen": 127.5, "approx. KL/rejected": 500.0, "epoch": 0.5167966406718656, "grad_norm": 4.13128137588501, "learning_rate": 4.740986416859687e-06, "logp/chosen": -354.0, "logp/rejected": -346.0, "loss": 0.591094970703125, "reward/accuracy": 0.6875, "reward/chosen": -1.1328125, "reward/margin": 1.4453125, "reward/rejected": -2.578125, "step": 1723 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 596.0, "epoch": 0.5170965806838632, "grad_norm": 3.875422954559326, "learning_rate": 4.736281411736079e-06, "logp/chosen": -332.0, "logp/rejected": -314.0, "loss": 0.36968994140625, "reward/accuracy": 0.75, "reward/chosen": -1.5, "reward/margin": 1.53125, "reward/rejected": -3.03125, "step": 1724 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 628.0, "epoch": 0.5173965206958608, "grad_norm": 2.8016622066497803, "learning_rate": 4.731576640770612e-06, "logp/chosen": -320.0, "logp/rejected": -270.0, "loss": 0.2898101806640625, "reward/accuracy": 0.875, "reward/chosen": -1.1875, "reward/margin": 2.046875, "reward/rejected": -3.234375, "step": 1725 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 692.0, "epoch": 0.5176964607078585, "grad_norm": 3.7333617210388184, "learning_rate": 4.726872108140697e-06, "logp/chosen": -304.0, "logp/rejected": -308.0, "loss": 0.57421875, "reward/accuracy": 0.625, "reward/chosen": -1.3671875, "reward/margin": 1.5859375, "reward/rejected": -2.953125, "step": 1726 }, { "approx. KL/chosen": 209.0, "approx. KL/rejected": 724.0, "epoch": 0.517996400719856, "grad_norm": 4.40104341506958, "learning_rate": 4.722167818023531e-06, "logp/chosen": -444.0, "logp/rejected": -472.0, "loss": 0.580078125, "reward/accuracy": 0.6875, "reward/chosen": -1.59375, "reward/margin": 1.7109375, "reward/rejected": -3.3125, "step": 1727 }, { "approx. KL/chosen": 255.0, "approx. KL/rejected": 804.0, "epoch": 0.5182963407318536, "grad_norm": 2.833852529525757, "learning_rate": 4.717463774596099e-06, "logp/chosen": -370.0, "logp/rejected": -338.0, "loss": 0.29962158203125, "reward/accuracy": 0.9375, "reward/chosen": -1.59375, "reward/margin": 1.984375, "reward/rejected": -3.578125, "step": 1728 }, { "approx. KL/chosen": 290.0, "approx. KL/rejected": 316.0, "epoch": 0.5185962807438512, "grad_norm": 8.738818168640137, "learning_rate": 4.7127599820351585e-06, "logp/chosen": -368.0, "logp/rejected": -326.0, "loss": 0.828369140625, "reward/accuracy": 0.8125, "reward/chosen": -1.546875, "reward/margin": 0.392578125, "reward/rejected": -1.9375, "step": 1729 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 960.0, "epoch": 0.5188962207558488, "grad_norm": 2.271249532699585, "learning_rate": 4.708056444517253e-06, "logp/chosen": -484.0, "logp/rejected": -398.0, "loss": 0.20452880859375, "reward/accuracy": 1.0, "reward/chosen": -1.0546875, "reward/margin": 2.4375, "reward/rejected": -3.5, "step": 1730 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 410.0, "epoch": 0.5191961607678465, "grad_norm": 5.5179595947265625, "learning_rate": 4.7033531662186974e-06, "logp/chosen": -300.0, "logp/rejected": -296.0, "loss": 0.7021484375, "reward/accuracy": 0.6875, "reward/chosen": -1.4296875, "reward/margin": 1.0703125, "reward/rejected": -2.5, "step": 1731 }, { "approx. KL/chosen": 84.0, "approx. KL/rejected": 648.0, "epoch": 0.519496100779844, "grad_norm": 3.8078811168670654, "learning_rate": 4.6986501513155745e-06, "logp/chosen": -336.0, "logp/rejected": -342.0, "loss": 0.3676605224609375, "reward/accuracy": 0.8125, "reward/chosen": -0.625, "reward/margin": 2.375, "reward/rejected": -3.0, "step": 1732 }, { "approx. KL/chosen": 244.0, "approx. KL/rejected": 832.0, "epoch": 0.5197960407918416, "grad_norm": 4.448182106018066, "learning_rate": 4.693947403983733e-06, "logp/chosen": -290.0, "logp/rejected": -322.0, "loss": 0.374908447265625, "reward/accuracy": 0.875, "reward/chosen": -1.453125, "reward/margin": 2.28125, "reward/rejected": -3.734375, "step": 1733 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 588.0, "epoch": 0.5200959808038392, "grad_norm": 2.925234317779541, "learning_rate": 4.689244928398787e-06, "logp/chosen": -376.0, "logp/rejected": -292.0, "loss": 0.32904052734375, "reward/accuracy": 0.9375, "reward/chosen": -1.109375, "reward/margin": 1.9609375, "reward/rejected": -3.0625, "step": 1734 }, { "approx. KL/chosen": 63.25, "approx. KL/rejected": 648.0, "epoch": 0.5203959208158369, "grad_norm": 2.5292556285858154, "learning_rate": 4.684542728736105e-06, "logp/chosen": -458.0, "logp/rejected": -298.0, "loss": 0.14892578125, "reward/accuracy": 1.0, "reward/chosen": -0.33203125, "reward/margin": 2.703125, "reward/rejected": -3.03125, "step": 1735 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 640.0, "epoch": 0.5206958608278345, "grad_norm": 4.419198989868164, "learning_rate": 4.679840809170815e-06, "logp/chosen": -356.0, "logp/rejected": -352.0, "loss": 0.5220947265625, "reward/accuracy": 0.75, "reward/chosen": -1.609375, "reward/margin": 1.359375, "reward/rejected": -2.96875, "step": 1736 }, { "approx. KL/chosen": 132.0, "approx. KL/rejected": 696.0, "epoch": 0.520995800839832, "grad_norm": 2.9425106048583984, "learning_rate": 4.6751391738777955e-06, "logp/chosen": -282.0, "logp/rejected": -370.0, "loss": 0.2225341796875, "reward/accuracy": 0.875, "reward/chosen": -1.1484375, "reward/margin": 2.34375, "reward/rejected": -3.484375, "step": 1737 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 516.0, "epoch": 0.5212957408518296, "grad_norm": 4.676953315734863, "learning_rate": 4.67043782703167e-06, "logp/chosen": -268.0, "logp/rejected": -340.0, "loss": 0.545623779296875, "reward/accuracy": 0.8125, "reward/chosen": -1.453125, "reward/margin": 1.203125, "reward/rejected": -2.65625, "step": 1738 }, { "approx. KL/chosen": 338.0, "approx. KL/rejected": 640.0, "epoch": 0.5215956808638272, "grad_norm": 4.730429649353027, "learning_rate": 4.665736772806806e-06, "logp/chosen": -410.0, "logp/rejected": -394.0, "loss": 0.564453125, "reward/accuracy": 0.625, "reward/chosen": -2.0, "reward/margin": 1.1171875, "reward/rejected": -3.125, "step": 1739 }, { "approx. KL/chosen": 500.0, "approx. KL/rejected": 700.0, "epoch": 0.5218956208758249, "grad_norm": 5.931525230407715, "learning_rate": 4.661036015377313e-06, "logp/chosen": -438.0, "logp/rejected": -420.0, "loss": 0.67852783203125, "reward/accuracy": 0.5625, "reward/chosen": -2.015625, "reward/margin": 0.875, "reward/rejected": -2.890625, "step": 1740 }, { "approx. KL/chosen": 109.5, "approx. KL/rejected": 390.0, "epoch": 0.5221955608878225, "grad_norm": 3.318300485610962, "learning_rate": 4.656335558917038e-06, "logp/chosen": -350.0, "logp/rejected": -310.0, "loss": 0.2956695556640625, "reward/accuracy": 0.9375, "reward/chosen": -0.84375, "reward/margin": 1.640625, "reward/rejected": -2.484375, "step": 1741 }, { "approx. KL/chosen": 93.0, "approx. KL/rejected": 490.0, "epoch": 0.52249550089982, "grad_norm": 4.306155204772949, "learning_rate": 4.651635407599559e-06, "logp/chosen": -344.0, "logp/rejected": -298.0, "loss": 0.5102386474609375, "reward/accuracy": 0.625, "reward/chosen": -0.765625, "reward/margin": 1.6875, "reward/rejected": -2.453125, "step": 1742 }, { "approx. KL/chosen": 145.0, "approx. KL/rejected": 438.0, "epoch": 0.5227954409118176, "grad_norm": 3.7860777378082275, "learning_rate": 4.646935565598182e-06, "logp/chosen": -394.0, "logp/rejected": -360.0, "loss": 0.518707275390625, "reward/accuracy": 0.75, "reward/chosen": -1.2890625, "reward/margin": 1.1953125, "reward/rejected": -2.484375, "step": 1743 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 544.0, "epoch": 0.5230953809238152, "grad_norm": 5.01774263381958, "learning_rate": 4.6422360370859395e-06, "logp/chosen": -372.0, "logp/rejected": -312.0, "loss": 0.5921630859375, "reward/accuracy": 0.75, "reward/chosen": -1.390625, "reward/margin": 1.3984375, "reward/rejected": -2.78125, "step": 1744 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 568.0, "epoch": 0.5233953209358129, "grad_norm": 2.9444656372070312, "learning_rate": 4.637536826235587e-06, "logp/chosen": -274.0, "logp/rejected": -328.0, "loss": 0.451995849609375, "reward/accuracy": 0.6875, "reward/chosen": -1.125, "reward/margin": 1.6015625, "reward/rejected": -2.734375, "step": 1745 }, { "approx. KL/chosen": 141.0, "approx. KL/rejected": 660.0, "epoch": 0.5236952609478105, "grad_norm": 2.7214109897613525, "learning_rate": 4.632837937219597e-06, "logp/chosen": -358.0, "logp/rejected": -372.0, "loss": 0.3151206970214844, "reward/accuracy": 0.8125, "reward/chosen": -1.2734375, "reward/margin": 2.03125, "reward/rejected": -3.3125, "step": 1746 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 452.0, "epoch": 0.523995200959808, "grad_norm": 5.534132957458496, "learning_rate": 4.628139374210154e-06, "logp/chosen": -290.0, "logp/rejected": -306.0, "loss": 0.5965576171875, "reward/accuracy": 0.6875, "reward/chosen": -1.3203125, "reward/margin": 1.0390625, "reward/rejected": -2.359375, "step": 1747 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 253.0, "epoch": 0.5242951409718056, "grad_norm": 5.789516448974609, "learning_rate": 4.623441141379157e-06, "logp/chosen": -354.0, "logp/rejected": -386.0, "loss": 0.87646484375, "reward/accuracy": 0.5, "reward/chosen": -1.5234375, "reward/margin": 0.09375, "reward/rejected": -1.6171875, "step": 1748 }, { "approx. KL/chosen": 123.0, "approx. KL/rejected": 564.0, "epoch": 0.5245950809838033, "grad_norm": 2.872307538986206, "learning_rate": 4.6187432428982084e-06, "logp/chosen": -284.0, "logp/rejected": -354.0, "loss": 0.322021484375, "reward/accuracy": 0.8125, "reward/chosen": -1.1328125, "reward/margin": 1.7890625, "reward/rejected": -2.921875, "step": 1749 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 478.0, "epoch": 0.5248950209958009, "grad_norm": 3.451046943664551, "learning_rate": 4.6140456829386155e-06, "logp/chosen": -356.0, "logp/rejected": -312.0, "loss": 0.3997802734375, "reward/accuracy": 0.75, "reward/chosen": -1.390625, "reward/margin": 1.390625, "reward/rejected": -2.78125, "step": 1750 }, { "approx. KL/chosen": 75.5, "approx. KL/rejected": 444.0, "epoch": 0.5251949610077984, "grad_norm": 2.391798257827759, "learning_rate": 4.609348465671386e-06, "logp/chosen": -328.0, "logp/rejected": -338.0, "loss": 0.2708740234375, "reward/accuracy": 0.875, "reward/chosen": -0.34375, "reward/margin": 1.8984375, "reward/rejected": -2.25, "step": 1751 }, { "approx. KL/chosen": 30.75, "approx. KL/rejected": 368.0, "epoch": 0.525494901019796, "grad_norm": 2.4735517501831055, "learning_rate": 4.604651595267221e-06, "logp/chosen": -310.0, "logp/rejected": -338.0, "loss": 0.2286376953125, "reward/accuracy": 0.9375, "reward/chosen": -0.439453125, "reward/margin": 1.9375, "reward/rejected": -2.375, "step": 1752 }, { "approx. KL/chosen": 68.5, "approx. KL/rejected": 800.0, "epoch": 0.5257948410317936, "grad_norm": 2.663789749145508, "learning_rate": 4.599955075896513e-06, "logp/chosen": -312.0, "logp/rejected": -280.0, "loss": 0.26270294189453125, "reward/accuracy": 0.875, "reward/chosen": -0.66015625, "reward/margin": 2.65625, "reward/rejected": -3.3125, "step": 1753 }, { "approx. KL/chosen": 56.25, "approx. KL/rejected": 556.0, "epoch": 0.5260947810437913, "grad_norm": 3.010777235031128, "learning_rate": 4.5952589117293475e-06, "logp/chosen": -235.0, "logp/rejected": -270.0, "loss": 0.2884521484375, "reward/accuracy": 0.875, "reward/chosen": -0.3671875, "reward/margin": 2.21875, "reward/rejected": -2.59375, "step": 1754 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 472.0, "epoch": 0.5263947210557889, "grad_norm": 3.9768831729888916, "learning_rate": 4.59056310693549e-06, "logp/chosen": -336.0, "logp/rejected": -338.0, "loss": 0.438018798828125, "reward/accuracy": 0.75, "reward/chosen": -1.140625, "reward/margin": 1.171875, "reward/rejected": -2.3125, "step": 1755 }, { "approx. KL/chosen": 120.0, "approx. KL/rejected": 398.0, "epoch": 0.5266946610677864, "grad_norm": 3.5967230796813965, "learning_rate": 4.585867665684387e-06, "logp/chosen": -412.0, "logp/rejected": -396.0, "loss": 0.3483123779296875, "reward/accuracy": 0.875, "reward/chosen": -0.703125, "reward/margin": 1.7265625, "reward/rejected": -2.4375, "step": 1756 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 612.0, "epoch": 0.526994601079784, "grad_norm": 4.13186502456665, "learning_rate": 4.581172592145167e-06, "logp/chosen": -296.0, "logp/rejected": -370.0, "loss": 0.44708251953125, "reward/accuracy": 0.875, "reward/chosen": -1.515625, "reward/margin": 1.234375, "reward/rejected": -2.75, "step": 1757 }, { "approx. KL/chosen": 73.5, "approx. KL/rejected": 552.0, "epoch": 0.5272945410917816, "grad_norm": 2.47357439994812, "learning_rate": 4.576477890486626e-06, "logp/chosen": -344.0, "logp/rejected": -380.0, "loss": 0.263916015625, "reward/accuracy": 0.875, "reward/chosen": -0.95703125, "reward/margin": 1.890625, "reward/rejected": -2.859375, "step": 1758 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 362.0, "epoch": 0.5275944811037793, "grad_norm": 7.1675848960876465, "learning_rate": 4.571783564877232e-06, "logp/chosen": -199.0, "logp/rejected": -238.0, "loss": 0.75433349609375, "reward/accuracy": 0.625, "reward/chosen": -1.5859375, "reward/margin": 0.5703125, "reward/rejected": -2.15625, "step": 1759 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 506.0, "epoch": 0.5278944211157769, "grad_norm": 5.064825057983398, "learning_rate": 4.567089619485123e-06, "logp/chosen": -342.0, "logp/rejected": -242.0, "loss": 0.682373046875, "reward/accuracy": 0.625, "reward/chosen": -1.2890625, "reward/margin": 1.109375, "reward/rejected": -2.390625, "step": 1760 }, { "approx. KL/chosen": 125.0, "approx. KL/rejected": 936.0, "epoch": 0.5281943611277744, "grad_norm": 2.0186614990234375, "learning_rate": 4.562396058478093e-06, "logp/chosen": -382.0, "logp/rejected": -342.0, "loss": 0.170318603515625, "reward/accuracy": 0.9375, "reward/chosen": -0.8984375, "reward/margin": 2.890625, "reward/rejected": -3.78125, "step": 1761 }, { "approx. KL/chosen": 129.0, "approx. KL/rejected": 580.0, "epoch": 0.528494301139772, "grad_norm": 3.381676435470581, "learning_rate": 4.557702886023599e-06, "logp/chosen": -314.0, "logp/rejected": -302.0, "loss": 0.4159698486328125, "reward/accuracy": 0.8125, "reward/chosen": -0.875, "reward/margin": 1.796875, "reward/rejected": -2.671875, "step": 1762 }, { "approx. KL/chosen": 348.0, "approx. KL/rejected": 1296.0, "epoch": 0.5287942411517697, "grad_norm": 3.1435937881469727, "learning_rate": 4.5530101062887505e-06, "logp/chosen": -284.0, "logp/rejected": -364.0, "loss": 0.3997154235839844, "reward/accuracy": 0.8125, "reward/chosen": -1.6171875, "reward/margin": 2.6875, "reward/rejected": -4.3125, "step": 1763 }, { "approx. KL/chosen": 106.0, "approx. KL/rejected": 524.0, "epoch": 0.5290941811637673, "grad_norm": 3.511538028717041, "learning_rate": 4.548317723440312e-06, "logp/chosen": -268.0, "logp/rejected": -253.0, "loss": 0.4296875, "reward/accuracy": 0.6875, "reward/chosen": -1.109375, "reward/margin": 1.4765625, "reward/rejected": -2.59375, "step": 1764 }, { "approx. KL/chosen": 142.0, "approx. KL/rejected": 1012.0, "epoch": 0.5293941211757649, "grad_norm": 2.6187334060668945, "learning_rate": 4.543625741644688e-06, "logp/chosen": -334.0, "logp/rejected": -348.0, "loss": 0.19986534118652344, "reward/accuracy": 0.9375, "reward/chosen": -1.0625, "reward/margin": 2.875, "reward/rejected": -3.953125, "step": 1765 }, { "approx. KL/chosen": 99.5, "approx. KL/rejected": 386.0, "epoch": 0.5296940611877624, "grad_norm": 2.474349021911621, "learning_rate": 4.538934165067937e-06, "logp/chosen": -492.0, "logp/rejected": -382.0, "loss": 0.23577880859375, "reward/accuracy": 0.9375, "reward/chosen": -0.66015625, "reward/margin": 1.828125, "reward/rejected": -2.484375, "step": 1766 }, { "approx. KL/chosen": 99.5, "approx. KL/rejected": 552.0, "epoch": 0.52999400119976, "grad_norm": 3.6425657272338867, "learning_rate": 4.534242997875752e-06, "logp/chosen": -406.0, "logp/rejected": -320.0, "loss": 0.39148712158203125, "reward/accuracy": 0.8125, "reward/chosen": -0.8046875, "reward/margin": 1.84375, "reward/rejected": -2.640625, "step": 1767 }, { "approx. KL/chosen": 292.0, "approx. KL/rejected": 580.0, "epoch": 0.5302939412117577, "grad_norm": 7.002904891967773, "learning_rate": 4.52955224423346e-06, "logp/chosen": -306.0, "logp/rejected": -296.0, "loss": 0.819305419921875, "reward/accuracy": 0.5, "reward/chosen": -1.7734375, "reward/margin": 1.078125, "reward/rejected": -2.859375, "step": 1768 }, { "approx. KL/chosen": 310.0, "approx. KL/rejected": 932.0, "epoch": 0.5305938812237553, "grad_norm": 4.321435451507568, "learning_rate": 4.524861908306028e-06, "logp/chosen": -334.0, "logp/rejected": -378.0, "loss": 0.51123046875, "reward/accuracy": 0.75, "reward/chosen": -1.8515625, "reward/margin": 1.8671875, "reward/rejected": -3.71875, "step": 1769 }, { "approx. KL/chosen": 344.0, "approx. KL/rejected": 904.0, "epoch": 0.5308938212357528, "grad_norm": 3.784604549407959, "learning_rate": 4.5201719942580446e-06, "logp/chosen": -298.0, "logp/rejected": -312.0, "loss": 0.309814453125, "reward/accuracy": 0.8125, "reward/chosen": -2.0, "reward/margin": 1.546875, "reward/rejected": -3.546875, "step": 1770 }, { "approx. KL/chosen": 97.0, "approx. KL/rejected": 700.0, "epoch": 0.5311937612477504, "grad_norm": 3.1849029064178467, "learning_rate": 4.5154825062537305e-06, "logp/chosen": -304.0, "logp/rejected": -266.0, "loss": 0.3916473388671875, "reward/accuracy": 0.8125, "reward/chosen": -0.97265625, "reward/margin": 2.171875, "reward/rejected": -3.140625, "step": 1771 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 840.0, "epoch": 0.531493701259748, "grad_norm": 4.033719062805176, "learning_rate": 4.510793448456922e-06, "logp/chosen": -306.0, "logp/rejected": -270.0, "loss": 0.353515625, "reward/accuracy": 0.8125, "reward/chosen": -1.921875, "reward/margin": 1.8203125, "reward/rejected": -3.734375, "step": 1772 }, { "approx. KL/chosen": 133.0, "approx. KL/rejected": 748.0, "epoch": 0.5317936412717457, "grad_norm": 1.4765503406524658, "learning_rate": 4.5061048250310785e-06, "logp/chosen": -418.0, "logp/rejected": -366.0, "loss": 0.122772216796875, "reward/accuracy": 1.0, "reward/chosen": -0.859375, "reward/margin": 2.6875, "reward/rejected": -3.546875, "step": 1773 }, { "approx. KL/chosen": 504.0, "approx. KL/rejected": 1392.0, "epoch": 0.5320935812837433, "grad_norm": 8.5331449508667, "learning_rate": 4.50141664013927e-06, "logp/chosen": -540.0, "logp/rejected": -438.0, "loss": 0.40019989013671875, "reward/accuracy": 0.8125, "reward/chosen": -2.703125, "reward/margin": 1.890625, "reward/rejected": -4.59375, "step": 1774 }, { "approx. KL/chosen": 424.0, "approx. KL/rejected": 1192.0, "epoch": 0.5323935212957408, "grad_norm": 7.096932888031006, "learning_rate": 4.496728897944179e-06, "logp/chosen": -300.0, "logp/rejected": -244.0, "loss": 0.650177001953125, "reward/accuracy": 0.75, "reward/chosen": -2.578125, "reward/margin": 1.609375, "reward/rejected": -4.1875, "step": 1775 }, { "approx. KL/chosen": 235.0, "approx. KL/rejected": 816.0, "epoch": 0.5326934613077384, "grad_norm": 3.792236804962158, "learning_rate": 4.492041602608096e-06, "logp/chosen": -332.0, "logp/rejected": -334.0, "loss": 0.476531982421875, "reward/accuracy": 0.8125, "reward/chosen": -1.6484375, "reward/margin": 1.7265625, "reward/rejected": -3.375, "step": 1776 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 520.0, "epoch": 0.532993401319736, "grad_norm": 3.7627036571502686, "learning_rate": 4.4873547582929125e-06, "logp/chosen": -356.0, "logp/rejected": -362.0, "loss": 0.601806640625, "reward/accuracy": 0.8125, "reward/chosen": -1.8359375, "reward/margin": 1.0625, "reward/rejected": -2.890625, "step": 1777 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 916.0, "epoch": 0.5332933413317337, "grad_norm": 3.6228573322296143, "learning_rate": 4.48266836916012e-06, "logp/chosen": -298.0, "logp/rejected": -352.0, "loss": 0.44427490234375, "reward/accuracy": 0.8125, "reward/chosen": -1.9609375, "reward/margin": 1.6953125, "reward/rejected": -3.65625, "step": 1778 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 436.0, "epoch": 0.5335932813437313, "grad_norm": 4.430014610290527, "learning_rate": 4.477982439370805e-06, "logp/chosen": -290.0, "logp/rejected": -296.0, "loss": 0.61492919921875, "reward/accuracy": 0.5625, "reward/chosen": -1.390625, "reward/margin": 0.9375, "reward/rejected": -2.328125, "step": 1779 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 1080.0, "epoch": 0.5338932213557288, "grad_norm": 2.792292594909668, "learning_rate": 4.473296973085649e-06, "logp/chosen": -254.0, "logp/rejected": -255.0, "loss": 0.277008056640625, "reward/accuracy": 0.75, "reward/chosen": -1.953125, "reward/margin": 2.28125, "reward/rejected": -4.21875, "step": 1780 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 868.0, "epoch": 0.5341931613677264, "grad_norm": 3.8782055377960205, "learning_rate": 4.468611974464922e-06, "logp/chosen": -298.0, "logp/rejected": -290.0, "loss": 0.4356985092163086, "reward/accuracy": 0.6875, "reward/chosen": -1.5546875, "reward/margin": 2.0, "reward/rejected": -3.546875, "step": 1781 }, { "approx. KL/chosen": 386.0, "approx. KL/rejected": 932.0, "epoch": 0.5344931013797241, "grad_norm": 3.9572768211364746, "learning_rate": 4.463927447668475e-06, "logp/chosen": -414.0, "logp/rejected": -378.0, "loss": 0.4094390869140625, "reward/accuracy": 0.6875, "reward/chosen": -1.8984375, "reward/margin": 2.0625, "reward/rejected": -3.953125, "step": 1782 }, { "approx. KL/chosen": 432.0, "approx. KL/rejected": 1368.0, "epoch": 0.5347930413917217, "grad_norm": 4.855175971984863, "learning_rate": 4.45924339685574e-06, "logp/chosen": -344.0, "logp/rejected": -376.0, "loss": 0.57513427734375, "reward/accuracy": 0.8125, "reward/chosen": -2.203125, "reward/margin": 2.265625, "reward/rejected": -4.46875, "step": 1783 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 632.0, "epoch": 0.5350929814037193, "grad_norm": 3.475890874862671, "learning_rate": 4.454559826185732e-06, "logp/chosen": -338.0, "logp/rejected": -318.0, "loss": 0.4449462890625, "reward/accuracy": 0.8125, "reward/chosen": -1.4921875, "reward/margin": 1.578125, "reward/rejected": -3.0625, "step": 1784 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 720.0, "epoch": 0.5353929214157168, "grad_norm": 5.660536289215088, "learning_rate": 4.4498767398170335e-06, "logp/chosen": -350.0, "logp/rejected": -334.0, "loss": 0.5369911193847656, "reward/accuracy": 0.6875, "reward/chosen": -1.9765625, "reward/margin": 1.421875, "reward/rejected": -3.390625, "step": 1785 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 1184.0, "epoch": 0.5356928614277144, "grad_norm": 2.1149842739105225, "learning_rate": 4.445194141907803e-06, "logp/chosen": -378.0, "logp/rejected": -284.0, "loss": 0.215606689453125, "reward/accuracy": 0.9375, "reward/chosen": -1.640625, "reward/margin": 2.71875, "reward/rejected": -4.34375, "step": 1786 }, { "approx. KL/chosen": 458.0, "approx. KL/rejected": 1232.0, "epoch": 0.5359928014397121, "grad_norm": 3.5342659950256348, "learning_rate": 4.44051203661576e-06, "logp/chosen": -255.0, "logp/rejected": -328.0, "loss": 0.40323638916015625, "reward/accuracy": 0.8125, "reward/chosen": -2.421875, "reward/margin": 2.046875, "reward/rejected": -4.46875, "step": 1787 }, { "approx. KL/chosen": 632.0, "approx. KL/rejected": 1176.0, "epoch": 0.5362927414517097, "grad_norm": 4.275646686553955, "learning_rate": 4.435830428098187e-06, "logp/chosen": -450.0, "logp/rejected": -324.0, "loss": 0.4244384765625, "reward/accuracy": 0.75, "reward/chosen": -3.046875, "reward/margin": 1.453125, "reward/rejected": -4.5, "step": 1788 }, { "approx. KL/chosen": 484.0, "approx. KL/rejected": 880.0, "epoch": 0.5365926814637073, "grad_norm": 4.102119445800781, "learning_rate": 4.431149320511929e-06, "logp/chosen": -460.0, "logp/rejected": -318.0, "loss": 0.654144287109375, "reward/accuracy": 0.75, "reward/chosen": -2.78125, "reward/margin": 0.97265625, "reward/rejected": -3.75, "step": 1789 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 732.0, "epoch": 0.5368926214757048, "grad_norm": 4.639474868774414, "learning_rate": 4.426468718013383e-06, "logp/chosen": -474.0, "logp/rejected": -396.0, "loss": 0.6329917907714844, "reward/accuracy": 0.75, "reward/chosen": -1.859375, "reward/margin": 1.4765625, "reward/rejected": -3.34375, "step": 1790 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 576.0, "epoch": 0.5371925614877024, "grad_norm": 3.993151903152466, "learning_rate": 4.421788624758502e-06, "logp/chosen": -296.0, "logp/rejected": -268.0, "loss": 0.505859375, "reward/accuracy": 0.75, "reward/chosen": -1.9296875, "reward/margin": 1.0, "reward/rejected": -2.9375, "step": 1791 }, { "approx. KL/chosen": 338.0, "approx. KL/rejected": 1000.0, "epoch": 0.5374925014997001, "grad_norm": 4.646515369415283, "learning_rate": 4.4171090449027785e-06, "logp/chosen": -418.0, "logp/rejected": -382.0, "loss": 0.424713134765625, "reward/accuracy": 0.75, "reward/chosen": -2.203125, "reward/margin": 1.9375, "reward/rejected": -4.15625, "step": 1792 }, { "approx. KL/chosen": 374.0, "approx. KL/rejected": 888.0, "epoch": 0.5377924415116977, "grad_norm": 4.2765021324157715, "learning_rate": 4.412429982601255e-06, "logp/chosen": -414.0, "logp/rejected": -422.0, "loss": 0.4951324462890625, "reward/accuracy": 0.8125, "reward/chosen": -2.4375, "reward/margin": 1.4140625, "reward/rejected": -3.84375, "step": 1793 }, { "approx. KL/chosen": 442.0, "approx. KL/rejected": 1032.0, "epoch": 0.5380923815236952, "grad_norm": 4.008137226104736, "learning_rate": 4.407751442008518e-06, "logp/chosen": -476.0, "logp/rejected": -484.0, "loss": 0.3812255859375, "reward/accuracy": 0.75, "reward/chosen": -2.546875, "reward/margin": 1.5234375, "reward/rejected": -4.0625, "step": 1794 }, { "approx. KL/chosen": 252.0, "approx. KL/rejected": 1696.0, "epoch": 0.5383923215356928, "grad_norm": 2.3877155780792236, "learning_rate": 4.403073427278682e-06, "logp/chosen": -370.0, "logp/rejected": -424.0, "loss": 0.22772216796875, "reward/accuracy": 0.9375, "reward/chosen": -1.96875, "reward/margin": 3.34375, "reward/rejected": -5.3125, "step": 1795 }, { "approx. KL/chosen": 510.0, "approx. KL/rejected": 952.0, "epoch": 0.5386922615476905, "grad_norm": 4.626900672912598, "learning_rate": 4.398395942565402e-06, "logp/chosen": -374.0, "logp/rejected": -372.0, "loss": 0.678253173828125, "reward/accuracy": 0.625, "reward/chosen": -2.75, "reward/margin": 1.0546875, "reward/rejected": -3.8125, "step": 1796 }, { "approx. KL/chosen": 392.0, "approx. KL/rejected": 1232.0, "epoch": 0.5389922015596881, "grad_norm": 5.365029811859131, "learning_rate": 4.393718992021857e-06, "logp/chosen": -324.0, "logp/rejected": -432.0, "loss": 0.54705810546875, "reward/accuracy": 0.6875, "reward/chosen": -2.140625, "reward/margin": 1.9765625, "reward/rejected": -4.125, "step": 1797 }, { "approx. KL/chosen": 440.0, "approx. KL/rejected": 1020.0, "epoch": 0.5392921415716857, "grad_norm": 4.936470031738281, "learning_rate": 4.389042579800755e-06, "logp/chosen": -308.0, "logp/rejected": -394.0, "loss": 0.424346923828125, "reward/accuracy": 0.8125, "reward/chosen": -2.421875, "reward/margin": 1.6328125, "reward/rejected": -4.0625, "step": 1798 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 1048.0, "epoch": 0.5395920815836832, "grad_norm": 4.087449073791504, "learning_rate": 4.384366710054325e-06, "logp/chosen": -406.0, "logp/rejected": -384.0, "loss": 0.37827301025390625, "reward/accuracy": 0.8125, "reward/chosen": -2.03125, "reward/margin": 1.96875, "reward/rejected": -4.0, "step": 1799 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 1012.0, "epoch": 0.5398920215956808, "grad_norm": 1.7720141410827637, "learning_rate": 4.379691386934317e-06, "logp/chosen": -374.0, "logp/rejected": -290.0, "loss": 0.1927490234375, "reward/accuracy": 0.9375, "reward/chosen": -1.78125, "reward/margin": 2.390625, "reward/rejected": -4.15625, "step": 1800 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 828.0, "epoch": 0.5401919616076785, "grad_norm": 3.342766523361206, "learning_rate": 4.375016614591988e-06, "logp/chosen": -394.0, "logp/rejected": -412.0, "loss": 0.33380126953125, "reward/accuracy": 0.875, "reward/chosen": -1.71875, "reward/margin": 1.9921875, "reward/rejected": -3.703125, "step": 1801 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 740.0, "epoch": 0.5404919016196761, "grad_norm": 3.7504706382751465, "learning_rate": 4.370342397178113e-06, "logp/chosen": -422.0, "logp/rejected": -338.0, "loss": 0.478424072265625, "reward/accuracy": 0.75, "reward/chosen": -1.9453125, "reward/margin": 1.4375, "reward/rejected": -3.390625, "step": 1802 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 788.0, "epoch": 0.5407918416316737, "grad_norm": 3.8305201530456543, "learning_rate": 4.365668738842975e-06, "logp/chosen": -410.0, "logp/rejected": -332.0, "loss": 0.367034912109375, "reward/accuracy": 0.875, "reward/chosen": -1.6953125, "reward/margin": 1.8515625, "reward/rejected": -3.546875, "step": 1803 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 1192.0, "epoch": 0.5410917816436712, "grad_norm": 2.96785306930542, "learning_rate": 4.360995643736354e-06, "logp/chosen": -386.0, "logp/rejected": -332.0, "loss": 0.3477897644042969, "reward/accuracy": 0.75, "reward/chosen": -1.6875, "reward/margin": 2.65625, "reward/rejected": -4.34375, "step": 1804 }, { "approx. KL/chosen": 254.0, "approx. KL/rejected": 1272.0, "epoch": 0.5413917216556688, "grad_norm": 3.9278781414031982, "learning_rate": 4.35632311600754e-06, "logp/chosen": -452.0, "logp/rejected": -330.0, "loss": 0.4735107421875, "reward/accuracy": 0.75, "reward/chosen": -1.96875, "reward/margin": 2.515625, "reward/rejected": -4.5, "step": 1805 }, { "approx. KL/chosen": 580.0, "approx. KL/rejected": 992.0, "epoch": 0.5416916616676665, "grad_norm": 4.037205696105957, "learning_rate": 4.351651159805305e-06, "logp/chosen": -290.0, "logp/rejected": -420.0, "loss": 0.447998046875, "reward/accuracy": 0.75, "reward/chosen": -2.96875, "reward/margin": 1.3515625, "reward/rejected": -4.3125, "step": 1806 }, { "approx. KL/chosen": 648.0, "approx. KL/rejected": 1056.0, "epoch": 0.5419916016796641, "grad_norm": 4.605172157287598, "learning_rate": 4.346979779277926e-06, "logp/chosen": -368.0, "logp/rejected": -226.0, "loss": 0.53790283203125, "reward/accuracy": 0.875, "reward/chosen": -3.03125, "reward/margin": 1.0625, "reward/rejected": -4.09375, "step": 1807 }, { "approx. KL/chosen": 624.0, "approx. KL/rejected": 1072.0, "epoch": 0.5422915416916617, "grad_norm": 5.12086820602417, "learning_rate": 4.342308978573164e-06, "logp/chosen": -428.0, "logp/rejected": -272.0, "loss": 0.58001708984375, "reward/accuracy": 0.6875, "reward/chosen": -3.0, "reward/margin": 1.375, "reward/rejected": -4.375, "step": 1808 }, { "approx. KL/chosen": 334.0, "approx. KL/rejected": 708.0, "epoch": 0.5425914817036592, "grad_norm": 3.8207106590270996, "learning_rate": 4.337638761838266e-06, "logp/chosen": -312.0, "logp/rejected": -356.0, "loss": 0.5543212890625, "reward/accuracy": 0.6875, "reward/chosen": -2.3125, "reward/margin": 0.89453125, "reward/rejected": -3.203125, "step": 1809 }, { "approx. KL/chosen": 290.0, "approx. KL/rejected": 1056.0, "epoch": 0.5428914217156569, "grad_norm": 3.4906439781188965, "learning_rate": 4.3329691332199605e-06, "logp/chosen": -346.0, "logp/rejected": -306.0, "loss": 0.4674072265625, "reward/accuracy": 0.75, "reward/chosen": -2.03125, "reward/margin": 2.03125, "reward/rejected": -4.0625, "step": 1810 }, { "approx. KL/chosen": 346.0, "approx. KL/rejected": 772.0, "epoch": 0.5431913617276545, "grad_norm": 4.563900947570801, "learning_rate": 4.328300096864451e-06, "logp/chosen": -322.0, "logp/rejected": -322.0, "loss": 0.57232666015625, "reward/accuracy": 0.5625, "reward/chosen": -2.296875, "reward/margin": 1.125, "reward/rejected": -3.421875, "step": 1811 }, { "approx. KL/chosen": 608.0, "approx. KL/rejected": 960.0, "epoch": 0.5434913017396521, "grad_norm": 5.909413814544678, "learning_rate": 4.323631656917419e-06, "logp/chosen": -408.0, "logp/rejected": -410.0, "loss": 0.724853515625, "reward/accuracy": 0.5, "reward/chosen": -3.171875, "reward/margin": 0.75390625, "reward/rejected": -3.921875, "step": 1812 }, { "approx. KL/chosen": 358.0, "approx. KL/rejected": 688.0, "epoch": 0.5437912417516497, "grad_norm": 4.979512691497803, "learning_rate": 4.318963817524014e-06, "logp/chosen": -354.0, "logp/rejected": -390.0, "loss": 0.70751953125, "reward/accuracy": 0.75, "reward/chosen": -2.3125, "reward/margin": 0.99609375, "reward/rejected": -3.3125, "step": 1813 }, { "approx. KL/chosen": 432.0, "approx. KL/rejected": 1040.0, "epoch": 0.5440911817636472, "grad_norm": 4.078782081604004, "learning_rate": 4.314296582828855e-06, "logp/chosen": -336.0, "logp/rejected": -306.0, "loss": 0.5328369140625, "reward/accuracy": 0.75, "reward/chosen": -2.5625, "reward/margin": 1.46875, "reward/rejected": -4.03125, "step": 1814 }, { "approx. KL/chosen": 454.0, "approx. KL/rejected": 1240.0, "epoch": 0.5443911217756449, "grad_norm": 4.66916561126709, "learning_rate": 4.309629956976023e-06, "logp/chosen": -350.0, "logp/rejected": -304.0, "loss": 0.5755615234375, "reward/accuracy": 0.625, "reward/chosen": -2.734375, "reward/margin": 1.65625, "reward/rejected": -4.40625, "step": 1815 }, { "approx. KL/chosen": 540.0, "approx. KL/rejected": 1248.0, "epoch": 0.5446910617876425, "grad_norm": 3.778512954711914, "learning_rate": 4.304963944109055e-06, "logp/chosen": -332.0, "logp/rejected": -500.0, "loss": 0.3448486328125, "reward/accuracy": 0.8125, "reward/chosen": -2.90625, "reward/margin": 1.890625, "reward/rejected": -4.8125, "step": 1816 }, { "approx. KL/chosen": 390.0, "approx. KL/rejected": 1144.0, "epoch": 0.5449910017996401, "grad_norm": 3.177859306335449, "learning_rate": 4.300298548370947e-06, "logp/chosen": -232.0, "logp/rejected": -278.0, "loss": 0.326141357421875, "reward/accuracy": 0.75, "reward/chosen": -2.5625, "reward/margin": 1.7734375, "reward/rejected": -4.3125, "step": 1817 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 852.0, "epoch": 0.5452909418116376, "grad_norm": 3.980295181274414, "learning_rate": 4.295633773904146e-06, "logp/chosen": -420.0, "logp/rejected": -268.0, "loss": 0.4030418395996094, "reward/accuracy": 0.8125, "reward/chosen": -1.5, "reward/margin": 2.0, "reward/rejected": -3.5, "step": 1818 }, { "approx. KL/chosen": 374.0, "approx. KL/rejected": 1360.0, "epoch": 0.5455908818236352, "grad_norm": 2.343109130859375, "learning_rate": 4.290969624850552e-06, "logp/chosen": -255.0, "logp/rejected": -370.0, "loss": 0.3232421875, "reward/accuracy": 0.9375, "reward/chosen": -2.453125, "reward/margin": 2.328125, "reward/rejected": -4.78125, "step": 1819 }, { "approx. KL/chosen": 352.0, "approx. KL/rejected": 1144.0, "epoch": 0.5458908218356329, "grad_norm": 2.8252604007720947, "learning_rate": 4.2863061053515e-06, "logp/chosen": -410.0, "logp/rejected": -360.0, "loss": 0.2825927734375, "reward/accuracy": 0.875, "reward/chosen": -2.25, "reward/margin": 2.1875, "reward/rejected": -4.4375, "step": 1820 }, { "approx. KL/chosen": 237.0, "approx. KL/rejected": 660.0, "epoch": 0.5461907618476305, "grad_norm": 3.7579472064971924, "learning_rate": 4.2816432195477745e-06, "logp/chosen": -406.0, "logp/rejected": -334.0, "loss": 0.4364013671875, "reward/accuracy": 0.8125, "reward/chosen": -1.84375, "reward/margin": 1.3515625, "reward/rejected": -3.1875, "step": 1821 }, { "approx. KL/chosen": 213.0, "approx. KL/rejected": 816.0, "epoch": 0.5464907018596281, "grad_norm": 3.419442653656006, "learning_rate": 4.276980971579593e-06, "logp/chosen": -264.0, "logp/rejected": -274.0, "loss": 0.41925048828125, "reward/accuracy": 0.8125, "reward/chosen": -1.703125, "reward/margin": 1.828125, "reward/rejected": -3.53125, "step": 1822 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 740.0, "epoch": 0.5467906418716256, "grad_norm": 2.9633562564849854, "learning_rate": 4.272319365586609e-06, "logp/chosen": -227.0, "logp/rejected": -248.0, "loss": 0.3726806640625, "reward/accuracy": 0.8125, "reward/chosen": -2.09375, "reward/margin": 1.5546875, "reward/rejected": -3.640625, "step": 1823 }, { "approx. KL/chosen": 438.0, "approx. KL/rejected": 1080.0, "epoch": 0.5470905818836233, "grad_norm": 3.621182441711426, "learning_rate": 4.267658405707907e-06, "logp/chosen": -256.0, "logp/rejected": -284.0, "loss": 0.3385009765625, "reward/accuracy": 0.875, "reward/chosen": -2.59375, "reward/margin": 1.6640625, "reward/rejected": -4.25, "step": 1824 }, { "approx. KL/chosen": 398.0, "approx. KL/rejected": 804.0, "epoch": 0.5473905218956209, "grad_norm": 4.680551528930664, "learning_rate": 4.262998096081992e-06, "logp/chosen": -404.0, "logp/rejected": -334.0, "loss": 0.554779052734375, "reward/accuracy": 0.8125, "reward/chosen": -2.546875, "reward/margin": 1.1640625, "reward/rejected": -3.71875, "step": 1825 }, { "approx. KL/chosen": 438.0, "approx. KL/rejected": 1184.0, "epoch": 0.5476904619076185, "grad_norm": 3.5768966674804688, "learning_rate": 4.2583384408467975e-06, "logp/chosen": -310.0, "logp/rejected": -308.0, "loss": 0.3375244140625, "reward/accuracy": 0.875, "reward/chosen": -2.484375, "reward/margin": 1.9609375, "reward/rejected": -4.4375, "step": 1826 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 1032.0, "epoch": 0.5479904019196161, "grad_norm": 2.7803139686584473, "learning_rate": 4.253679444139673e-06, "logp/chosen": -219.0, "logp/rejected": -274.0, "loss": 0.40252685546875, "reward/accuracy": 0.875, "reward/chosen": -2.109375, "reward/margin": 1.8671875, "reward/rejected": -3.984375, "step": 1827 }, { "approx. KL/chosen": 520.0, "approx. KL/rejected": 956.0, "epoch": 0.5482903419316136, "grad_norm": 4.447725772857666, "learning_rate": 4.249021110097385e-06, "logp/chosen": -434.0, "logp/rejected": -342.0, "loss": 0.5986328125, "reward/accuracy": 0.875, "reward/chosen": -2.828125, "reward/margin": 1.15625, "reward/rejected": -4.0, "step": 1828 }, { "approx. KL/chosen": 532.0, "approx. KL/rejected": 744.0, "epoch": 0.5485902819436113, "grad_norm": 4.525023460388184, "learning_rate": 4.244363442856113e-06, "logp/chosen": -320.0, "logp/rejected": -296.0, "loss": 0.59527587890625, "reward/accuracy": 0.75, "reward/chosen": -2.875, "reward/margin": 0.7265625, "reward/rejected": -3.59375, "step": 1829 }, { "approx. KL/chosen": 290.0, "approx. KL/rejected": 768.0, "epoch": 0.5488902219556089, "grad_norm": 3.6091463565826416, "learning_rate": 4.239706446551437e-06, "logp/chosen": -352.0, "logp/rejected": -372.0, "loss": 0.39923095703125, "reward/accuracy": 0.75, "reward/chosen": -2.1875, "reward/margin": 1.4296875, "reward/rejected": -3.625, "step": 1830 }, { "approx. KL/chosen": 308.0, "approx. KL/rejected": 720.0, "epoch": 0.5491901619676065, "grad_norm": 3.1837151050567627, "learning_rate": 4.23505012531835e-06, "logp/chosen": -221.0, "logp/rejected": -250.0, "loss": 0.4263916015625, "reward/accuracy": 0.8125, "reward/chosen": -2.234375, "reward/margin": 1.171875, "reward/rejected": -3.40625, "step": 1831 }, { "approx. KL/chosen": 460.0, "approx. KL/rejected": 1168.0, "epoch": 0.5494901019796041, "grad_norm": 3.6905696392059326, "learning_rate": 4.230394483291243e-06, "logp/chosen": -364.0, "logp/rejected": -376.0, "loss": 0.410888671875, "reward/accuracy": 0.8125, "reward/chosen": -2.703125, "reward/margin": 1.703125, "reward/rejected": -4.40625, "step": 1832 }, { "approx. KL/chosen": 410.0, "approx. KL/rejected": 1336.0, "epoch": 0.5497900419916016, "grad_norm": 2.4586641788482666, "learning_rate": 4.2257395246039005e-06, "logp/chosen": -426.0, "logp/rejected": -314.0, "loss": 0.1898193359375, "reward/accuracy": 0.9375, "reward/chosen": -2.09375, "reward/margin": 2.796875, "reward/rejected": -4.90625, "step": 1833 }, { "approx. KL/chosen": 502.0, "approx. KL/rejected": 784.0, "epoch": 0.5500899820035993, "grad_norm": 8.2230863571167, "learning_rate": 4.2210852533895054e-06, "logp/chosen": -302.0, "logp/rejected": -370.0, "loss": 0.43341064453125, "reward/accuracy": 0.8125, "reward/chosen": -2.25, "reward/margin": 1.1796875, "reward/rejected": -3.4375, "step": 1834 }, { "approx. KL/chosen": 418.0, "approx. KL/rejected": 876.0, "epoch": 0.5503899220155969, "grad_norm": 2.4952375888824463, "learning_rate": 4.2164316737806256e-06, "logp/chosen": -324.0, "logp/rejected": -245.0, "loss": 0.33819580078125, "reward/accuracy": 0.875, "reward/chosen": -2.5, "reward/margin": 1.5234375, "reward/rejected": -4.03125, "step": 1835 }, { "approx. KL/chosen": 294.0, "approx. KL/rejected": 828.0, "epoch": 0.5506898620275945, "grad_norm": 3.2260324954986572, "learning_rate": 4.211778789909216e-06, "logp/chosen": -324.0, "logp/rejected": -308.0, "loss": 0.36053466796875, "reward/accuracy": 0.8125, "reward/chosen": -2.0625, "reward/margin": 1.703125, "reward/rejected": -3.765625, "step": 1836 }, { "approx. KL/chosen": 426.0, "approx. KL/rejected": 980.0, "epoch": 0.5509898020395921, "grad_norm": 3.748710870742798, "learning_rate": 4.207126605906617e-06, "logp/chosen": -394.0, "logp/rejected": -482.0, "loss": 0.433624267578125, "reward/accuracy": 0.75, "reward/chosen": -2.765625, "reward/margin": 1.296875, "reward/rejected": -4.0625, "step": 1837 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 724.0, "epoch": 0.5512897420515896, "grad_norm": 3.5700509548187256, "learning_rate": 4.202475125903545e-06, "logp/chosen": -302.0, "logp/rejected": -310.0, "loss": 0.483642578125, "reward/accuracy": 0.75, "reward/chosen": -2.125, "reward/margin": 1.2265625, "reward/rejected": -3.359375, "step": 1838 }, { "approx. KL/chosen": 348.0, "approx. KL/rejected": 844.0, "epoch": 0.5515896820635873, "grad_norm": 3.2929177284240723, "learning_rate": 4.197824354030093e-06, "logp/chosen": -410.0, "logp/rejected": -342.0, "loss": 0.362945556640625, "reward/accuracy": 0.8125, "reward/chosen": -2.09375, "reward/margin": 1.578125, "reward/rejected": -3.671875, "step": 1839 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 940.0, "epoch": 0.5518896220755849, "grad_norm": 3.4419162273406982, "learning_rate": 4.19317429441572e-06, "logp/chosen": -266.0, "logp/rejected": -268.0, "loss": 0.345062255859375, "reward/accuracy": 0.875, "reward/chosen": -2.140625, "reward/margin": 1.8828125, "reward/rejected": -4.03125, "step": 1840 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 1024.0, "epoch": 0.5521895620875825, "grad_norm": 3.267024040222168, "learning_rate": 4.188524951189257e-06, "logp/chosen": -258.0, "logp/rejected": -322.0, "loss": 0.4958648681640625, "reward/accuracy": 0.75, "reward/chosen": -2.390625, "reward/margin": 1.578125, "reward/rejected": -3.96875, "step": 1841 }, { "approx. KL/chosen": 410.0, "approx. KL/rejected": 888.0, "epoch": 0.55248950209958, "grad_norm": 4.639177322387695, "learning_rate": 4.183876328478901e-06, "logp/chosen": -302.0, "logp/rejected": -332.0, "loss": 0.5472412109375, "reward/accuracy": 0.6875, "reward/chosen": -2.484375, "reward/margin": 1.25, "reward/rejected": -3.734375, "step": 1842 }, { "approx. KL/chosen": 308.0, "approx. KL/rejected": 904.0, "epoch": 0.5527894421115777, "grad_norm": 4.203171730041504, "learning_rate": 4.179228430412207e-06, "logp/chosen": -286.0, "logp/rejected": -312.0, "loss": 0.31768798828125, "reward/accuracy": 0.875, "reward/chosen": -2.25, "reward/margin": 1.640625, "reward/rejected": -3.890625, "step": 1843 }, { "approx. KL/chosen": 364.0, "approx. KL/rejected": 1144.0, "epoch": 0.5530893821235753, "grad_norm": 5.861119747161865, "learning_rate": 4.174581261116082e-06, "logp/chosen": -350.0, "logp/rejected": -358.0, "loss": 0.61566162109375, "reward/accuracy": 0.75, "reward/chosen": -2.375, "reward/margin": 1.6875, "reward/rejected": -4.0625, "step": 1844 }, { "approx. KL/chosen": 316.0, "approx. KL/rejected": 848.0, "epoch": 0.5533893221355729, "grad_norm": 3.700080633163452, "learning_rate": 4.169934824716791e-06, "logp/chosen": -460.0, "logp/rejected": -354.0, "loss": 0.42657470703125, "reward/accuracy": 0.75, "reward/chosen": -2.078125, "reward/margin": 1.625, "reward/rejected": -3.703125, "step": 1845 }, { "approx. KL/chosen": 330.0, "approx. KL/rejected": 1136.0, "epoch": 0.5536892621475705, "grad_norm": 3.5369060039520264, "learning_rate": 4.165289125339951e-06, "logp/chosen": -342.0, "logp/rejected": -260.0, "loss": 0.4314422607421875, "reward/accuracy": 0.8125, "reward/chosen": -2.234375, "reward/margin": 2.09375, "reward/rejected": -4.3125, "step": 1846 }, { "approx. KL/chosen": 404.0, "approx. KL/rejected": 816.0, "epoch": 0.553989202159568, "grad_norm": 5.672236919403076, "learning_rate": 4.160644167110517e-06, "logp/chosen": -300.0, "logp/rejected": -406.0, "loss": 0.4951171875, "reward/accuracy": 0.8125, "reward/chosen": -2.1875, "reward/margin": 1.328125, "reward/rejected": -3.515625, "step": 1847 }, { "approx. KL/chosen": 494.0, "approx. KL/rejected": 948.0, "epoch": 0.5542891421715657, "grad_norm": 4.230484962463379, "learning_rate": 4.155999954152794e-06, "logp/chosen": -272.0, "logp/rejected": -310.0, "loss": 0.3870849609375, "reward/accuracy": 0.8125, "reward/chosen": -2.671875, "reward/margin": 1.4140625, "reward/rejected": -4.09375, "step": 1848 }, { "approx. KL/chosen": 432.0, "approx. KL/rejected": 1080.0, "epoch": 0.5545890821835633, "grad_norm": 3.1827590465545654, "learning_rate": 4.151356490590416e-06, "logp/chosen": -422.0, "logp/rejected": -472.0, "loss": 0.3492431640625, "reward/accuracy": 0.75, "reward/chosen": -2.46875, "reward/margin": 1.765625, "reward/rejected": -4.21875, "step": 1849 }, { "approx. KL/chosen": 436.0, "approx. KL/rejected": 2320.0, "epoch": 0.5548890221955609, "grad_norm": 3.1219565868377686, "learning_rate": 4.146713780546359e-06, "logp/chosen": -362.0, "logp/rejected": -396.0, "loss": 0.31390380859375, "reward/accuracy": 0.875, "reward/chosen": -2.8125, "reward/margin": 2.78125, "reward/rejected": -5.59375, "step": 1850 }, { "approx. KL/chosen": 460.0, "approx. KL/rejected": 944.0, "epoch": 0.5551889622075585, "grad_norm": 4.337337017059326, "learning_rate": 4.142071828142929e-06, "logp/chosen": -302.0, "logp/rejected": -352.0, "loss": 0.368377685546875, "reward/accuracy": 0.8125, "reward/chosen": -2.46875, "reward/margin": 1.546875, "reward/rejected": -4.03125, "step": 1851 }, { "approx. KL/chosen": 338.0, "approx. KL/rejected": 1296.0, "epoch": 0.555488902219556, "grad_norm": 2.757173538208008, "learning_rate": 4.137430637501755e-06, "logp/chosen": -278.0, "logp/rejected": -290.0, "loss": 0.258941650390625, "reward/accuracy": 0.9375, "reward/chosen": -2.25, "reward/margin": 2.421875, "reward/rejected": -4.65625, "step": 1852 }, { "approx. KL/chosen": 354.0, "approx. KL/rejected": 1088.0, "epoch": 0.5557888422315537, "grad_norm": 3.080702066421509, "learning_rate": 4.132790212743797e-06, "logp/chosen": -262.0, "logp/rejected": -318.0, "loss": 0.437042236328125, "reward/accuracy": 0.6875, "reward/chosen": -2.46875, "reward/margin": 1.828125, "reward/rejected": -4.3125, "step": 1853 }, { "approx. KL/chosen": 448.0, "approx. KL/rejected": 1248.0, "epoch": 0.5560887822435513, "grad_norm": 3.9633853435516357, "learning_rate": 4.128150557989326e-06, "logp/chosen": -298.0, "logp/rejected": -360.0, "loss": 0.5301971435546875, "reward/accuracy": 0.6875, "reward/chosen": -2.765625, "reward/margin": 1.6875, "reward/rejected": -4.46875, "step": 1854 }, { "approx. KL/chosen": 600.0, "approx. KL/rejected": 952.0, "epoch": 0.5563887222555489, "grad_norm": 3.9381887912750244, "learning_rate": 4.123511677357935e-06, "logp/chosen": -292.0, "logp/rejected": -336.0, "loss": 0.66748046875, "reward/accuracy": 0.75, "reward/chosen": -3.328125, "reward/margin": 0.6875, "reward/rejected": -4.0, "step": 1855 }, { "approx. KL/chosen": 406.0, "approx. KL/rejected": 1144.0, "epoch": 0.5566886622675465, "grad_norm": 2.495142698287964, "learning_rate": 4.118873574968529e-06, "logp/chosen": -420.0, "logp/rejected": -322.0, "loss": 0.313720703125, "reward/accuracy": 0.8125, "reward/chosen": -2.515625, "reward/margin": 1.921875, "reward/rejected": -4.4375, "step": 1856 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 924.0, "epoch": 0.556988602279544, "grad_norm": 2.6397242546081543, "learning_rate": 4.114236254939323e-06, "logp/chosen": -378.0, "logp/rejected": -310.0, "loss": 0.4158935546875, "reward/accuracy": 0.75, "reward/chosen": -2.078125, "reward/margin": 1.8203125, "reward/rejected": -3.90625, "step": 1857 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 888.0, "epoch": 0.5572885422915417, "grad_norm": 1.540273666381836, "learning_rate": 4.109599721387835e-06, "logp/chosen": -372.0, "logp/rejected": -324.0, "loss": 0.12109375, "reward/accuracy": 1.0, "reward/chosen": -1.65625, "reward/margin": 2.375, "reward/rejected": -4.03125, "step": 1858 }, { "approx. KL/chosen": 506.0, "approx. KL/rejected": 768.0, "epoch": 0.5575884823035393, "grad_norm": 5.568326950073242, "learning_rate": 4.104963978430883e-06, "logp/chosen": -324.0, "logp/rejected": -324.0, "loss": 0.6773681640625, "reward/accuracy": 0.625, "reward/chosen": -2.4375, "reward/margin": 0.921875, "reward/rejected": -3.375, "step": 1859 }, { "approx. KL/chosen": 382.0, "approx. KL/rejected": 1216.0, "epoch": 0.5578884223155369, "grad_norm": 3.5332765579223633, "learning_rate": 4.100329030184588e-06, "logp/chosen": -326.0, "logp/rejected": -438.0, "loss": 0.363861083984375, "reward/accuracy": 0.8125, "reward/chosen": -2.5, "reward/margin": 1.7734375, "reward/rejected": -4.28125, "step": 1860 }, { "approx. KL/chosen": 478.0, "approx. KL/rejected": 1256.0, "epoch": 0.5581883623275345, "grad_norm": 2.7031400203704834, "learning_rate": 4.095694880764361e-06, "logp/chosen": -342.0, "logp/rejected": -340.0, "loss": 0.240692138671875, "reward/accuracy": 0.875, "reward/chosen": -2.71875, "reward/margin": 2.015625, "reward/rejected": -4.71875, "step": 1861 }, { "approx. KL/chosen": 404.0, "approx. KL/rejected": 1680.0, "epoch": 0.5584883023395321, "grad_norm": 2.3817741870880127, "learning_rate": 4.091061534284907e-06, "logp/chosen": -368.0, "logp/rejected": -416.0, "loss": 0.24779129028320312, "reward/accuracy": 0.8125, "reward/chosen": -2.515625, "reward/margin": 2.796875, "reward/rejected": -5.3125, "step": 1862 }, { "approx. KL/chosen": 528.0, "approx. KL/rejected": 1136.0, "epoch": 0.5587882423515297, "grad_norm": 2.7004783153533936, "learning_rate": 4.08642899486022e-06, "logp/chosen": -442.0, "logp/rejected": -402.0, "loss": 0.302490234375, "reward/accuracy": 0.875, "reward/chosen": -2.5, "reward/margin": 2.078125, "reward/rejected": -4.59375, "step": 1863 }, { "approx. KL/chosen": 548.0, "approx. KL/rejected": 1020.0, "epoch": 0.5590881823635273, "grad_norm": 4.456826686859131, "learning_rate": 4.0817972666035665e-06, "logp/chosen": -444.0, "logp/rejected": -404.0, "loss": 0.36016845703125, "reward/accuracy": 0.8125, "reward/chosen": -2.828125, "reward/margin": 1.484375, "reward/rejected": -4.3125, "step": 1864 }, { "approx. KL/chosen": 380.0, "approx. KL/rejected": 1408.0, "epoch": 0.5593881223755249, "grad_norm": 3.3853507041931152, "learning_rate": 4.077166353627506e-06, "logp/chosen": -378.0, "logp/rejected": -364.0, "loss": 0.38720703125, "reward/accuracy": 0.75, "reward/chosen": -2.359375, "reward/margin": 2.328125, "reward/rejected": -4.6875, "step": 1865 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 1472.0, "epoch": 0.5596880623875224, "grad_norm": 2.6007230281829834, "learning_rate": 4.072536260043865e-06, "logp/chosen": -462.0, "logp/rejected": -344.0, "loss": 0.1595306396484375, "reward/accuracy": 0.9375, "reward/chosen": -2.328125, "reward/margin": 2.84375, "reward/rejected": -5.1875, "step": 1866 }, { "approx. KL/chosen": 592.0, "approx. KL/rejected": 1384.0, "epoch": 0.5599880023995201, "grad_norm": 4.575057029724121, "learning_rate": 4.067906989963751e-06, "logp/chosen": -306.0, "logp/rejected": -222.0, "loss": 0.446868896484375, "reward/accuracy": 0.8125, "reward/chosen": -3.03125, "reward/margin": 1.796875, "reward/rejected": -4.8125, "step": 1867 }, { "approx. KL/chosen": 480.0, "approx. KL/rejected": 840.0, "epoch": 0.5602879424115177, "grad_norm": 5.723444938659668, "learning_rate": 4.063278547497529e-06, "logp/chosen": -324.0, "logp/rejected": -378.0, "loss": 0.68499755859375, "reward/accuracy": 0.625, "reward/chosen": -2.828125, "reward/margin": 0.890625, "reward/rejected": -3.71875, "step": 1868 }, { "approx. KL/chosen": 352.0, "approx. KL/rejected": 868.0, "epoch": 0.5605878824235153, "grad_norm": 4.197451591491699, "learning_rate": 4.058650936754837e-06, "logp/chosen": -390.0, "logp/rejected": -412.0, "loss": 0.5265960693359375, "reward/accuracy": 0.6875, "reward/chosen": -2.390625, "reward/margin": 1.3359375, "reward/rejected": -3.734375, "step": 1869 }, { "approx. KL/chosen": 462.0, "approx. KL/rejected": 1392.0, "epoch": 0.5608878224355129, "grad_norm": 3.6751675605773926, "learning_rate": 4.054024161844576e-06, "logp/chosen": -270.0, "logp/rejected": -460.0, "loss": 0.2288818359375, "reward/accuracy": 0.9375, "reward/chosen": -2.75, "reward/margin": 2.25, "reward/rejected": -5.0, "step": 1870 }, { "approx. KL/chosen": 588.0, "approx. KL/rejected": 1520.0, "epoch": 0.5611877624475105, "grad_norm": 5.273285388946533, "learning_rate": 4.0493982268748985e-06, "logp/chosen": -454.0, "logp/rejected": -436.0, "loss": 0.3998260498046875, "reward/accuracy": 0.6875, "reward/chosen": -2.984375, "reward/margin": 2.15625, "reward/rejected": -5.125, "step": 1871 }, { "approx. KL/chosen": 376.0, "approx. KL/rejected": 1312.0, "epoch": 0.5614877024595081, "grad_norm": 2.8661932945251465, "learning_rate": 4.044773135953218e-06, "logp/chosen": -268.0, "logp/rejected": -404.0, "loss": 0.2908782958984375, "reward/accuracy": 0.8125, "reward/chosen": -2.375, "reward/margin": 2.296875, "reward/rejected": -4.6875, "step": 1872 }, { "approx. KL/chosen": 414.0, "approx. KL/rejected": 1136.0, "epoch": 0.5617876424715057, "grad_norm": 4.724412441253662, "learning_rate": 4.04014889318619e-06, "logp/chosen": -274.0, "logp/rejected": -384.0, "loss": 0.429351806640625, "reward/accuracy": 0.8125, "reward/chosen": -2.546875, "reward/margin": 1.875, "reward/rejected": -4.4375, "step": 1873 }, { "approx. KL/chosen": 832.0, "approx. KL/rejected": 1424.0, "epoch": 0.5620875824835033, "grad_norm": 4.1085944175720215, "learning_rate": 4.035525502679724e-06, "logp/chosen": -356.0, "logp/rejected": -342.0, "loss": 0.46478271484375, "reward/accuracy": 0.6875, "reward/chosen": -3.703125, "reward/margin": 1.3203125, "reward/rejected": -5.03125, "step": 1874 }, { "approx. KL/chosen": 352.0, "approx. KL/rejected": 1020.0, "epoch": 0.562387522495501, "grad_norm": 3.0237483978271484, "learning_rate": 4.030902968538972e-06, "logp/chosen": -386.0, "logp/rejected": -336.0, "loss": 0.279205322265625, "reward/accuracy": 0.8125, "reward/chosen": -2.234375, "reward/margin": 2.0, "reward/rejected": -4.25, "step": 1875 }, { "approx. KL/chosen": 384.0, "approx. KL/rejected": 1504.0, "epoch": 0.5626874625074985, "grad_norm": 2.2242963314056396, "learning_rate": 4.026281294868322e-06, "logp/chosen": -368.0, "logp/rejected": -366.0, "loss": 0.16254425048828125, "reward/accuracy": 0.9375, "reward/chosen": -2.484375, "reward/margin": 2.71875, "reward/rejected": -5.1875, "step": 1876 }, { "approx. KL/chosen": 408.0, "approx. KL/rejected": 1152.0, "epoch": 0.5629874025194961, "grad_norm": 3.917125701904297, "learning_rate": 4.0216604857714044e-06, "logp/chosen": -290.0, "logp/rejected": -354.0, "loss": 0.46002197265625, "reward/accuracy": 0.75, "reward/chosen": -2.296875, "reward/margin": 1.7734375, "reward/rejected": -4.0625, "step": 1877 }, { "approx. KL/chosen": 836.0, "approx. KL/rejected": 1112.0, "epoch": 0.5632873425314937, "grad_norm": 5.8787431716918945, "learning_rate": 4.017040545351073e-06, "logp/chosen": -384.0, "logp/rejected": -251.0, "loss": 0.680419921875, "reward/accuracy": 0.6875, "reward/chosen": -3.4375, "reward/margin": 0.8671875, "reward/rejected": -4.3125, "step": 1878 }, { "approx. KL/chosen": 620.0, "approx. KL/rejected": 948.0, "epoch": 0.5635872825434913, "grad_norm": 6.442286014556885, "learning_rate": 4.012421477709416e-06, "logp/chosen": -326.0, "logp/rejected": -246.0, "loss": 0.60174560546875, "reward/accuracy": 0.6875, "reward/chosen": -3.09375, "reward/margin": 0.82421875, "reward/rejected": -3.921875, "step": 1879 }, { "approx. KL/chosen": 324.0, "approx. KL/rejected": 1336.0, "epoch": 0.563887222555489, "grad_norm": 3.390676498413086, "learning_rate": 4.007803286947748e-06, "logp/chosen": -310.0, "logp/rejected": -262.0, "loss": 0.306793212890625, "reward/accuracy": 0.8125, "reward/chosen": -2.1875, "reward/margin": 2.40625, "reward/rejected": -4.59375, "step": 1880 }, { "approx. KL/chosen": 370.0, "approx. KL/rejected": 2176.0, "epoch": 0.5641871625674865, "grad_norm": 1.1982862949371338, "learning_rate": 4.003185977166601e-06, "logp/chosen": -384.0, "logp/rejected": -406.0, "loss": 0.07698822021484375, "reward/accuracy": 1.0, "reward/chosen": -2.421875, "reward/margin": 3.6875, "reward/rejected": -6.09375, "step": 1881 }, { "approx. KL/chosen": 652.0, "approx. KL/rejected": 1424.0, "epoch": 0.5644871025794841, "grad_norm": 4.479726314544678, "learning_rate": 3.998569552465729e-06, "logp/chosen": -294.0, "logp/rejected": -336.0, "loss": 0.51324462890625, "reward/accuracy": 0.8125, "reward/chosen": -3.09375, "reward/margin": 1.703125, "reward/rejected": -4.78125, "step": 1882 }, { "approx. KL/chosen": 440.0, "approx. KL/rejected": 1304.0, "epoch": 0.5647870425914817, "grad_norm": 3.6795191764831543, "learning_rate": 3.993954016944093e-06, "logp/chosen": -258.0, "logp/rejected": -290.0, "loss": 0.4207763671875, "reward/accuracy": 0.8125, "reward/chosen": -2.671875, "reward/margin": 1.890625, "reward/rejected": -4.5625, "step": 1883 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 916.0, "epoch": 0.5650869826034793, "grad_norm": 3.244525671005249, "learning_rate": 3.989339374699872e-06, "logp/chosen": -440.0, "logp/rejected": -366.0, "loss": 0.3116455078125, "reward/accuracy": 0.8125, "reward/chosen": -2.0, "reward/margin": 1.7734375, "reward/rejected": -3.78125, "step": 1884 }, { "approx. KL/chosen": 442.0, "approx. KL/rejected": 1128.0, "epoch": 0.5653869226154769, "grad_norm": 5.645576477050781, "learning_rate": 3.98472562983045e-06, "logp/chosen": -242.0, "logp/rejected": -430.0, "loss": 0.683868408203125, "reward/accuracy": 0.6875, "reward/chosen": -2.515625, "reward/margin": 1.5546875, "reward/rejected": -4.0625, "step": 1885 }, { "approx. KL/chosen": 676.0, "approx. KL/rejected": 1344.0, "epoch": 0.5656868626274745, "grad_norm": 6.027020454406738, "learning_rate": 3.980112786432413e-06, "logp/chosen": -282.0, "logp/rejected": -292.0, "loss": 0.64984130859375, "reward/accuracy": 0.75, "reward/chosen": -3.140625, "reward/margin": 1.6640625, "reward/rejected": -4.8125, "step": 1886 }, { "approx. KL/chosen": 412.0, "approx. KL/rejected": 1664.0, "epoch": 0.5659868026394721, "grad_norm": 2.7135422229766846, "learning_rate": 3.9755008486015425e-06, "logp/chosen": -332.0, "logp/rejected": -412.0, "loss": 0.26214599609375, "reward/accuracy": 0.9375, "reward/chosen": -2.375, "reward/margin": 2.625, "reward/rejected": -5.0, "step": 1887 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 2112.0, "epoch": 0.5662867426514697, "grad_norm": 1.512479543685913, "learning_rate": 3.970889820432825e-06, "logp/chosen": -219.0, "logp/rejected": -368.0, "loss": 0.0744476318359375, "reward/accuracy": 1.0, "reward/chosen": -2.203125, "reward/margin": 4.0, "reward/rejected": -6.1875, "step": 1888 }, { "approx. KL/chosen": 494.0, "approx. KL/rejected": 1472.0, "epoch": 0.5665866826634673, "grad_norm": 4.599649906158447, "learning_rate": 3.9662797060204325e-06, "logp/chosen": -350.0, "logp/rejected": -310.0, "loss": 0.3458404541015625, "reward/accuracy": 0.75, "reward/chosen": -2.46875, "reward/margin": 2.3125, "reward/rejected": -4.78125, "step": 1889 }, { "approx. KL/chosen": 956.0, "approx. KL/rejected": 1544.0, "epoch": 0.5668866226754649, "grad_norm": 5.430084228515625, "learning_rate": 3.961670509457728e-06, "logp/chosen": -274.0, "logp/rejected": -312.0, "loss": 0.82476806640625, "reward/accuracy": 0.5, "reward/chosen": -4.125, "reward/margin": 0.8984375, "reward/rejected": -5.03125, "step": 1890 }, { "approx. KL/chosen": 498.0, "approx. KL/rejected": 1512.0, "epoch": 0.5671865626874625, "grad_norm": 3.0935840606689453, "learning_rate": 3.957062234837262e-06, "logp/chosen": -322.0, "logp/rejected": -316.0, "loss": 0.20098876953125, "reward/accuracy": 0.9375, "reward/chosen": -2.390625, "reward/margin": 2.71875, "reward/rejected": -5.125, "step": 1891 }, { "approx. KL/chosen": 406.0, "approx. KL/rejected": 1368.0, "epoch": 0.5674865026994601, "grad_norm": 2.440936326980591, "learning_rate": 3.952454886250758e-06, "logp/chosen": -242.0, "logp/rejected": -268.0, "loss": 0.203125, "reward/accuracy": 0.9375, "reward/chosen": -2.28125, "reward/margin": 2.703125, "reward/rejected": -4.96875, "step": 1892 }, { "approx. KL/chosen": 418.0, "approx. KL/rejected": 1520.0, "epoch": 0.5677864427114577, "grad_norm": 3.396407127380371, "learning_rate": 3.9478484677891275e-06, "logp/chosen": -406.0, "logp/rejected": -410.0, "loss": 0.2543792724609375, "reward/accuracy": 0.8125, "reward/chosen": -2.453125, "reward/margin": 2.78125, "reward/rejected": -5.25, "step": 1893 }, { "approx. KL/chosen": 432.0, "approx. KL/rejected": 1360.0, "epoch": 0.5680863827234554, "grad_norm": 3.8973639011383057, "learning_rate": 3.943242983542448e-06, "logp/chosen": -382.0, "logp/rejected": -288.0, "loss": 0.4501495361328125, "reward/accuracy": 0.75, "reward/chosen": -2.515625, "reward/margin": 2.171875, "reward/rejected": -4.6875, "step": 1894 }, { "approx. KL/chosen": 620.0, "approx. KL/rejected": 1552.0, "epoch": 0.5683863227354529, "grad_norm": 4.2090983390808105, "learning_rate": 3.938638437599974e-06, "logp/chosen": -434.0, "logp/rejected": -408.0, "loss": 0.3286781311035156, "reward/accuracy": 0.8125, "reward/chosen": -2.75, "reward/margin": 2.46875, "reward/rejected": -5.21875, "step": 1895 }, { "approx. KL/chosen": 430.0, "approx. KL/rejected": 1536.0, "epoch": 0.5686862627474505, "grad_norm": 2.5678179264068604, "learning_rate": 3.934034834050125e-06, "logp/chosen": -326.0, "logp/rejected": -440.0, "loss": 0.202850341796875, "reward/accuracy": 0.9375, "reward/chosen": -2.671875, "reward/margin": 2.4375, "reward/rejected": -5.125, "step": 1896 }, { "approx. KL/chosen": 402.0, "approx. KL/rejected": 1400.0, "epoch": 0.5689862027594481, "grad_norm": 2.6236190795898438, "learning_rate": 3.929432176980477e-06, "logp/chosen": -358.0, "logp/rejected": -356.0, "loss": 0.2930908203125, "reward/accuracy": 0.75, "reward/chosen": -2.578125, "reward/margin": 2.296875, "reward/rejected": -4.875, "step": 1897 }, { "approx. KL/chosen": 442.0, "approx. KL/rejected": 964.0, "epoch": 0.5692861427714457, "grad_norm": 4.7964558601379395, "learning_rate": 3.924830470477775e-06, "logp/chosen": -388.0, "logp/rejected": -300.0, "loss": 0.5745468139648438, "reward/accuracy": 0.75, "reward/chosen": -2.296875, "reward/margin": 1.609375, "reward/rejected": -3.921875, "step": 1898 }, { "approx. KL/chosen": 410.0, "approx. KL/rejected": 840.0, "epoch": 0.5695860827834434, "grad_norm": 4.736860752105713, "learning_rate": 3.920229718627916e-06, "logp/chosen": -338.0, "logp/rejected": -308.0, "loss": 0.538726806640625, "reward/accuracy": 0.6875, "reward/chosen": -2.609375, "reward/margin": 1.25, "reward/rejected": -3.859375, "step": 1899 }, { "approx. KL/chosen": 644.0, "approx. KL/rejected": 1112.0, "epoch": 0.5698860227954409, "grad_norm": 5.115039825439453, "learning_rate": 3.91562992551595e-06, "logp/chosen": -372.0, "logp/rejected": -408.0, "loss": 0.7644805908203125, "reward/accuracy": 0.5625, "reward/chosen": -3.234375, "reward/margin": 0.859375, "reward/rejected": -4.09375, "step": 1900 }, { "approx. KL/chosen": 616.0, "approx. KL/rejected": 1360.0, "epoch": 0.5701859628074385, "grad_norm": 5.0348920822143555, "learning_rate": 3.911031095226077e-06, "logp/chosen": -292.0, "logp/rejected": -298.0, "loss": 0.7132072448730469, "reward/accuracy": 0.625, "reward/chosen": -3.234375, "reward/margin": 1.5390625, "reward/rejected": -4.78125, "step": 1901 }, { "approx. KL/chosen": 836.0, "approx. KL/rejected": 1176.0, "epoch": 0.5704859028194361, "grad_norm": 7.126864910125732, "learning_rate": 3.906433231841635e-06, "logp/chosen": -380.0, "logp/rejected": -340.0, "loss": 0.63458251953125, "reward/accuracy": 0.625, "reward/chosen": -3.40625, "reward/margin": 1.171875, "reward/rejected": -4.5625, "step": 1902 }, { "approx. KL/chosen": 342.0, "approx. KL/rejected": 1240.0, "epoch": 0.5707858428314337, "grad_norm": 3.273268461227417, "learning_rate": 3.901836339445113e-06, "logp/chosen": -414.0, "logp/rejected": -478.0, "loss": 0.27111053466796875, "reward/accuracy": 0.875, "reward/chosen": -2.078125, "reward/margin": 2.46875, "reward/rejected": -4.5625, "step": 1903 }, { "approx. KL/chosen": 348.0, "approx. KL/rejected": 972.0, "epoch": 0.5710857828434314, "grad_norm": 3.7916088104248047, "learning_rate": 3.897240422118132e-06, "logp/chosen": -382.0, "logp/rejected": -330.0, "loss": 0.49420166015625, "reward/accuracy": 0.8125, "reward/chosen": -2.171875, "reward/margin": 1.734375, "reward/rejected": -3.921875, "step": 1904 }, { "approx. KL/chosen": 656.0, "approx. KL/rejected": 1840.0, "epoch": 0.5713857228554289, "grad_norm": 6.054385185241699, "learning_rate": 3.89264548394145e-06, "logp/chosen": -262.0, "logp/rejected": -260.0, "loss": 0.3509979248046875, "reward/accuracy": 0.75, "reward/chosen": -3.125, "reward/margin": 2.375, "reward/rejected": -5.5, "step": 1905 }, { "approx. KL/chosen": 240.0, "approx. KL/rejected": 1312.0, "epoch": 0.5716856628674265, "grad_norm": 3.149085521697998, "learning_rate": 3.888051528994957e-06, "logp/chosen": -388.0, "logp/rejected": -346.0, "loss": 0.3867645263671875, "reward/accuracy": 0.875, "reward/chosen": -1.6484375, "reward/margin": 2.90625, "reward/rejected": -4.5625, "step": 1906 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 1056.0, "epoch": 0.5719856028794241, "grad_norm": 3.8025026321411133, "learning_rate": 3.883458561357662e-06, "logp/chosen": -500.0, "logp/rejected": -440.0, "loss": 0.46453189849853516, "reward/accuracy": 0.8125, "reward/chosen": -1.75, "reward/margin": 2.109375, "reward/rejected": -3.84375, "step": 1907 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 896.0, "epoch": 0.5722855428914218, "grad_norm": 2.1404807567596436, "learning_rate": 3.878866585107706e-06, "logp/chosen": -312.0, "logp/rejected": -354.0, "loss": 0.250885009765625, "reward/accuracy": 1.0, "reward/chosen": -1.5625, "reward/margin": 2.046875, "reward/rejected": -3.609375, "step": 1908 }, { "approx. KL/chosen": 338.0, "approx. KL/rejected": 1096.0, "epoch": 0.5725854829034193, "grad_norm": 3.231858253479004, "learning_rate": 3.874275604322346e-06, "logp/chosen": -364.0, "logp/rejected": -358.0, "loss": 0.31732177734375, "reward/accuracy": 0.875, "reward/chosen": -1.953125, "reward/margin": 2.3125, "reward/rejected": -4.25, "step": 1909 }, { "approx. KL/chosen": 223.0, "approx. KL/rejected": 1056.0, "epoch": 0.5728854229154169, "grad_norm": 5.595136642456055, "learning_rate": 3.869685623077959e-06, "logp/chosen": -310.0, "logp/rejected": -338.0, "loss": 0.7032461166381836, "reward/accuracy": 0.6875, "reward/chosen": -1.734375, "reward/margin": 1.9609375, "reward/rejected": -3.6875, "step": 1910 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 828.0, "epoch": 0.5731853629274145, "grad_norm": 4.623121738433838, "learning_rate": 3.865096645450025e-06, "logp/chosen": -456.0, "logp/rejected": -396.0, "loss": 0.620361328125, "reward/accuracy": 0.6875, "reward/chosen": -1.984375, "reward/margin": 1.2890625, "reward/rejected": -3.28125, "step": 1911 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 1344.0, "epoch": 0.5734853029394121, "grad_norm": 1.6638474464416504, "learning_rate": 3.8605086755131435e-06, "logp/chosen": -274.0, "logp/rejected": -324.0, "loss": 0.0991363525390625, "reward/accuracy": 1.0, "reward/chosen": -1.7734375, "reward/margin": 3.15625, "reward/rejected": -4.9375, "step": 1912 }, { "approx. KL/chosen": 432.0, "approx. KL/rejected": 1048.0, "epoch": 0.5737852429514098, "grad_norm": 6.887319564819336, "learning_rate": 3.855921717341014e-06, "logp/chosen": -396.0, "logp/rejected": -312.0, "loss": 0.603607177734375, "reward/accuracy": 0.5625, "reward/chosen": -2.40625, "reward/margin": 1.5234375, "reward/rejected": -3.9375, "step": 1913 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 708.0, "epoch": 0.5740851829634073, "grad_norm": 3.3503024578094482, "learning_rate": 3.851335775006438e-06, "logp/chosen": -340.0, "logp/rejected": -354.0, "loss": 0.44216156005859375, "reward/accuracy": 0.625, "reward/chosen": -1.3359375, "reward/margin": 1.5703125, "reward/rejected": -2.90625, "step": 1914 }, { "approx. KL/chosen": 372.0, "approx. KL/rejected": 1616.0, "epoch": 0.5743851229754049, "grad_norm": 2.1354541778564453, "learning_rate": 3.846750852581319e-06, "logp/chosen": -245.0, "logp/rejected": -346.0, "loss": 0.18013763427734375, "reward/accuracy": 0.9375, "reward/chosen": -2.09375, "reward/margin": 2.984375, "reward/rejected": -5.0625, "step": 1915 }, { "approx. KL/chosen": 494.0, "approx. KL/rejected": 1160.0, "epoch": 0.5746850629874025, "grad_norm": 4.860141277313232, "learning_rate": 3.8421669541366455e-06, "logp/chosen": -386.0, "logp/rejected": -294.0, "loss": 0.5532913208007812, "reward/accuracy": 0.6875, "reward/chosen": -2.703125, "reward/margin": 1.6953125, "reward/rejected": -4.40625, "step": 1916 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 984.0, "epoch": 0.5749850029994001, "grad_norm": 2.8257884979248047, "learning_rate": 3.837584083742507e-06, "logp/chosen": -414.0, "logp/rejected": -400.0, "loss": 0.3139190673828125, "reward/accuracy": 0.8125, "reward/chosen": -1.8671875, "reward/margin": 2.171875, "reward/rejected": -4.03125, "step": 1917 }, { "approx. KL/chosen": 440.0, "approx. KL/rejected": 1064.0, "epoch": 0.5752849430113978, "grad_norm": 5.673099040985107, "learning_rate": 3.833002245468074e-06, "logp/chosen": -424.0, "logp/rejected": -312.0, "loss": 0.7027626037597656, "reward/accuracy": 0.6875, "reward/chosen": -2.265625, "reward/margin": 1.5703125, "reward/rejected": -3.828125, "step": 1918 }, { "approx. KL/chosen": 568.0, "approx. KL/rejected": 1352.0, "epoch": 0.5755848830233953, "grad_norm": 4.935585021972656, "learning_rate": 3.828421443381605e-06, "logp/chosen": -472.0, "logp/rejected": -456.0, "loss": 0.54608154296875, "reward/accuracy": 0.6875, "reward/chosen": -2.859375, "reward/margin": 1.546875, "reward/rejected": -4.40625, "step": 1919 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 1264.0, "epoch": 0.5758848230353929, "grad_norm": 4.942784309387207, "learning_rate": 3.823841681550433e-06, "logp/chosen": -412.0, "logp/rejected": -384.0, "loss": 0.4344482421875, "reward/accuracy": 0.75, "reward/chosen": -1.6171875, "reward/margin": 2.5625, "reward/rejected": -4.1875, "step": 1920 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 1128.0, "epoch": 0.5761847630473905, "grad_norm": 3.9100890159606934, "learning_rate": 3.819262964040972e-06, "logp/chosen": -414.0, "logp/rejected": -400.0, "loss": 0.353057861328125, "reward/accuracy": 0.875, "reward/chosen": -1.515625, "reward/margin": 2.515625, "reward/rejected": -4.03125, "step": 1921 }, { "approx. KL/chosen": 330.0, "approx. KL/rejected": 1496.0, "epoch": 0.5764847030593881, "grad_norm": 3.43033766746521, "learning_rate": 3.814685294918706e-06, "logp/chosen": -199.0, "logp/rejected": -243.0, "loss": 0.34377002716064453, "reward/accuracy": 0.875, "reward/chosen": -1.8828125, "reward/margin": 2.78125, "reward/rejected": -4.65625, "step": 1922 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 1032.0, "epoch": 0.5767846430713858, "grad_norm": 3.9225270748138428, "learning_rate": 3.81010867824819e-06, "logp/chosen": -424.0, "logp/rejected": -386.0, "loss": 0.361785888671875, "reward/accuracy": 0.8125, "reward/chosen": -1.578125, "reward/margin": 2.390625, "reward/rejected": -3.96875, "step": 1923 }, { "approx. KL/chosen": 142.0, "approx. KL/rejected": 912.0, "epoch": 0.5770845830833833, "grad_norm": 2.9598324298858643, "learning_rate": 3.8055331180930448e-06, "logp/chosen": -274.0, "logp/rejected": -282.0, "loss": 0.38531494140625, "reward/accuracy": 0.875, "reward/chosen": -1.3125, "reward/margin": 2.21875, "reward/rejected": -3.53125, "step": 1924 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 944.0, "epoch": 0.5773845230953809, "grad_norm": 4.359506607055664, "learning_rate": 3.800958618515951e-06, "logp/chosen": -340.0, "logp/rejected": -296.0, "loss": 0.5902099609375, "reward/accuracy": 0.6875, "reward/chosen": -1.4765625, "reward/margin": 1.9375, "reward/rejected": -3.40625, "step": 1925 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 760.0, "epoch": 0.5776844631073785, "grad_norm": 6.126040458679199, "learning_rate": 3.796385183578648e-06, "logp/chosen": -336.0, "logp/rejected": -350.0, "loss": 0.6513214111328125, "reward/accuracy": 0.625, "reward/chosen": -2.25, "reward/margin": 1.296875, "reward/rejected": -3.53125, "step": 1926 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 828.0, "epoch": 0.5779844031193762, "grad_norm": 5.781818866729736, "learning_rate": 3.7918128173419303e-06, "logp/chosen": -438.0, "logp/rejected": -354.0, "loss": 0.6970291137695312, "reward/accuracy": 0.625, "reward/chosen": -2.015625, "reward/margin": 1.3046875, "reward/rejected": -3.3125, "step": 1927 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 820.0, "epoch": 0.5782843431313738, "grad_norm": 2.686629056930542, "learning_rate": 3.7872415238656455e-06, "logp/chosen": -440.0, "logp/rejected": -400.0, "loss": 0.294464111328125, "reward/accuracy": 0.875, "reward/chosen": -1.484375, "reward/margin": 2.234375, "reward/rejected": -3.71875, "step": 1928 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 740.0, "epoch": 0.5785842831433713, "grad_norm": 3.0665245056152344, "learning_rate": 3.782671307208685e-06, "logp/chosen": -316.0, "logp/rejected": -294.0, "loss": 0.34063720703125, "reward/accuracy": 0.75, "reward/chosen": -1.3046875, "reward/margin": 1.9765625, "reward/rejected": -3.28125, "step": 1929 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 1020.0, "epoch": 0.5788842231553689, "grad_norm": 3.0522873401641846, "learning_rate": 3.7781021714289873e-06, "logp/chosen": -272.0, "logp/rejected": -241.0, "loss": 0.30564117431640625, "reward/accuracy": 0.8125, "reward/chosen": -1.359375, "reward/margin": 2.625, "reward/rejected": -3.984375, "step": 1930 }, { "approx. KL/chosen": 330.0, "approx. KL/rejected": 848.0, "epoch": 0.5791841631673665, "grad_norm": 3.965975284576416, "learning_rate": 3.773534120583528e-06, "logp/chosen": -380.0, "logp/rejected": -360.0, "loss": 0.52239990234375, "reward/accuracy": 0.6875, "reward/chosen": -1.9609375, "reward/margin": 1.671875, "reward/rejected": -3.625, "step": 1931 }, { "approx. KL/chosen": 255.0, "approx. KL/rejected": 748.0, "epoch": 0.5794841031793642, "grad_norm": 3.6240897178649902, "learning_rate": 3.7689671587283216e-06, "logp/chosen": -424.0, "logp/rejected": -430.0, "loss": 0.26416015625, "reward/accuracy": 0.875, "reward/chosen": -1.4765625, "reward/margin": 1.8359375, "reward/rejected": -3.3125, "step": 1932 }, { "approx. KL/chosen": 238.0, "approx. KL/rejected": 1080.0, "epoch": 0.5797840431913617, "grad_norm": 4.554226398468018, "learning_rate": 3.7644012899184173e-06, "logp/chosen": -241.0, "logp/rejected": -292.0, "loss": 0.5840339660644531, "reward/accuracy": 0.6875, "reward/chosen": -1.6640625, "reward/margin": 1.9140625, "reward/rejected": -3.578125, "step": 1933 }, { "approx. KL/chosen": 292.0, "approx. KL/rejected": 748.0, "epoch": 0.5800839832033593, "grad_norm": 6.882498741149902, "learning_rate": 3.7598365182078904e-06, "logp/chosen": -430.0, "logp/rejected": -412.0, "loss": 0.7567882537841797, "reward/accuracy": 0.8125, "reward/chosen": -1.5078125, "reward/margin": 1.6015625, "reward/rejected": -3.109375, "step": 1934 }, { "approx. KL/chosen": 254.0, "approx. KL/rejected": 1008.0, "epoch": 0.5803839232153569, "grad_norm": 4.062976837158203, "learning_rate": 3.7552728476498417e-06, "logp/chosen": -332.0, "logp/rejected": -384.0, "loss": 0.36505126953125, "reward/accuracy": 0.6875, "reward/chosen": -1.71875, "reward/margin": 2.125, "reward/rejected": -3.828125, "step": 1935 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 884.0, "epoch": 0.5806838632273545, "grad_norm": 4.413326740264893, "learning_rate": 3.7507102822963978e-06, "logp/chosen": -382.0, "logp/rejected": -438.0, "loss": 0.4105682373046875, "reward/accuracy": 0.8125, "reward/chosen": -1.59375, "reward/margin": 1.96875, "reward/rejected": -3.5625, "step": 1936 }, { "approx. KL/chosen": 209.0, "approx. KL/rejected": 740.0, "epoch": 0.5809838032393522, "grad_norm": 3.4736547470092773, "learning_rate": 3.7461488261987024e-06, "logp/chosen": -484.0, "logp/rejected": -436.0, "loss": 0.28204345703125, "reward/accuracy": 0.875, "reward/chosen": -0.92578125, "reward/margin": 2.109375, "reward/rejected": -3.03125, "step": 1937 }, { "approx. KL/chosen": 133.0, "approx. KL/rejected": 576.0, "epoch": 0.5812837432513497, "grad_norm": 4.472240447998047, "learning_rate": 3.7415884834069127e-06, "logp/chosen": -352.0, "logp/rejected": -338.0, "loss": 0.5377197265625, "reward/accuracy": 0.8125, "reward/chosen": -1.0078125, "reward/margin": 1.828125, "reward/rejected": -2.84375, "step": 1938 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 728.0, "epoch": 0.5815836832633473, "grad_norm": 4.050034046173096, "learning_rate": 3.7370292579702005e-06, "logp/chosen": -380.0, "logp/rejected": -350.0, "loss": 0.4323272705078125, "reward/accuracy": 0.75, "reward/chosen": -1.328125, "reward/margin": 1.8203125, "reward/rejected": -3.15625, "step": 1939 }, { "approx. KL/chosen": 234.0, "approx. KL/rejected": 1272.0, "epoch": 0.5818836232753449, "grad_norm": 3.2980360984802246, "learning_rate": 3.732471153936741e-06, "logp/chosen": -378.0, "logp/rejected": -424.0, "loss": 0.4736328125, "reward/accuracy": 0.8125, "reward/chosen": -1.4375, "reward/margin": 2.921875, "reward/rejected": -4.34375, "step": 1940 }, { "approx. KL/chosen": 444.0, "approx. KL/rejected": 1256.0, "epoch": 0.5821835632873426, "grad_norm": 4.0578789710998535, "learning_rate": 3.7279141753537186e-06, "logp/chosen": -308.0, "logp/rejected": -346.0, "loss": 0.4097747802734375, "reward/accuracy": 0.75, "reward/chosen": -2.453125, "reward/margin": 1.9296875, "reward/rejected": -4.375, "step": 1941 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 656.0, "epoch": 0.5824835032993402, "grad_norm": 5.123271942138672, "learning_rate": 3.7233583262673156e-06, "logp/chosen": -240.0, "logp/rejected": -229.0, "loss": 0.608154296875, "reward/accuracy": 0.75, "reward/chosen": -1.3828125, "reward/margin": 1.640625, "reward/rejected": -3.015625, "step": 1942 }, { "approx. KL/chosen": 506.0, "approx. KL/rejected": 868.0, "epoch": 0.5827834433113377, "grad_norm": 9.359366416931152, "learning_rate": 3.7188036107227123e-06, "logp/chosen": -324.0, "logp/rejected": -312.0, "loss": 0.92315673828125, "reward/accuracy": 0.5625, "reward/chosen": -2.453125, "reward/margin": 0.9296875, "reward/rejected": -3.375, "step": 1943 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 1064.0, "epoch": 0.5830833833233353, "grad_norm": 2.4721038341522217, "learning_rate": 3.714250032764082e-06, "logp/chosen": -302.0, "logp/rejected": -284.0, "loss": 0.28266143798828125, "reward/accuracy": 0.875, "reward/chosen": -1.5703125, "reward/margin": 2.5, "reward/rejected": -4.0625, "step": 1944 }, { "approx. KL/chosen": 366.0, "approx. KL/rejected": 748.0, "epoch": 0.5833833233353329, "grad_norm": 4.091844081878662, "learning_rate": 3.709697596434587e-06, "logp/chosen": -282.0, "logp/rejected": -324.0, "loss": 0.5941619873046875, "reward/accuracy": 0.75, "reward/chosen": -1.953125, "reward/margin": 1.109375, "reward/rejected": -3.0625, "step": 1945 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 920.0, "epoch": 0.5836832633473306, "grad_norm": 4.4634690284729, "learning_rate": 3.70514630577638e-06, "logp/chosen": -308.0, "logp/rejected": -406.0, "loss": 0.569732666015625, "reward/accuracy": 0.6875, "reward/chosen": -1.703125, "reward/margin": 1.84375, "reward/rejected": -3.546875, "step": 1946 }, { "approx. KL/chosen": 252.0, "approx. KL/rejected": 1320.0, "epoch": 0.5839832033593282, "grad_norm": 2.7652664184570312, "learning_rate": 3.7005961648305898e-06, "logp/chosen": -270.0, "logp/rejected": -306.0, "loss": 0.2589273452758789, "reward/accuracy": 0.8125, "reward/chosen": -1.6328125, "reward/margin": 2.875, "reward/rejected": -4.5, "step": 1947 }, { "approx. KL/chosen": 226.0, "approx. KL/rejected": 364.0, "epoch": 0.5842831433713257, "grad_norm": 4.853309154510498, "learning_rate": 3.6960471776373307e-06, "logp/chosen": -218.0, "logp/rejected": -272.0, "loss": 0.5321044921875, "reward/accuracy": 0.6875, "reward/chosen": -1.5703125, "reward/margin": 0.78125, "reward/rejected": -2.359375, "step": 1948 }, { "approx. KL/chosen": 213.0, "approx. KL/rejected": 804.0, "epoch": 0.5845830833833233, "grad_norm": 4.98748779296875, "learning_rate": 3.6914993482356898e-06, "logp/chosen": -332.0, "logp/rejected": -310.0, "loss": 0.5615081787109375, "reward/accuracy": 0.75, "reward/chosen": -1.5234375, "reward/margin": 1.875, "reward/rejected": -3.390625, "step": 1949 }, { "approx. KL/chosen": 215.0, "approx. KL/rejected": 936.0, "epoch": 0.584883023395321, "grad_norm": 4.1334052085876465, "learning_rate": 3.6869526806637247e-06, "logp/chosen": -450.0, "logp/rejected": -394.0, "loss": 0.4071044921875, "reward/accuracy": 0.75, "reward/chosen": -1.6171875, "reward/margin": 2.109375, "reward/rejected": -3.71875, "step": 1950 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 564.0, "epoch": 0.5851829634073186, "grad_norm": 3.4161441326141357, "learning_rate": 3.6824071789584653e-06, "logp/chosen": -290.0, "logp/rejected": -246.0, "loss": 0.43792724609375, "reward/accuracy": 0.75, "reward/chosen": -1.5078125, "reward/margin": 1.296875, "reward/rejected": -2.796875, "step": 1951 }, { "approx. KL/chosen": 168.0, "approx. KL/rejected": 488.0, "epoch": 0.5854829034193162, "grad_norm": 3.464824914932251, "learning_rate": 3.677862847155902e-06, "logp/chosen": -249.0, "logp/rejected": -290.0, "loss": 0.3809814453125, "reward/accuracy": 0.875, "reward/chosen": -1.375, "reward/margin": 1.296875, "reward/rejected": -2.671875, "step": 1952 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 980.0, "epoch": 0.5857828434313137, "grad_norm": 3.743023633956909, "learning_rate": 3.6733196892909895e-06, "logp/chosen": -243.0, "logp/rejected": -236.0, "loss": 0.3428192138671875, "reward/accuracy": 0.9375, "reward/chosen": -1.5625, "reward/margin": 2.171875, "reward/rejected": -3.734375, "step": 1953 }, { "approx. KL/chosen": 332.0, "approx. KL/rejected": 532.0, "epoch": 0.5860827834433113, "grad_norm": 4.920973300933838, "learning_rate": 3.6687777093976395e-06, "logp/chosen": -244.0, "logp/rejected": -304.0, "loss": 0.7227783203125, "reward/accuracy": 0.5625, "reward/chosen": -1.984375, "reward/margin": 0.66015625, "reward/rejected": -2.640625, "step": 1954 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 492.0, "epoch": 0.586382723455309, "grad_norm": 6.1244425773620605, "learning_rate": 3.664236911508717e-06, "logp/chosen": -430.0, "logp/rejected": -398.0, "loss": 0.919647216796875, "reward/accuracy": 0.625, "reward/chosen": -1.796875, "reward/margin": 0.74609375, "reward/rejected": -2.546875, "step": 1955 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 560.0, "epoch": 0.5866826634673066, "grad_norm": 5.023801326751709, "learning_rate": 3.6596972996560375e-06, "logp/chosen": -253.0, "logp/rejected": -249.0, "loss": 0.4616546630859375, "reward/accuracy": 0.8125, "reward/chosen": -1.1953125, "reward/margin": 1.640625, "reward/rejected": -2.84375, "step": 1956 }, { "approx. KL/chosen": 105.5, "approx. KL/rejected": 1048.0, "epoch": 0.5869826034793041, "grad_norm": 2.20739483833313, "learning_rate": 3.655158877870364e-06, "logp/chosen": -470.0, "logp/rejected": -452.0, "loss": 0.21790313720703125, "reward/accuracy": 0.875, "reward/chosen": -0.953125, "reward/margin": 2.609375, "reward/rejected": -3.5625, "step": 1957 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 836.0, "epoch": 0.5872825434913017, "grad_norm": 2.8468875885009766, "learning_rate": 3.650621650181404e-06, "logp/chosen": -490.0, "logp/rejected": -374.0, "loss": 0.25568389892578125, "reward/accuracy": 0.875, "reward/chosen": -1.25, "reward/margin": 2.265625, "reward/rejected": -3.515625, "step": 1958 }, { "approx. KL/chosen": 114.0, "approx. KL/rejected": 764.0, "epoch": 0.5875824835032993, "grad_norm": 3.2605631351470947, "learning_rate": 3.6460856206178013e-06, "logp/chosen": -390.0, "logp/rejected": -316.0, "loss": 0.3314399719238281, "reward/accuracy": 0.75, "reward/chosen": -0.96484375, "reward/margin": 2.421875, "reward/rejected": -3.390625, "step": 1959 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 480.0, "epoch": 0.587882423515297, "grad_norm": 2.787111282348633, "learning_rate": 3.64155079320714e-06, "logp/chosen": -316.0, "logp/rejected": -256.0, "loss": 0.326171875, "reward/accuracy": 0.8125, "reward/chosen": -1.2578125, "reward/margin": 1.4296875, "reward/rejected": -2.6875, "step": 1960 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 600.0, "epoch": 0.5881823635272946, "grad_norm": 4.1234846115112305, "learning_rate": 3.6370171719759317e-06, "logp/chosen": -314.0, "logp/rejected": -324.0, "loss": 0.52227783203125, "reward/accuracy": 0.8125, "reward/chosen": -1.359375, "reward/margin": 1.5078125, "reward/rejected": -2.875, "step": 1961 }, { "approx. KL/chosen": 206.0, "approx. KL/rejected": 704.0, "epoch": 0.5884823035392921, "grad_norm": 3.1411068439483643, "learning_rate": 3.6324847609496228e-06, "logp/chosen": -356.0, "logp/rejected": -280.0, "loss": 0.25970458984375, "reward/accuracy": 0.875, "reward/chosen": -1.421875, "reward/margin": 1.9453125, "reward/rejected": -3.359375, "step": 1962 }, { "approx. KL/chosen": 177.0, "approx. KL/rejected": 640.0, "epoch": 0.5887822435512897, "grad_norm": 3.1157209873199463, "learning_rate": 3.627953564152583e-06, "logp/chosen": -312.0, "logp/rejected": -330.0, "loss": 0.4451904296875, "reward/accuracy": 0.8125, "reward/chosen": -1.2109375, "reward/margin": 1.59375, "reward/rejected": -2.8125, "step": 1963 }, { "approx. KL/chosen": 354.0, "approx. KL/rejected": 516.0, "epoch": 0.5890821835632873, "grad_norm": 4.983583450317383, "learning_rate": 3.6234235856081017e-06, "logp/chosen": -310.0, "logp/rejected": -398.0, "loss": 0.74871826171875, "reward/accuracy": 0.625, "reward/chosen": -1.9609375, "reward/margin": 0.625, "reward/rejected": -2.578125, "step": 1964 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 764.0, "epoch": 0.589382123575285, "grad_norm": 4.599042892456055, "learning_rate": 3.6188948293383875e-06, "logp/chosen": -239.0, "logp/rejected": -312.0, "loss": 0.41595458984375, "reward/accuracy": 0.8125, "reward/chosen": -1.6640625, "reward/margin": 1.546875, "reward/rejected": -3.203125, "step": 1965 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 1088.0, "epoch": 0.5896820635872826, "grad_norm": 2.3458826541900635, "learning_rate": 3.6143672993645657e-06, "logp/chosen": -390.0, "logp/rejected": -308.0, "loss": 0.28063201904296875, "reward/accuracy": 0.9375, "reward/chosen": -1.203125, "reward/margin": 2.5625, "reward/rejected": -3.765625, "step": 1966 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 374.0, "epoch": 0.5899820035992801, "grad_norm": 6.625907897949219, "learning_rate": 3.6098409997066706e-06, "logp/chosen": -294.0, "logp/rejected": -224.0, "loss": 0.9278564453125, "reward/accuracy": 0.625, "reward/chosen": -2.046875, "reward/margin": 0.3671875, "reward/rejected": -2.40625, "step": 1967 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 592.0, "epoch": 0.5902819436112777, "grad_norm": 8.595162391662598, "learning_rate": 3.6053159343836475e-06, "logp/chosen": -434.0, "logp/rejected": -434.0, "loss": 0.7266845703125, "reward/accuracy": 0.625, "reward/chosen": -2.265625, "reward/margin": 0.7578125, "reward/rejected": -3.03125, "step": 1968 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 748.0, "epoch": 0.5905818836232753, "grad_norm": 3.7900216579437256, "learning_rate": 3.6007921074133412e-06, "logp/chosen": -300.0, "logp/rejected": -314.0, "loss": 0.49182891845703125, "reward/accuracy": 0.75, "reward/chosen": -1.8125, "reward/margin": 1.171875, "reward/rejected": -2.984375, "step": 1969 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 680.0, "epoch": 0.590881823635273, "grad_norm": 3.3209047317504883, "learning_rate": 3.596269522812498e-06, "logp/chosen": -322.0, "logp/rejected": -338.0, "loss": 0.35825347900390625, "reward/accuracy": 0.8125, "reward/chosen": -1.5, "reward/margin": 1.671875, "reward/rejected": -3.171875, "step": 1970 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 652.0, "epoch": 0.5911817636472706, "grad_norm": 3.7932040691375732, "learning_rate": 3.591748184596764e-06, "logp/chosen": -310.0, "logp/rejected": -370.0, "loss": 0.429931640625, "reward/accuracy": 0.8125, "reward/chosen": -1.203125, "reward/margin": 1.75, "reward/rejected": -2.953125, "step": 1971 }, { "approx. KL/chosen": 206.0, "approx. KL/rejected": 724.0, "epoch": 0.5914817036592681, "grad_norm": 3.209529399871826, "learning_rate": 3.587228096780675e-06, "logp/chosen": -348.0, "logp/rejected": -398.0, "loss": 0.277801513671875, "reward/accuracy": 0.8125, "reward/chosen": -1.4375, "reward/margin": 1.875, "reward/rejected": -3.3125, "step": 1972 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 648.0, "epoch": 0.5917816436712657, "grad_norm": 6.013415813446045, "learning_rate": 3.5827092633776618e-06, "logp/chosen": -338.0, "logp/rejected": -342.0, "loss": 0.70977783203125, "reward/accuracy": 0.6875, "reward/chosen": -1.484375, "reward/margin": 1.203125, "reward/rejected": -2.6875, "step": 1973 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 466.0, "epoch": 0.5920815836832634, "grad_norm": 3.8853981494903564, "learning_rate": 3.5781916884000323e-06, "logp/chosen": -472.0, "logp/rejected": -328.0, "loss": 0.528076171875, "reward/accuracy": 0.75, "reward/chosen": -1.15625, "reward/margin": 1.2421875, "reward/rejected": -2.40625, "step": 1974 }, { "approx. KL/chosen": 420.0, "approx. KL/rejected": 568.0, "epoch": 0.592381523695261, "grad_norm": 6.3289079666137695, "learning_rate": 3.5736753758589858e-06, "logp/chosen": -362.0, "logp/rejected": -278.0, "loss": 0.867919921875, "reward/accuracy": 0.625, "reward/chosen": -1.8671875, "reward/margin": 0.9296875, "reward/rejected": -2.796875, "step": 1975 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 556.0, "epoch": 0.5926814637072585, "grad_norm": 5.667235851287842, "learning_rate": 3.5691603297645976e-06, "logp/chosen": -214.0, "logp/rejected": -288.0, "loss": 0.709716796875, "reward/accuracy": 0.5625, "reward/chosen": -1.640625, "reward/margin": 0.96484375, "reward/rejected": -2.609375, "step": 1976 }, { "approx. KL/chosen": 175.0, "approx. KL/rejected": 510.0, "epoch": 0.5929814037192561, "grad_norm": 4.105966091156006, "learning_rate": 3.564646554125819e-06, "logp/chosen": -402.0, "logp/rejected": -310.0, "loss": 0.5589599609375, "reward/accuracy": 0.8125, "reward/chosen": -1.203125, "reward/margin": 1.5625, "reward/rejected": -2.765625, "step": 1977 }, { "approx. KL/chosen": 229.0, "approx. KL/rejected": 432.0, "epoch": 0.5932813437312537, "grad_norm": 6.735074996948242, "learning_rate": 3.5601340529504712e-06, "logp/chosen": -366.0, "logp/rejected": -356.0, "loss": 0.68408203125, "reward/accuracy": 0.6875, "reward/chosen": -1.5546875, "reward/margin": 0.8203125, "reward/rejected": -2.375, "step": 1978 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 648.0, "epoch": 0.5935812837432514, "grad_norm": 4.665014743804932, "learning_rate": 3.5556228302452443e-06, "logp/chosen": -334.0, "logp/rejected": -416.0, "loss": 0.6610565185546875, "reward/accuracy": 0.6875, "reward/chosen": -1.140625, "reward/margin": 1.5, "reward/rejected": -2.640625, "step": 1979 }, { "approx. KL/chosen": 184.0, "approx. KL/rejected": 290.0, "epoch": 0.593881223755249, "grad_norm": 5.65282678604126, "learning_rate": 3.5511128900156957e-06, "logp/chosen": -310.0, "logp/rejected": -344.0, "loss": 0.70751953125, "reward/accuracy": 0.5625, "reward/chosen": -1.3671875, "reward/margin": 0.392578125, "reward/rejected": -1.7578125, "step": 1980 }, { "approx. KL/chosen": 92.5, "approx. KL/rejected": 632.0, "epoch": 0.5941811637672465, "grad_norm": 3.152987241744995, "learning_rate": 3.546604236266242e-06, "logp/chosen": -241.0, "logp/rejected": -366.0, "loss": 0.5126953125, "reward/accuracy": 0.625, "reward/chosen": -1.09375, "reward/margin": 1.484375, "reward/rejected": -2.578125, "step": 1981 }, { "approx. KL/chosen": 154.0, "approx. KL/rejected": 227.0, "epoch": 0.5944811037792441, "grad_norm": 5.713879585266113, "learning_rate": 3.5420968730001604e-06, "logp/chosen": -314.0, "logp/rejected": -268.0, "loss": 0.7518310546875, "reward/accuracy": 0.5, "reward/chosen": -1.21875, "reward/margin": 0.390625, "reward/rejected": -1.609375, "step": 1982 }, { "approx. KL/chosen": 245.0, "approx. KL/rejected": 800.0, "epoch": 0.5947810437912417, "grad_norm": 2.9899191856384277, "learning_rate": 3.537590804219575e-06, "logp/chosen": -252.0, "logp/rejected": -255.0, "loss": 0.337738037109375, "reward/accuracy": 0.875, "reward/chosen": -1.75, "reward/margin": 1.6171875, "reward/rejected": -3.359375, "step": 1983 }, { "approx. KL/chosen": 100.0, "approx. KL/rejected": 412.0, "epoch": 0.5950809838032394, "grad_norm": 8.846712112426758, "learning_rate": 3.5330860339254684e-06, "logp/chosen": -426.0, "logp/rejected": -380.0, "loss": 0.48431396484375, "reward/accuracy": 0.75, "reward/chosen": -0.79296875, "reward/margin": 1.53125, "reward/rejected": -2.328125, "step": 1984 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 454.0, "epoch": 0.595380923815237, "grad_norm": 3.527029514312744, "learning_rate": 3.5285825661176678e-06, "logp/chosen": -376.0, "logp/rejected": -436.0, "loss": 0.492919921875, "reward/accuracy": 0.8125, "reward/chosen": -1.265625, "reward/margin": 1.359375, "reward/rejected": -2.625, "step": 1985 }, { "approx. KL/chosen": 209.0, "approx. KL/rejected": 640.0, "epoch": 0.5956808638272345, "grad_norm": 5.071999549865723, "learning_rate": 3.5240804047948412e-06, "logp/chosen": -230.0, "logp/rejected": -262.0, "loss": 0.736328125, "reward/accuracy": 0.625, "reward/chosen": -1.6796875, "reward/margin": 1.046875, "reward/rejected": -2.71875, "step": 1986 }, { "approx. KL/chosen": 232.0, "approx. KL/rejected": 904.0, "epoch": 0.5959808038392321, "grad_norm": 3.325411081314087, "learning_rate": 3.519579553954503e-06, "logp/chosen": -294.0, "logp/rejected": -306.0, "loss": 0.2935791015625, "reward/accuracy": 0.875, "reward/chosen": -1.3125, "reward/margin": 2.4375, "reward/rejected": -3.765625, "step": 1987 }, { "approx. KL/chosen": 125.0, "approx. KL/rejected": 708.0, "epoch": 0.5962807438512298, "grad_norm": 3.854050874710083, "learning_rate": 3.5150800175929945e-06, "logp/chosen": -314.0, "logp/rejected": -362.0, "loss": 0.36175537109375, "reward/accuracy": 0.875, "reward/chosen": -1.25, "reward/margin": 1.953125, "reward/rejected": -3.203125, "step": 1988 }, { "approx. KL/chosen": 118.0, "approx. KL/rejected": 400.0, "epoch": 0.5965806838632274, "grad_norm": 3.504406213760376, "learning_rate": 3.5105817997054976e-06, "logp/chosen": -348.0, "logp/rejected": -378.0, "loss": 0.35009765625, "reward/accuracy": 0.875, "reward/chosen": -1.3203125, "reward/margin": 1.3125, "reward/rejected": -2.625, "step": 1989 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 362.0, "epoch": 0.596880623875225, "grad_norm": 5.246623992919922, "learning_rate": 3.5060849042860206e-06, "logp/chosen": -348.0, "logp/rejected": -340.0, "loss": 0.758209228515625, "reward/accuracy": 0.5625, "reward/chosen": -1.4921875, "reward/margin": 0.8125, "reward/rejected": -2.3125, "step": 1990 }, { "approx. KL/chosen": 244.0, "approx. KL/rejected": 532.0, "epoch": 0.5971805638872225, "grad_norm": 3.8388168811798096, "learning_rate": 3.501589335327399e-06, "logp/chosen": -330.0, "logp/rejected": -366.0, "loss": 0.4024658203125, "reward/accuracy": 0.9375, "reward/chosen": -1.6640625, "reward/margin": 1.375, "reward/rejected": -3.03125, "step": 1991 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 716.0, "epoch": 0.5974805038992201, "grad_norm": 2.3196053504943848, "learning_rate": 3.4970950968212903e-06, "logp/chosen": -233.0, "logp/rejected": -268.0, "loss": 0.2416229248046875, "reward/accuracy": 1.0, "reward/chosen": -1.6875, "reward/margin": 1.8046875, "reward/rejected": -3.5, "step": 1992 }, { "approx. KL/chosen": 92.0, "approx. KL/rejected": 346.0, "epoch": 0.5977804439112178, "grad_norm": 3.6677086353302, "learning_rate": 3.4926021927581666e-06, "logp/chosen": -282.0, "logp/rejected": -280.0, "loss": 0.41295623779296875, "reward/accuracy": 0.8125, "reward/chosen": -0.98828125, "reward/margin": 1.171875, "reward/rejected": -2.15625, "step": 1993 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 636.0, "epoch": 0.5980803839232154, "grad_norm": 3.859194040298462, "learning_rate": 3.4881106271273203e-06, "logp/chosen": -412.0, "logp/rejected": -404.0, "loss": 0.44781494140625, "reward/accuracy": 0.8125, "reward/chosen": -1.28125, "reward/margin": 1.6796875, "reward/rejected": -2.953125, "step": 1994 }, { "approx. KL/chosen": 102.0, "approx. KL/rejected": 398.0, "epoch": 0.598380323935213, "grad_norm": 3.5451090335845947, "learning_rate": 3.4836204039168534e-06, "logp/chosen": -276.0, "logp/rejected": -272.0, "loss": 0.4449462890625, "reward/accuracy": 0.6875, "reward/chosen": -1.1328125, "reward/margin": 1.3515625, "reward/rejected": -2.484375, "step": 1995 }, { "approx. KL/chosen": 85.0, "approx. KL/rejected": 560.0, "epoch": 0.5986802639472105, "grad_norm": 2.0859827995300293, "learning_rate": 3.4791315271136765e-06, "logp/chosen": -436.0, "logp/rejected": -396.0, "loss": 0.1615142822265625, "reward/accuracy": 1.0, "reward/chosen": -0.8125, "reward/margin": 2.21875, "reward/rejected": -3.03125, "step": 1996 }, { "approx. KL/chosen": 69.0, "approx. KL/rejected": 528.0, "epoch": 0.5989802039592081, "grad_norm": 1.8209011554718018, "learning_rate": 3.4746440007035067e-06, "logp/chosen": -316.0, "logp/rejected": -386.0, "loss": 0.208221435546875, "reward/accuracy": 1.0, "reward/chosen": -0.9375, "reward/margin": 2.015625, "reward/rejected": -2.953125, "step": 1997 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 470.0, "epoch": 0.5992801439712058, "grad_norm": 3.185243606567383, "learning_rate": 3.4701578286708536e-06, "logp/chosen": -412.0, "logp/rejected": -340.0, "loss": 0.4283447265625, "reward/accuracy": 0.8125, "reward/chosen": -1.2734375, "reward/margin": 1.28125, "reward/rejected": -2.5625, "step": 1998 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 672.0, "epoch": 0.5995800839832034, "grad_norm": 3.4738223552703857, "learning_rate": 3.4656730149990346e-06, "logp/chosen": -324.0, "logp/rejected": -388.0, "loss": 0.458282470703125, "reward/accuracy": 0.75, "reward/chosen": -1.0546875, "reward/margin": 2.03125, "reward/rejected": -3.078125, "step": 1999 }, { "approx. KL/chosen": 84.5, "approx. KL/rejected": 436.0, "epoch": 0.5998800239952009, "grad_norm": 3.079333782196045, "learning_rate": 3.4611895636701555e-06, "logp/chosen": -268.0, "logp/rejected": -302.0, "loss": 0.4583740234375, "reward/accuracy": 0.75, "reward/chosen": -1.0234375, "reward/margin": 1.421875, "reward/rejected": -2.453125, "step": 2000 }, { "approx. KL/chosen": 215.0, "approx. KL/rejected": 496.0, "epoch": 0.6001799640071985, "grad_norm": 3.4159092903137207, "learning_rate": 3.4567074786651166e-06, "logp/chosen": -346.0, "logp/rejected": -424.0, "loss": 0.3912353515625, "reward/accuracy": 0.75, "reward/chosen": -1.3671875, "reward/margin": 1.140625, "reward/rejected": -2.5, "step": 2001 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 392.0, "epoch": 0.6004799040191962, "grad_norm": 5.6891984939575195, "learning_rate": 3.452226763963597e-06, "logp/chosen": -302.0, "logp/rejected": -484.0, "loss": 0.82080078125, "reward/accuracy": 0.75, "reward/chosen": -1.625, "reward/margin": 0.6015625, "reward/rejected": -2.21875, "step": 2002 }, { "approx. KL/chosen": 80.0, "approx. KL/rejected": 848.0, "epoch": 0.6007798440311938, "grad_norm": 2.9184794425964355, "learning_rate": 3.447747423544067e-06, "logp/chosen": -334.0, "logp/rejected": -376.0, "loss": 0.33840370178222656, "reward/accuracy": 0.8125, "reward/chosen": -1.015625, "reward/margin": 2.109375, "reward/rejected": -3.125, "step": 2003 }, { "approx. KL/chosen": 400.0, "approx. KL/rejected": 588.0, "epoch": 0.6010797840431914, "grad_norm": 5.15627384185791, "learning_rate": 3.443269461383773e-06, "logp/chosen": -438.0, "logp/rejected": -294.0, "loss": 0.616485595703125, "reward/accuracy": 0.6875, "reward/chosen": -2.171875, "reward/margin": 1.0234375, "reward/rejected": -3.203125, "step": 2004 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 544.0, "epoch": 0.6013797240551889, "grad_norm": 3.9091343879699707, "learning_rate": 3.438792881458739e-06, "logp/chosen": -382.0, "logp/rejected": -370.0, "loss": 0.47126007080078125, "reward/accuracy": 0.8125, "reward/chosen": -1.109375, "reward/margin": 1.8125, "reward/rejected": -2.921875, "step": 2005 }, { "approx. KL/chosen": 112.5, "approx. KL/rejected": 792.0, "epoch": 0.6016796640671865, "grad_norm": 2.9379982948303223, "learning_rate": 3.4343176877437646e-06, "logp/chosen": -288.0, "logp/rejected": -328.0, "loss": 0.2453155517578125, "reward/accuracy": 0.875, "reward/chosen": -1.1015625, "reward/margin": 2.40625, "reward/rejected": -3.515625, "step": 2006 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 392.0, "epoch": 0.6019796040791842, "grad_norm": 3.761068820953369, "learning_rate": 3.429843884212408e-06, "logp/chosen": -266.0, "logp/rejected": -280.0, "loss": 0.6031494140625, "reward/accuracy": 0.5625, "reward/chosen": -1.625, "reward/margin": 0.69140625, "reward/rejected": -2.328125, "step": 2007 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 828.0, "epoch": 0.6022795440911818, "grad_norm": 3.0737457275390625, "learning_rate": 3.4253714748370053e-06, "logp/chosen": -374.0, "logp/rejected": -390.0, "loss": 0.248809814453125, "reward/accuracy": 0.9375, "reward/chosen": -1.328125, "reward/margin": 2.109375, "reward/rejected": -3.421875, "step": 2008 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 664.0, "epoch": 0.6025794841031794, "grad_norm": 5.412962436676025, "learning_rate": 3.420900463588648e-06, "logp/chosen": -256.0, "logp/rejected": -216.0, "loss": 0.36627197265625, "reward/accuracy": 0.9375, "reward/chosen": -0.8671875, "reward/margin": 2.1875, "reward/rejected": -3.0625, "step": 2009 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 352.0, "epoch": 0.6028794241151769, "grad_norm": 4.77442741394043, "learning_rate": 3.4164308544371884e-06, "logp/chosen": -350.0, "logp/rejected": -292.0, "loss": 0.6142578125, "reward/accuracy": 0.5, "reward/chosen": -1.6328125, "reward/margin": 0.5546875, "reward/rejected": -2.1875, "step": 2010 }, { "approx. KL/chosen": 125.5, "approx. KL/rejected": 476.0, "epoch": 0.6031793641271745, "grad_norm": 3.078526496887207, "learning_rate": 3.4119626513512343e-06, "logp/chosen": -312.0, "logp/rejected": -376.0, "loss": 0.393798828125, "reward/accuracy": 0.8125, "reward/chosen": -1.390625, "reward/margin": 1.21875, "reward/rejected": -2.609375, "step": 2011 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 366.0, "epoch": 0.6034793041391722, "grad_norm": 4.298441410064697, "learning_rate": 3.4074958582981416e-06, "logp/chosen": -466.0, "logp/rejected": -456.0, "loss": 0.5693740844726562, "reward/accuracy": 0.625, "reward/chosen": -1.28125, "reward/margin": 0.9140625, "reward/rejected": -2.203125, "step": 2012 }, { "approx. KL/chosen": 241.0, "approx. KL/rejected": 536.0, "epoch": 0.6037792441511698, "grad_norm": 4.672429084777832, "learning_rate": 3.403030479244017e-06, "logp/chosen": -290.0, "logp/rejected": -312.0, "loss": 0.3900146484375, "reward/accuracy": 0.75, "reward/chosen": -1.5625, "reward/margin": 1.4375, "reward/rejected": -3.0, "step": 2013 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 716.0, "epoch": 0.6040791841631674, "grad_norm": 4.1968889236450195, "learning_rate": 3.398566518153713e-06, "logp/chosen": -292.0, "logp/rejected": -314.0, "loss": 0.54205322265625, "reward/accuracy": 0.625, "reward/chosen": -1.1953125, "reward/margin": 1.6015625, "reward/rejected": -2.796875, "step": 2014 }, { "approx. KL/chosen": 237.0, "approx. KL/rejected": 904.0, "epoch": 0.6043791241751649, "grad_norm": 4.675299644470215, "learning_rate": 3.39410397899082e-06, "logp/chosen": -374.0, "logp/rejected": -294.0, "loss": 0.4610443115234375, "reward/accuracy": 0.75, "reward/chosen": -1.7265625, "reward/margin": 1.9453125, "reward/rejected": -3.671875, "step": 2015 }, { "approx. KL/chosen": 221.0, "approx. KL/rejected": 183.0, "epoch": 0.6046790641871626, "grad_norm": 4.820202827453613, "learning_rate": 3.3896428657176695e-06, "logp/chosen": -356.0, "logp/rejected": -300.0, "loss": 0.75634765625, "reward/accuracy": 0.6875, "reward/chosen": -1.3203125, "reward/margin": 0.4453125, "reward/rejected": -1.7578125, "step": 2016 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 860.0, "epoch": 0.6049790041991602, "grad_norm": 2.622840166091919, "learning_rate": 3.385183182295322e-06, "logp/chosen": -404.0, "logp/rejected": -436.0, "loss": 0.18994140625, "reward/accuracy": 1.0, "reward/chosen": -1.234375, "reward/margin": 2.53125, "reward/rejected": -3.75, "step": 2017 }, { "approx. KL/chosen": 241.0, "approx. KL/rejected": 640.0, "epoch": 0.6052789442111578, "grad_norm": 4.2666215896606445, "learning_rate": 3.380724932683571e-06, "logp/chosen": -330.0, "logp/rejected": -352.0, "loss": 0.41534423828125, "reward/accuracy": 0.8125, "reward/chosen": -1.75, "reward/margin": 1.3984375, "reward/rejected": -3.15625, "step": 2018 }, { "approx. KL/chosen": 215.0, "approx. KL/rejected": 668.0, "epoch": 0.6055788842231554, "grad_norm": 3.7374613285064697, "learning_rate": 3.376268120840939e-06, "logp/chosen": -360.0, "logp/rejected": -446.0, "loss": 0.48382568359375, "reward/accuracy": 0.6875, "reward/chosen": -1.6015625, "reward/margin": 1.53125, "reward/rejected": -3.140625, "step": 2019 }, { "approx. KL/chosen": 488.0, "approx. KL/rejected": 720.0, "epoch": 0.6058788242351529, "grad_norm": 6.0031514167785645, "learning_rate": 3.371812750724669e-06, "logp/chosen": -222.0, "logp/rejected": -235.0, "loss": 0.8612060546875, "reward/accuracy": 0.5, "reward/chosen": -2.234375, "reward/margin": 0.8125, "reward/rejected": -3.046875, "step": 2020 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 852.0, "epoch": 0.6061787642471506, "grad_norm": 3.7380969524383545, "learning_rate": 3.3673588262907268e-06, "logp/chosen": -278.0, "logp/rejected": -314.0, "loss": 0.42279052734375, "reward/accuracy": 0.8125, "reward/chosen": -1.6171875, "reward/margin": 2.171875, "reward/rejected": -3.78125, "step": 2021 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 812.0, "epoch": 0.6064787042591482, "grad_norm": 3.303807258605957, "learning_rate": 3.3629063514937887e-06, "logp/chosen": -444.0, "logp/rejected": -490.0, "loss": 0.3272705078125, "reward/accuracy": 0.875, "reward/chosen": -1.4921875, "reward/margin": 1.7890625, "reward/rejected": -3.28125, "step": 2022 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 576.0, "epoch": 0.6067786442711458, "grad_norm": 3.1799750328063965, "learning_rate": 3.35845533028725e-06, "logp/chosen": -382.0, "logp/rejected": -350.0, "loss": 0.370941162109375, "reward/accuracy": 0.8125, "reward/chosen": -1.828125, "reward/margin": 1.375, "reward/rejected": -3.203125, "step": 2023 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 788.0, "epoch": 0.6070785842831433, "grad_norm": 4.449514389038086, "learning_rate": 3.354005766623213e-06, "logp/chosen": -304.0, "logp/rejected": -266.0, "loss": 0.427734375, "reward/accuracy": 0.875, "reward/chosen": -1.6015625, "reward/margin": 1.90625, "reward/rejected": -3.5, "step": 2024 }, { "approx. KL/chosen": 184.0, "approx. KL/rejected": 492.0, "epoch": 0.6073785242951409, "grad_norm": 5.0773844718933105, "learning_rate": 3.3495576644524874e-06, "logp/chosen": -308.0, "logp/rejected": -404.0, "loss": 0.60565185546875, "reward/accuracy": 0.6875, "reward/chosen": -1.5859375, "reward/margin": 1.0859375, "reward/rejected": -2.671875, "step": 2025 }, { "approx. KL/chosen": 171.0, "approx. KL/rejected": 604.0, "epoch": 0.6076784643071386, "grad_norm": 3.0450754165649414, "learning_rate": 3.34511102772458e-06, "logp/chosen": -444.0, "logp/rejected": -390.0, "loss": 0.31707763671875, "reward/accuracy": 0.875, "reward/chosen": -1.3671875, "reward/margin": 1.625, "reward/rejected": -3.0, "step": 2026 }, { "approx. KL/chosen": 588.0, "approx. KL/rejected": 1112.0, "epoch": 0.6079784043191362, "grad_norm": 7.6224894523620605, "learning_rate": 3.3406658603877017e-06, "logp/chosen": -378.0, "logp/rejected": -374.0, "loss": 1.000732421875, "reward/accuracy": 0.625, "reward/chosen": -2.640625, "reward/margin": 1.1796875, "reward/rejected": -3.8125, "step": 2027 }, { "approx. KL/chosen": 324.0, "approx. KL/rejected": 792.0, "epoch": 0.6082783443311338, "grad_norm": 4.018031597137451, "learning_rate": 3.3362221663887586e-06, "logp/chosen": -310.0, "logp/rejected": -282.0, "loss": 0.642242431640625, "reward/accuracy": 0.6875, "reward/chosen": -1.8984375, "reward/margin": 1.671875, "reward/rejected": -3.578125, "step": 2028 }, { "approx. KL/chosen": 175.0, "approx. KL/rejected": 528.0, "epoch": 0.6085782843431313, "grad_norm": 4.901126861572266, "learning_rate": 3.331779949673346e-06, "logp/chosen": -466.0, "logp/rejected": -342.0, "loss": 0.7479248046875, "reward/accuracy": 0.6875, "reward/chosen": -1.4140625, "reward/margin": 1.15625, "reward/rejected": -2.5625, "step": 2029 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 784.0, "epoch": 0.608878224355129, "grad_norm": 4.627245903015137, "learning_rate": 3.327339214185751e-06, "logp/chosen": -288.0, "logp/rejected": -410.0, "loss": 0.5799713134765625, "reward/accuracy": 0.6875, "reward/chosen": -2.125, "reward/margin": 1.28125, "reward/rejected": -3.421875, "step": 2030 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 456.0, "epoch": 0.6091781643671266, "grad_norm": 4.098769664764404, "learning_rate": 3.3228999638689384e-06, "logp/chosen": -402.0, "logp/rejected": -324.0, "loss": 0.5279541015625, "reward/accuracy": 0.75, "reward/chosen": -1.5546875, "reward/margin": 1.0859375, "reward/rejected": -2.640625, "step": 2031 }, { "approx. KL/chosen": 253.0, "approx. KL/rejected": 1016.0, "epoch": 0.6094781043791242, "grad_norm": 3.540306806564331, "learning_rate": 3.318462202664563e-06, "logp/chosen": -330.0, "logp/rejected": -346.0, "loss": 0.39281463623046875, "reward/accuracy": 0.8125, "reward/chosen": -1.8515625, "reward/margin": 2.0, "reward/rejected": -3.859375, "step": 2032 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 668.0, "epoch": 0.6097780443911218, "grad_norm": 4.248137474060059, "learning_rate": 3.314025934512951e-06, "logp/chosen": -318.0, "logp/rejected": -294.0, "loss": 0.54736328125, "reward/accuracy": 0.75, "reward/chosen": -1.453125, "reward/margin": 1.6796875, "reward/rejected": -3.140625, "step": 2033 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 832.0, "epoch": 0.6100779844031193, "grad_norm": 2.1294286251068115, "learning_rate": 3.309591163353108e-06, "logp/chosen": -376.0, "logp/rejected": -382.0, "loss": 0.28497314453125, "reward/accuracy": 0.875, "reward/chosen": -1.5, "reward/margin": 2.140625, "reward/rejected": -3.640625, "step": 2034 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 1328.0, "epoch": 0.610377924415117, "grad_norm": 4.535073757171631, "learning_rate": 3.305157893122708e-06, "logp/chosen": -298.0, "logp/rejected": -350.0, "loss": 0.5379180908203125, "reward/accuracy": 0.625, "reward/chosen": -1.96875, "reward/margin": 2.171875, "reward/rejected": -4.125, "step": 2035 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 1232.0, "epoch": 0.6106778644271146, "grad_norm": 2.9246268272399902, "learning_rate": 3.300726127758088e-06, "logp/chosen": -370.0, "logp/rejected": -428.0, "loss": 0.2588768005371094, "reward/accuracy": 0.875, "reward/chosen": -1.7109375, "reward/margin": 2.75, "reward/rejected": -4.46875, "step": 2036 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 1104.0, "epoch": 0.6109778044391122, "grad_norm": 2.8136658668518066, "learning_rate": 3.2962958711942565e-06, "logp/chosen": -352.0, "logp/rejected": -432.0, "loss": 0.26666259765625, "reward/accuracy": 0.8125, "reward/chosen": -2.03125, "reward/margin": 2.140625, "reward/rejected": -4.1875, "step": 2037 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 780.0, "epoch": 0.6112777444511098, "grad_norm": 2.8131752014160156, "learning_rate": 3.2918671273648777e-06, "logp/chosen": -402.0, "logp/rejected": -328.0, "loss": 0.37127685546875, "reward/accuracy": 0.875, "reward/chosen": -2.125, "reward/margin": 1.5078125, "reward/rejected": -3.625, "step": 2038 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 636.0, "epoch": 0.6115776844631073, "grad_norm": 3.843963384628296, "learning_rate": 3.287439900202274e-06, "logp/chosen": -322.0, "logp/rejected": -378.0, "loss": 0.51922607421875, "reward/accuracy": 0.8125, "reward/chosen": -1.65625, "reward/margin": 1.5390625, "reward/rejected": -3.203125, "step": 2039 }, { "approx. KL/chosen": 310.0, "approx. KL/rejected": 576.0, "epoch": 0.611877624475105, "grad_norm": 7.539618492126465, "learning_rate": 3.2830141936374222e-06, "logp/chosen": -374.0, "logp/rejected": -358.0, "loss": 0.70330810546875, "reward/accuracy": 0.75, "reward/chosen": -1.8125, "reward/margin": 1.0625, "reward/rejected": -2.875, "step": 2040 }, { "approx. KL/chosen": 348.0, "approx. KL/rejected": 692.0, "epoch": 0.6121775644871026, "grad_norm": 5.909560680389404, "learning_rate": 3.278590011599944e-06, "logp/chosen": -444.0, "logp/rejected": -380.0, "loss": 0.66339111328125, "reward/accuracy": 0.625, "reward/chosen": -2.34375, "reward/margin": 0.8984375, "reward/rejected": -3.234375, "step": 2041 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 478.0, "epoch": 0.6124775044991002, "grad_norm": 4.5501251220703125, "learning_rate": 3.2741673580181133e-06, "logp/chosen": -404.0, "logp/rejected": -528.0, "loss": 0.526336669921875, "reward/accuracy": 0.75, "reward/chosen": -1.25, "reward/margin": 1.34375, "reward/rejected": -2.59375, "step": 2042 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 880.0, "epoch": 0.6127774445110978, "grad_norm": 3.9431002140045166, "learning_rate": 3.269746236818844e-06, "logp/chosen": -292.0, "logp/rejected": -342.0, "loss": 0.472808837890625, "reward/accuracy": 0.75, "reward/chosen": -1.9296875, "reward/margin": 1.640625, "reward/rejected": -3.5625, "step": 2043 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 556.0, "epoch": 0.6130773845230953, "grad_norm": 4.434960842132568, "learning_rate": 3.2653266519276905e-06, "logp/chosen": -342.0, "logp/rejected": -418.0, "loss": 0.4923095703125, "reward/accuracy": 0.6875, "reward/chosen": -1.7578125, "reward/margin": 1.09375, "reward/rejected": -2.859375, "step": 2044 }, { "approx. KL/chosen": 254.0, "approx. KL/rejected": 848.0, "epoch": 0.613377324535093, "grad_norm": 4.090981483459473, "learning_rate": 3.2609086072688444e-06, "logp/chosen": -372.0, "logp/rejected": -398.0, "loss": 0.382232666015625, "reward/accuracy": 0.875, "reward/chosen": -1.6171875, "reward/margin": 1.953125, "reward/rejected": -3.5625, "step": 2045 }, { "approx. KL/chosen": 237.0, "approx. KL/rejected": 752.0, "epoch": 0.6136772645470906, "grad_norm": 3.1299564838409424, "learning_rate": 3.256492106765124e-06, "logp/chosen": -432.0, "logp/rejected": -414.0, "loss": 0.34619140625, "reward/accuracy": 0.8125, "reward/chosen": -1.765625, "reward/margin": 1.453125, "reward/rejected": -3.21875, "step": 2046 }, { "approx. KL/chosen": 167.0, "approx. KL/rejected": 688.0, "epoch": 0.6139772045590882, "grad_norm": 4.297765254974365, "learning_rate": 3.2520771543379828e-06, "logp/chosen": -320.0, "logp/rejected": -304.0, "loss": 0.565185546875, "reward/accuracy": 0.75, "reward/chosen": -1.4453125, "reward/margin": 1.2734375, "reward/rejected": -2.71875, "step": 2047 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 544.0, "epoch": 0.6142771445710857, "grad_norm": 4.105167388916016, "learning_rate": 3.2476637539074962e-06, "logp/chosen": -382.0, "logp/rejected": -424.0, "loss": 0.4451904296875, "reward/accuracy": 0.75, "reward/chosen": -1.4921875, "reward/margin": 1.328125, "reward/rejected": -2.8125, "step": 2048 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 996.0, "epoch": 0.6145770845830834, "grad_norm": 3.6624324321746826, "learning_rate": 3.2432519093923654e-06, "logp/chosen": -368.0, "logp/rejected": -358.0, "loss": 0.337982177734375, "reward/accuracy": 0.8125, "reward/chosen": -2.109375, "reward/margin": 1.9453125, "reward/rejected": -4.0625, "step": 2049 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 624.0, "epoch": 0.614877024595081, "grad_norm": 4.107011795043945, "learning_rate": 3.2388416247099034e-06, "logp/chosen": -312.0, "logp/rejected": -324.0, "loss": 0.587646484375, "reward/accuracy": 0.75, "reward/chosen": -1.8515625, "reward/margin": 1.109375, "reward/rejected": -2.96875, "step": 2050 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 544.0, "epoch": 0.6151769646070786, "grad_norm": 3.9142990112304688, "learning_rate": 3.2344329037760447e-06, "logp/chosen": -352.0, "logp/rejected": -346.0, "loss": 0.53631591796875, "reward/accuracy": 0.6875, "reward/chosen": -1.5859375, "reward/margin": 1.1171875, "reward/rejected": -2.703125, "step": 2051 }, { "approx. KL/chosen": 255.0, "approx. KL/rejected": 716.0, "epoch": 0.6154769046190762, "grad_norm": 4.843515872955322, "learning_rate": 3.2300257505053322e-06, "logp/chosen": -318.0, "logp/rejected": -266.0, "loss": 0.5272216796875, "reward/accuracy": 0.75, "reward/chosen": -1.765625, "reward/margin": 1.6640625, "reward/rejected": -3.421875, "step": 2052 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 808.0, "epoch": 0.6157768446310737, "grad_norm": 6.3553314208984375, "learning_rate": 3.2256201688109184e-06, "logp/chosen": -430.0, "logp/rejected": -358.0, "loss": 0.34454345703125, "reward/accuracy": 0.875, "reward/chosen": -1.6875, "reward/margin": 1.703125, "reward/rejected": -3.390625, "step": 2053 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 1040.0, "epoch": 0.6160767846430714, "grad_norm": 3.270047426223755, "learning_rate": 3.2212161626045606e-06, "logp/chosen": -430.0, "logp/rejected": -446.0, "loss": 0.334136962890625, "reward/accuracy": 0.75, "reward/chosen": -1.7265625, "reward/margin": 2.140625, "reward/rejected": -3.875, "step": 2054 }, { "approx. KL/chosen": 168.0, "approx. KL/rejected": 700.0, "epoch": 0.616376724655069, "grad_norm": 2.6761362552642822, "learning_rate": 3.2168137357966123e-06, "logp/chosen": -402.0, "logp/rejected": -368.0, "loss": 0.27423095703125, "reward/accuracy": 0.875, "reward/chosen": -1.1796875, "reward/margin": 2.0625, "reward/rejected": -3.234375, "step": 2055 }, { "approx. KL/chosen": 252.0, "approx. KL/rejected": 748.0, "epoch": 0.6166766646670666, "grad_norm": 3.6031973361968994, "learning_rate": 3.2124128922960305e-06, "logp/chosen": -370.0, "logp/rejected": -330.0, "loss": 0.4659576416015625, "reward/accuracy": 0.875, "reward/chosen": -1.515625, "reward/margin": 1.84375, "reward/rejected": -3.359375, "step": 2056 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 552.0, "epoch": 0.6169766046790642, "grad_norm": 4.06423807144165, "learning_rate": 3.2080136360103655e-06, "logp/chosen": -304.0, "logp/rejected": -312.0, "loss": 0.46026611328125, "reward/accuracy": 0.75, "reward/chosen": -1.71875, "reward/margin": 1.28125, "reward/rejected": -3.0, "step": 2057 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 748.0, "epoch": 0.6172765446910617, "grad_norm": 5.268946170806885, "learning_rate": 3.203615970845755e-06, "logp/chosen": -312.0, "logp/rejected": -326.0, "loss": 0.430267333984375, "reward/accuracy": 0.8125, "reward/chosen": -1.984375, "reward/margin": 1.5859375, "reward/rejected": -3.5625, "step": 2058 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 832.0, "epoch": 0.6175764847030594, "grad_norm": 5.054721355438232, "learning_rate": 3.199219900706929e-06, "logp/chosen": -270.0, "logp/rejected": -326.0, "loss": 0.5418548583984375, "reward/accuracy": 0.6875, "reward/chosen": -1.7421875, "reward/margin": 1.7109375, "reward/rejected": -3.453125, "step": 2059 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 716.0, "epoch": 0.617876424715057, "grad_norm": 5.05187463760376, "learning_rate": 3.1948254294971953e-06, "logp/chosen": -416.0, "logp/rejected": -410.0, "loss": 0.651611328125, "reward/accuracy": 0.6875, "reward/chosen": -1.5, "reward/margin": 1.625, "reward/rejected": -3.125, "step": 2060 }, { "approx. KL/chosen": 125.5, "approx. KL/rejected": 696.0, "epoch": 0.6181763647270546, "grad_norm": 3.2988882064819336, "learning_rate": 3.190432561118446e-06, "logp/chosen": -390.0, "logp/rejected": -332.0, "loss": 0.395233154296875, "reward/accuracy": 0.75, "reward/chosen": -1.25, "reward/margin": 1.9296875, "reward/rejected": -3.171875, "step": 2061 }, { "approx. KL/chosen": 340.0, "approx. KL/rejected": 592.0, "epoch": 0.6184763047390522, "grad_norm": 4.935662269592285, "learning_rate": 3.1860412994711486e-06, "logp/chosen": -396.0, "logp/rejected": -444.0, "loss": 0.8187255859375, "reward/accuracy": 0.5625, "reward/chosen": -2.3125, "reward/margin": 0.640625, "reward/rejected": -2.953125, "step": 2062 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 808.0, "epoch": 0.6187762447510498, "grad_norm": 3.3700790405273438, "learning_rate": 3.181651648454346e-06, "logp/chosen": -278.0, "logp/rejected": -330.0, "loss": 0.39208984375, "reward/accuracy": 0.8125, "reward/chosen": -2.046875, "reward/margin": 1.4453125, "reward/rejected": -3.5, "step": 2063 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 936.0, "epoch": 0.6190761847630474, "grad_norm": 1.93683922290802, "learning_rate": 3.1772636119656496e-06, "logp/chosen": -358.0, "logp/rejected": -344.0, "loss": 0.1888885498046875, "reward/accuracy": 1.0, "reward/chosen": -1.4921875, "reward/margin": 2.25, "reward/rejected": -3.75, "step": 2064 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 744.0, "epoch": 0.619376124775045, "grad_norm": 3.245903968811035, "learning_rate": 3.1728771939012336e-06, "logp/chosen": -280.0, "logp/rejected": -241.0, "loss": 0.32611083984375, "reward/accuracy": 0.75, "reward/chosen": -1.6875, "reward/margin": 1.7265625, "reward/rejected": -3.40625, "step": 2065 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 616.0, "epoch": 0.6196760647870426, "grad_norm": 3.9340124130249023, "learning_rate": 3.1684923981558418e-06, "logp/chosen": -292.0, "logp/rejected": -410.0, "loss": 0.4061279296875, "reward/accuracy": 0.75, "reward/chosen": -1.296875, "reward/margin": 1.515625, "reward/rejected": -2.8125, "step": 2066 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 740.0, "epoch": 0.6199760047990401, "grad_norm": 4.143760681152344, "learning_rate": 3.1641092286227737e-06, "logp/chosen": -334.0, "logp/rejected": -384.0, "loss": 0.44866943359375, "reward/accuracy": 0.8125, "reward/chosen": -1.4296875, "reward/margin": 1.859375, "reward/rejected": -3.296875, "step": 2067 }, { "approx. KL/chosen": 314.0, "approx. KL/rejected": 912.0, "epoch": 0.6202759448110378, "grad_norm": 3.9922802448272705, "learning_rate": 3.1597276891938867e-06, "logp/chosen": -346.0, "logp/rejected": -346.0, "loss": 0.3033447265625, "reward/accuracy": 0.75, "reward/chosen": -2.265625, "reward/margin": 1.7109375, "reward/rejected": -3.984375, "step": 2068 }, { "approx. KL/chosen": 238.0, "approx. KL/rejected": 1368.0, "epoch": 0.6205758848230354, "grad_norm": 4.055628776550293, "learning_rate": 3.155347783759587e-06, "logp/chosen": -312.0, "logp/rejected": -372.0, "loss": 0.4100341796875, "reward/accuracy": 0.75, "reward/chosen": -1.609375, "reward/margin": 2.390625, "reward/rejected": -4.0, "step": 2069 }, { "approx. KL/chosen": 171.0, "approx. KL/rejected": 644.0, "epoch": 0.620875824835033, "grad_norm": 2.2541098594665527, "learning_rate": 3.1509695162088334e-06, "logp/chosen": -254.0, "logp/rejected": -199.0, "loss": 0.2603759765625, "reward/accuracy": 1.0, "reward/chosen": -1.515625, "reward/margin": 1.8125, "reward/rejected": -3.328125, "step": 2070 }, { "approx. KL/chosen": 233.0, "approx. KL/rejected": 720.0, "epoch": 0.6211757648470306, "grad_norm": 4.370351791381836, "learning_rate": 3.1465928904291305e-06, "logp/chosen": -328.0, "logp/rejected": -366.0, "loss": 0.448333740234375, "reward/accuracy": 0.8125, "reward/chosen": -1.609375, "reward/margin": 1.7734375, "reward/rejected": -3.375, "step": 2071 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 764.0, "epoch": 0.6214757048590281, "grad_norm": 4.35134220123291, "learning_rate": 3.1422179103065236e-06, "logp/chosen": -324.0, "logp/rejected": -326.0, "loss": 0.41143798828125, "reward/accuracy": 0.75, "reward/chosen": -1.8515625, "reward/margin": 1.6015625, "reward/rejected": -3.453125, "step": 2072 }, { "approx. KL/chosen": 380.0, "approx. KL/rejected": 1000.0, "epoch": 0.6217756448710258, "grad_norm": 4.446979522705078, "learning_rate": 3.137844579725601e-06, "logp/chosen": -352.0, "logp/rejected": -366.0, "loss": 0.429962158203125, "reward/accuracy": 0.75, "reward/chosen": -2.28125, "reward/margin": 1.8046875, "reward/rejected": -4.09375, "step": 2073 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 1344.0, "epoch": 0.6220755848830234, "grad_norm": 3.9912405014038086, "learning_rate": 3.1334729025694767e-06, "logp/chosen": -350.0, "logp/rejected": -436.0, "loss": 0.5345687866210938, "reward/accuracy": 0.75, "reward/chosen": -1.34375, "reward/margin": 2.515625, "reward/rejected": -3.875, "step": 2074 }, { "approx. KL/chosen": 175.0, "approx. KL/rejected": 928.0, "epoch": 0.622375524895021, "grad_norm": 2.9426167011260986, "learning_rate": 3.129102882719806e-06, "logp/chosen": -162.0, "logp/rejected": -220.0, "loss": 0.3863525390625, "reward/accuracy": 0.75, "reward/chosen": -1.4375, "reward/margin": 2.109375, "reward/rejected": -3.5625, "step": 2075 }, { "approx. KL/chosen": 243.0, "approx. KL/rejected": 704.0, "epoch": 0.6226754649070186, "grad_norm": 5.526017665863037, "learning_rate": 3.1247345240567696e-06, "logp/chosen": -348.0, "logp/rejected": -402.0, "loss": 0.741729736328125, "reward/accuracy": 0.5625, "reward/chosen": -1.8671875, "reward/margin": 1.1015625, "reward/rejected": -2.96875, "step": 2076 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 580.0, "epoch": 0.6229754049190162, "grad_norm": 4.1507887840271, "learning_rate": 3.120367830459072e-06, "logp/chosen": -268.0, "logp/rejected": -288.0, "loss": 0.4722900390625, "reward/accuracy": 0.75, "reward/chosen": -1.96875, "reward/margin": 1.1484375, "reward/rejected": -3.125, "step": 2077 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 532.0, "epoch": 0.6232753449310138, "grad_norm": 3.926478385925293, "learning_rate": 3.116002805803945e-06, "logp/chosen": -402.0, "logp/rejected": -264.0, "loss": 0.4732666015625, "reward/accuracy": 0.875, "reward/chosen": -1.71875, "reward/margin": 1.25, "reward/rejected": -2.96875, "step": 2078 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 1456.0, "epoch": 0.6235752849430114, "grad_norm": 2.192455291748047, "learning_rate": 3.1116394539671256e-06, "logp/chosen": -370.0, "logp/rejected": -548.0, "loss": 0.2169647216796875, "reward/accuracy": 0.9375, "reward/chosen": -1.84375, "reward/margin": 2.640625, "reward/rejected": -4.46875, "step": 2079 }, { "approx. KL/chosen": 232.0, "approx. KL/rejected": 438.0, "epoch": 0.623875224955009, "grad_norm": 5.796082019805908, "learning_rate": 3.1072777788228793e-06, "logp/chosen": -422.0, "logp/rejected": -280.0, "loss": 0.49176025390625, "reward/accuracy": 0.6875, "reward/chosen": -1.5546875, "reward/margin": 1.09375, "reward/rejected": -2.65625, "step": 2080 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 652.0, "epoch": 0.6241751649670066, "grad_norm": 3.0972068309783936, "learning_rate": 3.102917784243976e-06, "logp/chosen": -354.0, "logp/rejected": -334.0, "loss": 0.3368072509765625, "reward/accuracy": 0.8125, "reward/chosen": -1.2109375, "reward/margin": 2.078125, "reward/rejected": -3.296875, "step": 2081 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 848.0, "epoch": 0.6244751049790042, "grad_norm": 3.6517081260681152, "learning_rate": 3.0985594741016944e-06, "logp/chosen": -414.0, "logp/rejected": -326.0, "loss": 0.527130126953125, "reward/accuracy": 0.75, "reward/chosen": -1.3203125, "reward/margin": 1.859375, "reward/rejected": -3.1875, "step": 2082 }, { "approx. KL/chosen": 109.5, "approx. KL/rejected": 640.0, "epoch": 0.6247750449910018, "grad_norm": 3.5708160400390625, "learning_rate": 3.0942028522658202e-06, "logp/chosen": -306.0, "logp/rejected": -300.0, "loss": 0.37320709228515625, "reward/accuracy": 0.8125, "reward/chosen": -1.0234375, "reward/margin": 1.828125, "reward/rejected": -2.84375, "step": 2083 }, { "approx. KL/chosen": 310.0, "approx. KL/rejected": 992.0, "epoch": 0.6250749850029994, "grad_norm": 5.762585639953613, "learning_rate": 3.089847922604634e-06, "logp/chosen": -372.0, "logp/rejected": -460.0, "loss": 0.4817047119140625, "reward/accuracy": 0.6875, "reward/chosen": -1.828125, "reward/margin": 2.0625, "reward/rejected": -3.890625, "step": 2084 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 840.0, "epoch": 0.625374925014997, "grad_norm": 3.270951271057129, "learning_rate": 3.085494688984919e-06, "logp/chosen": -227.0, "logp/rejected": -256.0, "loss": 0.38739013671875, "reward/accuracy": 0.8125, "reward/chosen": -1.7265625, "reward/margin": 1.8359375, "reward/rejected": -3.5625, "step": 2085 }, { "approx. KL/chosen": 328.0, "approx. KL/rejected": 760.0, "epoch": 0.6256748650269947, "grad_norm": 4.952917575836182, "learning_rate": 3.0811431552719513e-06, "logp/chosen": -408.0, "logp/rejected": -336.0, "loss": 0.48699951171875, "reward/accuracy": 0.8125, "reward/chosen": -1.9375, "reward/margin": 1.6484375, "reward/rejected": -3.578125, "step": 2086 }, { "approx. KL/chosen": 406.0, "approx. KL/rejected": 668.0, "epoch": 0.6259748050389922, "grad_norm": 4.029341220855713, "learning_rate": 3.0767933253294972e-06, "logp/chosen": -412.0, "logp/rejected": -338.0, "loss": 0.441070556640625, "reward/accuracy": 0.6875, "reward/chosen": -2.21875, "reward/margin": 1.3125, "reward/rejected": -3.546875, "step": 2087 }, { "approx. KL/chosen": 225.0, "approx. KL/rejected": 1168.0, "epoch": 0.6262747450509898, "grad_norm": 2.639522075653076, "learning_rate": 3.0724452030198136e-06, "logp/chosen": -340.0, "logp/rejected": -452.0, "loss": 0.2260894775390625, "reward/accuracy": 0.9375, "reward/chosen": -1.7578125, "reward/margin": 2.53125, "reward/rejected": -4.28125, "step": 2088 }, { "approx. KL/chosen": 233.0, "approx. KL/rejected": 736.0, "epoch": 0.6265746850629874, "grad_norm": 3.560023784637451, "learning_rate": 3.0680987922036324e-06, "logp/chosen": -229.0, "logp/rejected": -266.0, "loss": 0.4564208984375, "reward/accuracy": 0.75, "reward/chosen": -1.6484375, "reward/margin": 1.671875, "reward/rejected": -3.3125, "step": 2089 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 644.0, "epoch": 0.626874625074985, "grad_norm": 3.96856689453125, "learning_rate": 3.063754096740175e-06, "logp/chosen": -394.0, "logp/rejected": -362.0, "loss": 0.5773162841796875, "reward/accuracy": 0.5625, "reward/chosen": -1.6328125, "reward/margin": 1.1875, "reward/rejected": -2.8125, "step": 2090 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 656.0, "epoch": 0.6271745650869825, "grad_norm": 3.3060317039489746, "learning_rate": 3.0594111204871346e-06, "logp/chosen": -390.0, "logp/rejected": -378.0, "loss": 0.40191650390625, "reward/accuracy": 0.875, "reward/chosen": -1.90625, "reward/margin": 1.5234375, "reward/rejected": -3.4375, "step": 2091 }, { "approx. KL/chosen": 344.0, "approx. KL/rejected": 560.0, "epoch": 0.6274745050989802, "grad_norm": 4.9555487632751465, "learning_rate": 3.055069867300682e-06, "logp/chosen": -354.0, "logp/rejected": -270.0, "loss": 0.776123046875, "reward/accuracy": 0.6875, "reward/chosen": -2.25, "reward/margin": 0.6015625, "reward/rejected": -2.859375, "step": 2092 }, { "approx. KL/chosen": 147.0, "approx. KL/rejected": 552.0, "epoch": 0.6277744451109778, "grad_norm": 3.407470464706421, "learning_rate": 3.0507303410354527e-06, "logp/chosen": -232.0, "logp/rejected": -254.0, "loss": 0.442352294921875, "reward/accuracy": 0.6875, "reward/chosen": -1.2421875, "reward/margin": 1.546875, "reward/rejected": -2.796875, "step": 2093 }, { "approx. KL/chosen": 308.0, "approx. KL/rejected": 1008.0, "epoch": 0.6280743851229754, "grad_norm": 3.497098922729492, "learning_rate": 3.046392545544552e-06, "logp/chosen": -278.0, "logp/rejected": -416.0, "loss": 0.43292236328125, "reward/accuracy": 0.6875, "reward/chosen": -2.03125, "reward/margin": 1.6875, "reward/rejected": -3.71875, "step": 2094 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 804.0, "epoch": 0.628374325134973, "grad_norm": 5.4329514503479, "learning_rate": 3.0420564846795497e-06, "logp/chosen": -444.0, "logp/rejected": -318.0, "loss": 0.4755859375, "reward/accuracy": 0.625, "reward/chosen": -2.015625, "reward/margin": 1.4609375, "reward/rejected": -3.484375, "step": 2095 }, { "approx. KL/chosen": 215.0, "approx. KL/rejected": 1024.0, "epoch": 0.6286742651469706, "grad_norm": 2.036776542663574, "learning_rate": 3.0377221622904725e-06, "logp/chosen": -450.0, "logp/rejected": -424.0, "loss": 0.207122802734375, "reward/accuracy": 0.9375, "reward/chosen": -1.6953125, "reward/margin": 2.359375, "reward/rejected": -4.0625, "step": 2096 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 760.0, "epoch": 0.6289742051589682, "grad_norm": 2.791405200958252, "learning_rate": 3.0333895822258073e-06, "logp/chosen": -448.0, "logp/rejected": -348.0, "loss": 0.257781982421875, "reward/accuracy": 0.9375, "reward/chosen": -1.5390625, "reward/margin": 1.9375, "reward/rejected": -3.46875, "step": 2097 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 552.0, "epoch": 0.6292741451709658, "grad_norm": 4.554195880889893, "learning_rate": 3.0290587483324878e-06, "logp/chosen": -306.0, "logp/rejected": -340.0, "loss": 0.659088134765625, "reward/accuracy": 0.6875, "reward/chosen": -1.6484375, "reward/margin": 1.203125, "reward/rejected": -2.84375, "step": 2098 }, { "approx. KL/chosen": 86.0, "approx. KL/rejected": 1120.0, "epoch": 0.6295740851829634, "grad_norm": 2.9901716709136963, "learning_rate": 3.024729664455903e-06, "logp/chosen": -312.0, "logp/rejected": -424.0, "loss": 0.27541351318359375, "reward/accuracy": 0.875, "reward/chosen": -0.9375, "reward/margin": 2.8125, "reward/rejected": -3.75, "step": 2099 }, { "approx. KL/chosen": 122.0, "approx. KL/rejected": 1080.0, "epoch": 0.629874025194961, "grad_norm": 1.7814695835113525, "learning_rate": 3.0204023344398847e-06, "logp/chosen": -308.0, "logp/rejected": -312.0, "loss": 0.1736907958984375, "reward/accuracy": 0.9375, "reward/chosen": -1.1796875, "reward/margin": 2.921875, "reward/rejected": -4.09375, "step": 2100 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 784.0, "epoch": 0.6301739652069586, "grad_norm": 2.940263032913208, "learning_rate": 3.016076762126712e-06, "logp/chosen": -328.0, "logp/rejected": -498.0, "loss": 0.265625, "reward/accuracy": 0.875, "reward/chosen": -1.4140625, "reward/margin": 2.15625, "reward/rejected": -3.578125, "step": 2101 }, { "approx. KL/chosen": 434.0, "approx. KL/rejected": 788.0, "epoch": 0.6304739052189562, "grad_norm": 6.624047756195068, "learning_rate": 3.011752951357097e-06, "logp/chosen": -348.0, "logp/rejected": -436.0, "loss": 0.63751220703125, "reward/accuracy": 0.5625, "reward/chosen": -2.296875, "reward/margin": 0.96875, "reward/rejected": -3.265625, "step": 2102 }, { "approx. KL/chosen": 234.0, "approx. KL/rejected": 490.0, "epoch": 0.6307738452309538, "grad_norm": 6.029907703399658, "learning_rate": 3.0074309059701907e-06, "logp/chosen": -556.0, "logp/rejected": -382.0, "loss": 0.811279296875, "reward/accuracy": 0.625, "reward/chosen": -1.8515625, "reward/margin": 0.75390625, "reward/rejected": -2.609375, "step": 2103 }, { "approx. KL/chosen": 212.0, "approx. KL/rejected": 860.0, "epoch": 0.6310737852429514, "grad_norm": 3.9079642295837402, "learning_rate": 3.003110629803577e-06, "logp/chosen": -416.0, "logp/rejected": -306.0, "loss": 0.626708984375, "reward/accuracy": 0.625, "reward/chosen": -1.3046875, "reward/margin": 1.96875, "reward/rejected": -3.28125, "step": 2104 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 572.0, "epoch": 0.6313737252549491, "grad_norm": 6.922399997711182, "learning_rate": 2.9987921266932706e-06, "logp/chosen": -344.0, "logp/rejected": -288.0, "loss": 0.5523681640625, "reward/accuracy": 0.6875, "reward/chosen": -1.6015625, "reward/margin": 1.1328125, "reward/rejected": -2.734375, "step": 2105 }, { "approx. KL/chosen": 83.0, "approx. KL/rejected": 700.0, "epoch": 0.6316736652669466, "grad_norm": 2.8174397945404053, "learning_rate": 2.9944754004737087e-06, "logp/chosen": -227.0, "logp/rejected": -252.0, "loss": 0.32672119140625, "reward/accuracy": 0.8125, "reward/chosen": -0.99609375, "reward/margin": 2.09375, "reward/rejected": -3.078125, "step": 2106 }, { "approx. KL/chosen": 378.0, "approx. KL/rejected": 836.0, "epoch": 0.6319736052789442, "grad_norm": 4.7125725746154785, "learning_rate": 2.9901604549777506e-06, "logp/chosen": -420.0, "logp/rejected": -406.0, "loss": 0.319610595703125, "reward/accuracy": 0.875, "reward/chosen": -2.109375, "reward/margin": 1.65625, "reward/rejected": -3.765625, "step": 2107 }, { "approx. KL/chosen": 121.0, "approx. KL/rejected": 536.0, "epoch": 0.6322735452909418, "grad_norm": 2.70941424369812, "learning_rate": 2.9858472940366766e-06, "logp/chosen": -296.0, "logp/rejected": -302.0, "loss": 0.294677734375, "reward/accuracy": 0.9375, "reward/chosen": -1.1484375, "reward/margin": 1.78125, "reward/rejected": -2.921875, "step": 2108 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 728.0, "epoch": 0.6325734853029394, "grad_norm": 3.4772746562957764, "learning_rate": 2.981535921480182e-06, "logp/chosen": -316.0, "logp/rejected": -280.0, "loss": 0.4183349609375, "reward/accuracy": 0.8125, "reward/chosen": -1.3046875, "reward/margin": 2.09375, "reward/rejected": -3.390625, "step": 2109 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 900.0, "epoch": 0.6328734253149371, "grad_norm": 4.480722427368164, "learning_rate": 2.9772263411363746e-06, "logp/chosen": -322.0, "logp/rejected": -354.0, "loss": 0.374359130859375, "reward/accuracy": 0.8125, "reward/chosen": -1.4140625, "reward/margin": 2.3125, "reward/rejected": -3.71875, "step": 2110 }, { "approx. KL/chosen": 111.0, "approx. KL/rejected": 416.0, "epoch": 0.6331733653269346, "grad_norm": 3.511202812194824, "learning_rate": 2.9729185568317685e-06, "logp/chosen": -235.0, "logp/rejected": -236.0, "loss": 0.4051513671875, "reward/accuracy": 0.8125, "reward/chosen": -1.125, "reward/margin": 1.375, "reward/rejected": -2.5, "step": 2111 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 520.0, "epoch": 0.6334733053389322, "grad_norm": 5.127079010009766, "learning_rate": 2.968612572391286e-06, "logp/chosen": -296.0, "logp/rejected": -328.0, "loss": 0.5616455078125, "reward/accuracy": 0.6875, "reward/chosen": -1.359375, "reward/margin": 1.328125, "reward/rejected": -2.6875, "step": 2112 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 800.0, "epoch": 0.6337732453509298, "grad_norm": 3.067364454269409, "learning_rate": 2.964308391638249e-06, "logp/chosen": -225.0, "logp/rejected": -406.0, "loss": 0.2873382568359375, "reward/accuracy": 0.875, "reward/chosen": -1.2890625, "reward/margin": 2.375, "reward/rejected": -3.671875, "step": 2113 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 616.0, "epoch": 0.6340731853629274, "grad_norm": 4.170029163360596, "learning_rate": 2.960006018394379e-06, "logp/chosen": -398.0, "logp/rejected": -450.0, "loss": 0.523712158203125, "reward/accuracy": 0.75, "reward/chosen": -1.609375, "reward/margin": 1.0703125, "reward/rejected": -2.6875, "step": 2114 }, { "approx. KL/chosen": 104.0, "approx. KL/rejected": 648.0, "epoch": 0.634373125374925, "grad_norm": 2.4630374908447266, "learning_rate": 2.955705456479794e-06, "logp/chosen": -199.0, "logp/rejected": -212.0, "loss": 0.36688232421875, "reward/accuracy": 0.8125, "reward/chosen": -1.0859375, "reward/margin": 1.75, "reward/rejected": -2.828125, "step": 2115 }, { "approx. KL/chosen": 245.0, "approx. KL/rejected": 1144.0, "epoch": 0.6346730653869226, "grad_norm": 3.253699541091919, "learning_rate": 2.9514067097130005e-06, "logp/chosen": -235.0, "logp/rejected": -306.0, "loss": 0.269012451171875, "reward/accuracy": 0.875, "reward/chosen": -1.5390625, "reward/margin": 2.40625, "reward/rejected": -3.953125, "step": 2116 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 512.0, "epoch": 0.6349730053989202, "grad_norm": 2.8989903926849365, "learning_rate": 2.9471097819108953e-06, "logp/chosen": -452.0, "logp/rejected": -360.0, "loss": 0.3231201171875, "reward/accuracy": 0.8125, "reward/chosen": -1.4296875, "reward/margin": 1.2734375, "reward/rejected": -2.703125, "step": 2117 }, { "approx. KL/chosen": 102.0, "approx. KL/rejected": 576.0, "epoch": 0.6352729454109178, "grad_norm": 3.758812665939331, "learning_rate": 2.94281467688876e-06, "logp/chosen": -318.0, "logp/rejected": -346.0, "loss": 0.3041839599609375, "reward/accuracy": 0.875, "reward/chosen": -1.1328125, "reward/margin": 1.9140625, "reward/rejected": -3.046875, "step": 2118 }, { "approx. KL/chosen": 370.0, "approx. KL/rejected": 1288.0, "epoch": 0.6355728854229155, "grad_norm": 3.869310140609741, "learning_rate": 2.938521398460257e-06, "logp/chosen": -394.0, "logp/rejected": -326.0, "loss": 0.44838428497314453, "reward/accuracy": 0.875, "reward/chosen": -2.109375, "reward/margin": 2.28125, "reward/rejected": -4.40625, "step": 2119 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 844.0, "epoch": 0.635872825434913, "grad_norm": 3.871114730834961, "learning_rate": 2.93422995043743e-06, "logp/chosen": -268.0, "logp/rejected": -340.0, "loss": 0.41379547119140625, "reward/accuracy": 0.75, "reward/chosen": -1.328125, "reward/margin": 2.109375, "reward/rejected": -3.4375, "step": 2120 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 1112.0, "epoch": 0.6361727654469106, "grad_norm": 3.706775188446045, "learning_rate": 2.929940336630692e-06, "logp/chosen": -376.0, "logp/rejected": -436.0, "loss": 0.42559814453125, "reward/accuracy": 0.75, "reward/chosen": -1.7109375, "reward/margin": 1.9140625, "reward/rejected": -3.625, "step": 2121 }, { "approx. KL/chosen": 376.0, "approx. KL/rejected": 640.0, "epoch": 0.6364727054589082, "grad_norm": 8.135274887084961, "learning_rate": 2.9256525608488317e-06, "logp/chosen": -334.0, "logp/rejected": -332.0, "loss": 0.6661376953125, "reward/accuracy": 0.75, "reward/chosen": -1.875, "reward/margin": 1.2109375, "reward/rejected": -3.09375, "step": 2122 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 416.0, "epoch": 0.6367726454709058, "grad_norm": 4.839533805847168, "learning_rate": 2.9213666268990026e-06, "logp/chosen": -362.0, "logp/rejected": -424.0, "loss": 0.6640625, "reward/accuracy": 0.6875, "reward/chosen": -1.6484375, "reward/margin": 0.9453125, "reward/rejected": -2.59375, "step": 2123 }, { "approx. KL/chosen": 324.0, "approx. KL/rejected": 764.0, "epoch": 0.6370725854829035, "grad_norm": 5.124302864074707, "learning_rate": 2.917082538586726e-06, "logp/chosen": -452.0, "logp/rejected": -480.0, "loss": 0.618743896484375, "reward/accuracy": 0.6875, "reward/chosen": -2.0, "reward/margin": 1.2109375, "reward/rejected": -3.203125, "step": 2124 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 676.0, "epoch": 0.637372525494901, "grad_norm": 4.802201747894287, "learning_rate": 2.912800299715885e-06, "logp/chosen": -328.0, "logp/rejected": -334.0, "loss": 0.70672607421875, "reward/accuracy": 0.75, "reward/chosen": -1.75, "reward/margin": 1.2890625, "reward/rejected": -3.046875, "step": 2125 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 816.0, "epoch": 0.6376724655068986, "grad_norm": 4.123416900634766, "learning_rate": 2.9085199140887155e-06, "logp/chosen": -278.0, "logp/rejected": -354.0, "loss": 0.37628173828125, "reward/accuracy": 0.75, "reward/chosen": -1.4609375, "reward/margin": 1.7734375, "reward/rejected": -3.234375, "step": 2126 }, { "approx. KL/chosen": 358.0, "approx. KL/rejected": 520.0, "epoch": 0.6379724055188962, "grad_norm": 6.033300876617432, "learning_rate": 2.9042413855058127e-06, "logp/chosen": -260.0, "logp/rejected": -300.0, "loss": 0.63739013671875, "reward/accuracy": 0.625, "reward/chosen": -1.6640625, "reward/margin": 1.0859375, "reward/rejected": -2.75, "step": 2127 }, { "approx. KL/chosen": 249.0, "approx. KL/rejected": 900.0, "epoch": 0.6382723455308938, "grad_norm": 2.2529678344726562, "learning_rate": 2.8999647177661184e-06, "logp/chosen": -300.0, "logp/rejected": -286.0, "loss": 0.2231292724609375, "reward/accuracy": 0.875, "reward/chosen": -1.53125, "reward/margin": 2.140625, "reward/rejected": -3.671875, "step": 2128 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 992.0, "epoch": 0.6385722855428915, "grad_norm": 4.326500415802002, "learning_rate": 2.895689914666928e-06, "logp/chosen": -310.0, "logp/rejected": -424.0, "loss": 0.5081024169921875, "reward/accuracy": 0.75, "reward/chosen": -1.7109375, "reward/margin": 1.9609375, "reward/rejected": -3.671875, "step": 2129 }, { "approx. KL/chosen": 142.0, "approx. KL/rejected": 552.0, "epoch": 0.638872225554889, "grad_norm": 3.701479434967041, "learning_rate": 2.891416980003876e-06, "logp/chosen": -434.0, "logp/rejected": -420.0, "loss": 0.47796630859375, "reward/accuracy": 0.6875, "reward/chosen": -1.21875, "reward/margin": 1.546875, "reward/rejected": -2.765625, "step": 2130 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 788.0, "epoch": 0.6391721655668866, "grad_norm": 4.472229480743408, "learning_rate": 2.887145917570946e-06, "logp/chosen": -246.0, "logp/rejected": -312.0, "loss": 0.456634521484375, "reward/accuracy": 0.6875, "reward/chosen": -1.734375, "reward/margin": 1.7265625, "reward/rejected": -3.453125, "step": 2131 }, { "approx. KL/chosen": 246.0, "approx. KL/rejected": 904.0, "epoch": 0.6394721055788842, "grad_norm": 4.054142475128174, "learning_rate": 2.882876731160444e-06, "logp/chosen": -374.0, "logp/rejected": -332.0, "loss": 0.387115478515625, "reward/accuracy": 0.75, "reward/chosen": -1.4921875, "reward/margin": 2.265625, "reward/rejected": -3.75, "step": 2132 }, { "approx. KL/chosen": 1048.0, "approx. KL/rejected": 908.0, "epoch": 0.6397720455908819, "grad_norm": 16.180601119995117, "learning_rate": 2.8786094245630263e-06, "logp/chosen": -398.0, "logp/rejected": -392.0, "loss": 1.178924560546875, "reward/accuracy": 0.5625, "reward/chosen": -2.359375, "reward/margin": 0.74609375, "reward/rejected": -3.09375, "step": 2133 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 688.0, "epoch": 0.6400719856028795, "grad_norm": 2.965893030166626, "learning_rate": 2.874344001567669e-06, "logp/chosen": -372.0, "logp/rejected": -356.0, "loss": 0.3890380859375, "reward/accuracy": 0.8125, "reward/chosen": -1.1484375, "reward/margin": 1.8359375, "reward/rejected": -2.984375, "step": 2134 }, { "approx. KL/chosen": 233.0, "approx. KL/rejected": 840.0, "epoch": 0.640371925614877, "grad_norm": 3.4256410598754883, "learning_rate": 2.8700804659616852e-06, "logp/chosen": -300.0, "logp/rejected": -404.0, "loss": 0.24517822265625, "reward/accuracy": 0.9375, "reward/chosen": -1.7265625, "reward/margin": 2.078125, "reward/rejected": -3.8125, "step": 2135 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 760.0, "epoch": 0.6406718656268746, "grad_norm": 4.052453517913818, "learning_rate": 2.865818821530705e-06, "logp/chosen": -302.0, "logp/rejected": -316.0, "loss": 0.359375, "reward/accuracy": 0.875, "reward/chosen": -1.171875, "reward/margin": 1.6875, "reward/rejected": -2.859375, "step": 2136 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 736.0, "epoch": 0.6409718056388722, "grad_norm": 6.267385482788086, "learning_rate": 2.8615590720586795e-06, "logp/chosen": -378.0, "logp/rejected": -328.0, "loss": 0.7440185546875, "reward/accuracy": 0.625, "reward/chosen": -1.765625, "reward/margin": 1.171875, "reward/rejected": -2.9375, "step": 2137 }, { "approx. KL/chosen": 107.0, "approx. KL/rejected": 816.0, "epoch": 0.6412717456508699, "grad_norm": 3.2271764278411865, "learning_rate": 2.857301221327879e-06, "logp/chosen": -378.0, "logp/rejected": -338.0, "loss": 0.25710296630859375, "reward/accuracy": 0.875, "reward/chosen": -0.953125, "reward/margin": 2.234375, "reward/rejected": -3.1875, "step": 2138 }, { "approx. KL/chosen": 219.0, "approx. KL/rejected": 424.0, "epoch": 0.6415716856628674, "grad_norm": 4.680198669433594, "learning_rate": 2.853045273118891e-06, "logp/chosen": -332.0, "logp/rejected": -384.0, "loss": 0.6264495849609375, "reward/accuracy": 0.625, "reward/chosen": -1.640625, "reward/margin": 0.703125, "reward/rejected": -2.34375, "step": 2139 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 540.0, "epoch": 0.641871625674865, "grad_norm": 3.5259952545166016, "learning_rate": 2.8487912312106103e-06, "logp/chosen": -322.0, "logp/rejected": -328.0, "loss": 0.4664306640625, "reward/accuracy": 0.6875, "reward/chosen": -1.3359375, "reward/margin": 1.3984375, "reward/rejected": -2.734375, "step": 2140 }, { "approx. KL/chosen": 84.0, "approx. KL/rejected": 664.0, "epoch": 0.6421715656868626, "grad_norm": 3.24287748336792, "learning_rate": 2.844539099380238e-06, "logp/chosen": -402.0, "logp/rejected": -392.0, "loss": 0.33624267578125, "reward/accuracy": 0.875, "reward/chosen": -1.109375, "reward/margin": 1.90625, "reward/rejected": -3.015625, "step": 2141 }, { "approx. KL/chosen": 221.0, "approx. KL/rejected": 860.0, "epoch": 0.6424715056988602, "grad_norm": 1.9422770738601685, "learning_rate": 2.840288881403279e-06, "logp/chosen": -292.0, "logp/rejected": -231.0, "loss": 0.238311767578125, "reward/accuracy": 0.9375, "reward/chosen": -1.53125, "reward/margin": 2.171875, "reward/rejected": -3.71875, "step": 2142 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 520.0, "epoch": 0.6427714457108579, "grad_norm": 5.422247886657715, "learning_rate": 2.8360405810535454e-06, "logp/chosen": -368.0, "logp/rejected": -272.0, "loss": 0.6729736328125, "reward/accuracy": 0.75, "reward/chosen": -1.3203125, "reward/margin": 1.15625, "reward/rejected": -2.46875, "step": 2143 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 680.0, "epoch": 0.6430713857228554, "grad_norm": 3.341460704803467, "learning_rate": 2.831794202103137e-06, "logp/chosen": -268.0, "logp/rejected": -274.0, "loss": 0.4161376953125, "reward/accuracy": 0.8125, "reward/chosen": -1.453125, "reward/margin": 1.5625, "reward/rejected": -3.015625, "step": 2144 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 952.0, "epoch": 0.643371325734853, "grad_norm": 4.251412391662598, "learning_rate": 2.827549748322459e-06, "logp/chosen": -328.0, "logp/rejected": -306.0, "loss": 0.6408615112304688, "reward/accuracy": 0.75, "reward/chosen": -1.484375, "reward/margin": 1.953125, "reward/rejected": -3.4375, "step": 2145 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 1032.0, "epoch": 0.6436712657468506, "grad_norm": 3.3060572147369385, "learning_rate": 2.823307223480193e-06, "logp/chosen": -284.0, "logp/rejected": -324.0, "loss": 0.35980224609375, "reward/accuracy": 0.875, "reward/chosen": -2.03125, "reward/margin": 2.046875, "reward/rejected": -4.09375, "step": 2146 }, { "approx. KL/chosen": 218.0, "approx. KL/rejected": 480.0, "epoch": 0.6439712057588483, "grad_norm": 4.466633319854736, "learning_rate": 2.8190666313433207e-06, "logp/chosen": -203.0, "logp/rejected": -222.0, "loss": 0.6722412109375, "reward/accuracy": 0.875, "reward/chosen": -1.265625, "reward/margin": 1.2890625, "reward/rejected": -2.546875, "step": 2147 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 608.0, "epoch": 0.6442711457708459, "grad_norm": 5.169397830963135, "learning_rate": 2.814827975677099e-06, "logp/chosen": -219.0, "logp/rejected": -233.0, "loss": 0.5367431640625, "reward/accuracy": 0.75, "reward/chosen": -1.171875, "reward/margin": 1.453125, "reward/rejected": -2.625, "step": 2148 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 764.0, "epoch": 0.6445710857828434, "grad_norm": 2.781708002090454, "learning_rate": 2.8105912602450736e-06, "logp/chosen": -382.0, "logp/rejected": -400.0, "loss": 0.26763916015625, "reward/accuracy": 0.875, "reward/chosen": -1.1640625, "reward/margin": 2.15625, "reward/rejected": -3.328125, "step": 2149 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 1048.0, "epoch": 0.644871025794841, "grad_norm": 3.5795984268188477, "learning_rate": 2.806356488809059e-06, "logp/chosen": -205.0, "logp/rejected": -348.0, "loss": 0.3588714599609375, "reward/accuracy": 0.8125, "reward/chosen": -1.140625, "reward/margin": 2.484375, "reward/rejected": -3.625, "step": 2150 }, { "approx. KL/chosen": 93.0, "approx. KL/rejected": 524.0, "epoch": 0.6451709658068386, "grad_norm": 2.8081746101379395, "learning_rate": 2.8021236651291497e-06, "logp/chosen": -306.0, "logp/rejected": -324.0, "loss": 0.33258056640625, "reward/accuracy": 0.875, "reward/chosen": -1.125, "reward/margin": 1.609375, "reward/rejected": -2.734375, "step": 2151 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 1008.0, "epoch": 0.6454709058188363, "grad_norm": 3.195364475250244, "learning_rate": 2.7978927929637046e-06, "logp/chosen": -312.0, "logp/rejected": -300.0, "loss": 0.32784271240234375, "reward/accuracy": 0.875, "reward/chosen": -1.390625, "reward/margin": 2.578125, "reward/rejected": -3.96875, "step": 2152 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 644.0, "epoch": 0.6457708458308339, "grad_norm": 3.0537097454071045, "learning_rate": 2.793663876069359e-06, "logp/chosen": -442.0, "logp/rejected": -354.0, "loss": 0.30096435546875, "reward/accuracy": 0.875, "reward/chosen": -1.1796875, "reward/margin": 1.8671875, "reward/rejected": -3.046875, "step": 2153 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 708.0, "epoch": 0.6460707858428314, "grad_norm": 5.0283284187316895, "learning_rate": 2.7894369182010016e-06, "logp/chosen": -308.0, "logp/rejected": -302.0, "loss": 0.76171875, "reward/accuracy": 0.625, "reward/chosen": -1.828125, "reward/margin": 0.890625, "reward/rejected": -2.71875, "step": 2154 }, { "approx. KL/chosen": 149.0, "approx. KL/rejected": 872.0, "epoch": 0.646370725854829, "grad_norm": 4.200366497039795, "learning_rate": 2.7852119231117946e-06, "logp/chosen": -318.0, "logp/rejected": -368.0, "loss": 0.3966827392578125, "reward/accuracy": 0.75, "reward/chosen": -1.375, "reward/margin": 2.09375, "reward/rejected": -3.46875, "step": 2155 }, { "approx. KL/chosen": 146.0, "approx. KL/rejected": 1000.0, "epoch": 0.6466706658668266, "grad_norm": 2.803877592086792, "learning_rate": 2.7809888945531403e-06, "logp/chosen": -482.0, "logp/rejected": -344.0, "loss": 0.290435791015625, "reward/accuracy": 0.875, "reward/chosen": -1.40625, "reward/margin": 2.578125, "reward/rejected": -3.984375, "step": 2156 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 632.0, "epoch": 0.6469706058788243, "grad_norm": 4.90876579284668, "learning_rate": 2.7767678362747106e-06, "logp/chosen": -306.0, "logp/rejected": -346.0, "loss": 0.586334228515625, "reward/accuracy": 0.5625, "reward/chosen": -1.859375, "reward/margin": 1.203125, "reward/rejected": -3.0625, "step": 2157 }, { "approx. KL/chosen": 149.0, "approx. KL/rejected": 1392.0, "epoch": 0.6472705458908218, "grad_norm": 2.222236156463623, "learning_rate": 2.7725487520244182e-06, "logp/chosen": -398.0, "logp/rejected": -300.0, "loss": 0.20157241821289062, "reward/accuracy": 0.875, "reward/chosen": -1.125, "reward/margin": 3.546875, "reward/rejected": -4.65625, "step": 2158 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 780.0, "epoch": 0.6475704859028194, "grad_norm": 2.231904983520508, "learning_rate": 2.768331645548432e-06, "logp/chosen": -362.0, "logp/rejected": -282.0, "loss": 0.2522735595703125, "reward/accuracy": 0.9375, "reward/chosen": -1.3515625, "reward/margin": 2.078125, "reward/rejected": -3.4375, "step": 2159 }, { "approx. KL/chosen": 130.0, "approx. KL/rejected": 828.0, "epoch": 0.647870425914817, "grad_norm": 2.9045729637145996, "learning_rate": 2.764116520591151e-06, "logp/chosen": -470.0, "logp/rejected": -498.0, "loss": 0.30096435546875, "reward/accuracy": 0.9375, "reward/chosen": -1.0546875, "reward/margin": 2.421875, "reward/rejected": -3.484375, "step": 2160 }, { "approx. KL/chosen": 246.0, "approx. KL/rejected": 1232.0, "epoch": 0.6481703659268147, "grad_norm": 2.328866720199585, "learning_rate": 2.759903380895231e-06, "logp/chosen": -348.0, "logp/rejected": -422.0, "loss": 0.2236328125, "reward/accuracy": 0.875, "reward/chosen": -1.8515625, "reward/margin": 2.515625, "reward/rejected": -4.375, "step": 2161 }, { "approx. KL/chosen": 235.0, "approx. KL/rejected": 484.0, "epoch": 0.6484703059388123, "grad_norm": 4.922508716583252, "learning_rate": 2.7556922302015508e-06, "logp/chosen": -344.0, "logp/rejected": -336.0, "loss": 0.58209228515625, "reward/accuracy": 0.6875, "reward/chosen": -1.2578125, "reward/margin": 1.40625, "reward/rejected": -2.671875, "step": 2162 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 528.0, "epoch": 0.6487702459508098, "grad_norm": 4.549445629119873, "learning_rate": 2.7514830722492355e-06, "logp/chosen": -418.0, "logp/rejected": -406.0, "loss": 0.537109375, "reward/accuracy": 0.625, "reward/chosen": -1.890625, "reward/margin": 0.95703125, "reward/rejected": -2.84375, "step": 2163 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 872.0, "epoch": 0.6490701859628074, "grad_norm": 4.4958600997924805, "learning_rate": 2.7472759107756315e-06, "logp/chosen": -312.0, "logp/rejected": -272.0, "loss": 0.4260711669921875, "reward/accuracy": 0.8125, "reward/chosen": -1.5078125, "reward/margin": 2.140625, "reward/rejected": -3.640625, "step": 2164 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 478.0, "epoch": 0.649370125974805, "grad_norm": 5.51969051361084, "learning_rate": 2.7430707495163163e-06, "logp/chosen": -314.0, "logp/rejected": -274.0, "loss": 0.742919921875, "reward/accuracy": 0.6875, "reward/chosen": -1.5859375, "reward/margin": 0.98046875, "reward/rejected": -2.5625, "step": 2165 }, { "approx. KL/chosen": 394.0, "approx. KL/rejected": 792.0, "epoch": 0.6496700659868027, "grad_norm": 4.553469657897949, "learning_rate": 2.738867592205089e-06, "logp/chosen": -356.0, "logp/rejected": -290.0, "loss": 0.34747314453125, "reward/accuracy": 0.8125, "reward/chosen": -1.75, "reward/margin": 1.8984375, "reward/rejected": -3.65625, "step": 2166 }, { "approx. KL/chosen": 74.0, "approx. KL/rejected": 708.0, "epoch": 0.6499700059988003, "grad_norm": 3.478633403778076, "learning_rate": 2.7346664425739757e-06, "logp/chosen": -326.0, "logp/rejected": -304.0, "loss": 0.33905029296875, "reward/accuracy": 0.875, "reward/chosen": -0.9921875, "reward/margin": 2.109375, "reward/rejected": -3.09375, "step": 2167 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 1016.0, "epoch": 0.6502699460107978, "grad_norm": 5.005650043487549, "learning_rate": 2.7304673043532114e-06, "logp/chosen": -390.0, "logp/rejected": -398.0, "loss": 0.5010528564453125, "reward/accuracy": 0.6875, "reward/chosen": -1.453125, "reward/margin": 2.25, "reward/rejected": -3.6875, "step": 2168 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 612.0, "epoch": 0.6505698860227954, "grad_norm": 3.6151771545410156, "learning_rate": 2.7262701812712545e-06, "logp/chosen": -396.0, "logp/rejected": -358.0, "loss": 0.406005859375, "reward/accuracy": 0.8125, "reward/chosen": -1.2109375, "reward/margin": 1.5078125, "reward/rejected": -2.71875, "step": 2169 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 1048.0, "epoch": 0.650869826034793, "grad_norm": 3.8331315517425537, "learning_rate": 2.7220750770547622e-06, "logp/chosen": -374.0, "logp/rejected": -352.0, "loss": 0.4510765075683594, "reward/accuracy": 0.8125, "reward/chosen": -1.4609375, "reward/margin": 2.28125, "reward/rejected": -3.734375, "step": 2170 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 780.0, "epoch": 0.6511697660467907, "grad_norm": 3.548349142074585, "learning_rate": 2.717881995428611e-06, "logp/chosen": -422.0, "logp/rejected": -356.0, "loss": 0.349853515625, "reward/accuracy": 0.8125, "reward/chosen": -1.5234375, "reward/margin": 1.8515625, "reward/rejected": -3.375, "step": 2171 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 1424.0, "epoch": 0.6514697060587883, "grad_norm": 1.814414620399475, "learning_rate": 2.713690940115873e-06, "logp/chosen": -400.0, "logp/rejected": -374.0, "loss": 0.15157318115234375, "reward/accuracy": 0.9375, "reward/chosen": -1.4609375, "reward/margin": 3.359375, "reward/rejected": -4.8125, "step": 2172 }, { "approx. KL/chosen": 150.0, "approx. KL/rejected": 604.0, "epoch": 0.6517696460707858, "grad_norm": 3.3462042808532715, "learning_rate": 2.709501914837827e-06, "logp/chosen": -320.0, "logp/rejected": -358.0, "loss": 0.347137451171875, "reward/accuracy": 0.8125, "reward/chosen": -1.0546875, "reward/margin": 1.7265625, "reward/rejected": -2.78125, "step": 2173 }, { "approx. KL/chosen": 500.0, "approx. KL/rejected": 1232.0, "epoch": 0.6520695860827834, "grad_norm": 2.5513412952423096, "learning_rate": 2.7053149233139466e-06, "logp/chosen": -282.0, "logp/rejected": -358.0, "loss": 0.16912841796875, "reward/accuracy": 1.0, "reward/chosen": -2.328125, "reward/margin": 2.25, "reward/rejected": -4.5625, "step": 2174 }, { "approx. KL/chosen": 187.0, "approx. KL/rejected": 720.0, "epoch": 0.652369526094781, "grad_norm": 2.8711788654327393, "learning_rate": 2.701129969261899e-06, "logp/chosen": -466.0, "logp/rejected": -370.0, "loss": 0.21759033203125, "reward/accuracy": 0.9375, "reward/chosen": -1.296875, "reward/margin": 2.125, "reward/rejected": -3.4375, "step": 2175 }, { "approx. KL/chosen": 187.0, "approx. KL/rejected": 278.0, "epoch": 0.6526694661067787, "grad_norm": 4.784427165985107, "learning_rate": 2.696947056397541e-06, "logp/chosen": -225.0, "logp/rejected": -306.0, "loss": 0.69793701171875, "reward/accuracy": 0.625, "reward/chosen": -1.5234375, "reward/margin": 0.43359375, "reward/rejected": -1.953125, "step": 2176 }, { "approx. KL/chosen": 184.0, "approx. KL/rejected": 808.0, "epoch": 0.6529694061187763, "grad_norm": 1.9373035430908203, "learning_rate": 2.6927661884349227e-06, "logp/chosen": -334.0, "logp/rejected": -282.0, "loss": 0.209259033203125, "reward/accuracy": 1.0, "reward/chosen": -1.15625, "reward/margin": 2.0625, "reward/rejected": -3.21875, "step": 2177 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 824.0, "epoch": 0.6532693461307738, "grad_norm": 4.275570869445801, "learning_rate": 2.688587369086272e-06, "logp/chosen": -284.0, "logp/rejected": -354.0, "loss": 0.599151611328125, "reward/accuracy": 0.625, "reward/chosen": -2.0, "reward/margin": 1.2578125, "reward/rejected": -3.265625, "step": 2178 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 824.0, "epoch": 0.6535692861427714, "grad_norm": 3.2808074951171875, "learning_rate": 2.6844106020620064e-06, "logp/chosen": -452.0, "logp/rejected": -374.0, "loss": 0.4445037841796875, "reward/accuracy": 0.75, "reward/chosen": -1.2265625, "reward/margin": 2.015625, "reward/rejected": -3.25, "step": 2179 }, { "approx. KL/chosen": 292.0, "approx. KL/rejected": 1304.0, "epoch": 0.6538692261547691, "grad_norm": 3.502798557281494, "learning_rate": 2.680235891070707e-06, "logp/chosen": -342.0, "logp/rejected": -356.0, "loss": 0.2947807312011719, "reward/accuracy": 0.875, "reward/chosen": -1.5234375, "reward/margin": 2.890625, "reward/rejected": -4.40625, "step": 2180 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 856.0, "epoch": 0.6541691661667667, "grad_norm": 5.707287788391113, "learning_rate": 2.6760632398191442e-06, "logp/chosen": -276.0, "logp/rejected": -244.0, "loss": 0.775390625, "reward/accuracy": 0.5625, "reward/chosen": -1.859375, "reward/margin": 1.1640625, "reward/rejected": -3.03125, "step": 2181 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 484.0, "epoch": 0.6544691061787642, "grad_norm": 4.5813164710998535, "learning_rate": 2.6718926520122496e-06, "logp/chosen": -282.0, "logp/rejected": -266.0, "loss": 0.677337646484375, "reward/accuracy": 0.6875, "reward/chosen": -1.5859375, "reward/margin": 1.0078125, "reward/rejected": -2.59375, "step": 2182 }, { "approx. KL/chosen": 352.0, "approx. KL/rejected": 528.0, "epoch": 0.6547690461907618, "grad_norm": 6.551177978515625, "learning_rate": 2.667724131353131e-06, "logp/chosen": -430.0, "logp/rejected": -368.0, "loss": 0.4771728515625, "reward/accuracy": 0.75, "reward/chosen": -1.859375, "reward/margin": 1.078125, "reward/rejected": -2.9375, "step": 2183 }, { "approx. KL/chosen": 446.0, "approx. KL/rejected": 548.0, "epoch": 0.6550689862027594, "grad_norm": 5.845427513122559, "learning_rate": 2.6635576815430497e-06, "logp/chosen": -344.0, "logp/rejected": -294.0, "loss": 0.9949951171875, "reward/accuracy": 0.4375, "reward/chosen": -2.421875, "reward/margin": 0.10302734375, "reward/rejected": -2.53125, "step": 2184 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 764.0, "epoch": 0.6553689262147571, "grad_norm": 4.934048175811768, "learning_rate": 2.6593933062814385e-06, "logp/chosen": -255.0, "logp/rejected": -300.0, "loss": 0.5682373046875, "reward/accuracy": 0.75, "reward/chosen": -1.09375, "reward/margin": 1.6875, "reward/rejected": -2.78125, "step": 2185 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 1008.0, "epoch": 0.6556688662267547, "grad_norm": 3.087146520614624, "learning_rate": 2.6552310092658817e-06, "logp/chosen": -352.0, "logp/rejected": -352.0, "loss": 0.373291015625, "reward/accuracy": 0.8125, "reward/chosen": -1.125, "reward/margin": 2.09375, "reward/rejected": -3.234375, "step": 2186 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 956.0, "epoch": 0.6559688062387522, "grad_norm": 4.61385440826416, "learning_rate": 2.6510707941921245e-06, "logp/chosen": -214.0, "logp/rejected": -280.0, "loss": 0.6068878173828125, "reward/accuracy": 0.625, "reward/chosen": -0.9921875, "reward/margin": 2.09375, "reward/rejected": -3.09375, "step": 2187 }, { "approx. KL/chosen": 63.25, "approx. KL/rejected": 752.0, "epoch": 0.6562687462507498, "grad_norm": 1.765334963798523, "learning_rate": 2.6469126647540565e-06, "logp/chosen": -464.0, "logp/rejected": -400.0, "loss": 0.14272308349609375, "reward/accuracy": 0.9375, "reward/chosen": -0.7734375, "reward/margin": 2.765625, "reward/rejected": -3.53125, "step": 2188 }, { "approx. KL/chosen": 142.0, "approx. KL/rejected": 532.0, "epoch": 0.6565686862627474, "grad_norm": 5.677891731262207, "learning_rate": 2.6427566246437214e-06, "logp/chosen": -316.0, "logp/rejected": -318.0, "loss": 0.440185546875, "reward/accuracy": 0.625, "reward/chosen": -1.265625, "reward/margin": 1.4140625, "reward/rejected": -2.671875, "step": 2189 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 676.0, "epoch": 0.6568686262747451, "grad_norm": 2.3621747493743896, "learning_rate": 2.6386026775513015e-06, "logp/chosen": -276.0, "logp/rejected": -284.0, "loss": 0.241058349609375, "reward/accuracy": 0.875, "reward/chosen": -1.2265625, "reward/margin": 2.015625, "reward/rejected": -3.25, "step": 2190 }, { "approx. KL/chosen": 171.0, "approx. KL/rejected": 888.0, "epoch": 0.6571685662867427, "grad_norm": 4.291972637176514, "learning_rate": 2.634450827165128e-06, "logp/chosen": -292.0, "logp/rejected": -346.0, "loss": 0.4833831787109375, "reward/accuracy": 0.8125, "reward/chosen": -1.359375, "reward/margin": 2.09375, "reward/rejected": -3.453125, "step": 2191 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 640.0, "epoch": 0.6574685062987402, "grad_norm": 3.2500064373016357, "learning_rate": 2.6303010771716635e-06, "logp/chosen": -229.0, "logp/rejected": -264.0, "loss": 0.3186798095703125, "reward/accuracy": 0.8125, "reward/chosen": -1.234375, "reward/margin": 1.9140625, "reward/rejected": -3.140625, "step": 2192 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 552.0, "epoch": 0.6577684463107378, "grad_norm": 3.7434301376342773, "learning_rate": 2.626153431255515e-06, "logp/chosen": -354.0, "logp/rejected": -296.0, "loss": 0.4972991943359375, "reward/accuracy": 0.75, "reward/chosen": -1.21875, "reward/margin": 1.4921875, "reward/rejected": -2.703125, "step": 2193 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 756.0, "epoch": 0.6580683863227355, "grad_norm": 5.459071636199951, "learning_rate": 2.622007893099407e-06, "logp/chosen": -332.0, "logp/rejected": -316.0, "loss": 0.366485595703125, "reward/accuracy": 0.8125, "reward/chosen": -1.078125, "reward/margin": 2.25, "reward/rejected": -3.328125, "step": 2194 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 980.0, "epoch": 0.6583683263347331, "grad_norm": 3.8881614208221436, "learning_rate": 2.617864466384207e-06, "logp/chosen": -366.0, "logp/rejected": -328.0, "loss": 0.337005615234375, "reward/accuracy": 0.6875, "reward/chosen": -1.65625, "reward/margin": 2.0, "reward/rejected": -3.65625, "step": 2195 }, { "approx. KL/chosen": 446.0, "approx. KL/rejected": 968.0, "epoch": 0.6586682663467307, "grad_norm": 8.020649909973145, "learning_rate": 2.6137231547888964e-06, "logp/chosen": -264.0, "logp/rejected": -286.0, "loss": 0.530364990234375, "reward/accuracy": 0.6875, "reward/chosen": -2.078125, "reward/margin": 1.65625, "reward/rejected": -3.734375, "step": 2196 }, { "approx. KL/chosen": 99.0, "approx. KL/rejected": 484.0, "epoch": 0.6589682063587282, "grad_norm": 2.972771167755127, "learning_rate": 2.6095839619905882e-06, "logp/chosen": -306.0, "logp/rejected": -274.0, "loss": 0.306396484375, "reward/accuracy": 0.9375, "reward/chosen": -1.015625, "reward/margin": 1.7265625, "reward/rejected": -2.734375, "step": 2197 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 756.0, "epoch": 0.6592681463707258, "grad_norm": 2.9851324558258057, "learning_rate": 2.605446891664508e-06, "logp/chosen": -298.0, "logp/rejected": -390.0, "loss": 0.2897453308105469, "reward/accuracy": 0.8125, "reward/chosen": -1.1640625, "reward/margin": 2.125, "reward/rejected": -3.28125, "step": 2198 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 536.0, "epoch": 0.6595680863827235, "grad_norm": 4.205842971801758, "learning_rate": 2.601311947483998e-06, "logp/chosen": -332.0, "logp/rejected": -384.0, "loss": 0.560943603515625, "reward/accuracy": 0.6875, "reward/chosen": -1.3359375, "reward/margin": 1.265625, "reward/rejected": -2.59375, "step": 2199 }, { "approx. KL/chosen": 324.0, "approx. KL/rejected": 652.0, "epoch": 0.6598680263947211, "grad_norm": 4.355844974517822, "learning_rate": 2.59717913312051e-06, "logp/chosen": -386.0, "logp/rejected": -386.0, "loss": 0.53582763671875, "reward/accuracy": 0.6875, "reward/chosen": -1.9375, "reward/margin": 1.3046875, "reward/rejected": -3.25, "step": 2200 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 752.0, "epoch": 0.6601679664067187, "grad_norm": 8.821385383605957, "learning_rate": 2.5930484522436126e-06, "logp/chosen": -382.0, "logp/rejected": -336.0, "loss": 0.740203857421875, "reward/accuracy": 0.5, "reward/chosen": -1.828125, "reward/margin": 1.0859375, "reward/rejected": -2.921875, "step": 2201 }, { "approx. KL/chosen": 232.0, "approx. KL/rejected": 1256.0, "epoch": 0.6604679064187162, "grad_norm": 4.458759784698486, "learning_rate": 2.5889199085209732e-06, "logp/chosen": -296.0, "logp/rejected": -386.0, "loss": 0.45159912109375, "reward/accuracy": 0.6875, "reward/chosen": -1.7421875, "reward/margin": 2.265625, "reward/rejected": -4.0, "step": 2202 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 452.0, "epoch": 0.6607678464307138, "grad_norm": 4.881795406341553, "learning_rate": 2.5847935056183627e-06, "logp/chosen": -356.0, "logp/rejected": -330.0, "loss": 0.520904541015625, "reward/accuracy": 0.75, "reward/chosen": -1.3828125, "reward/margin": 1.1953125, "reward/rejected": -2.578125, "step": 2203 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 688.0, "epoch": 0.6610677864427115, "grad_norm": 3.906876564025879, "learning_rate": 2.580669247199651e-06, "logp/chosen": -390.0, "logp/rejected": -338.0, "loss": 0.4625244140625, "reward/accuracy": 0.6875, "reward/chosen": -1.828125, "reward/margin": 1.21875, "reward/rejected": -3.046875, "step": 2204 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 422.0, "epoch": 0.6613677264547091, "grad_norm": 4.43521785736084, "learning_rate": 2.576547136926807e-06, "logp/chosen": -412.0, "logp/rejected": -338.0, "loss": 0.48590087890625, "reward/accuracy": 0.6875, "reward/chosen": -1.453125, "reward/margin": 1.0390625, "reward/rejected": -2.5, "step": 2205 }, { "approx. KL/chosen": 109.0, "approx. KL/rejected": 804.0, "epoch": 0.6616676664667066, "grad_norm": 3.40663480758667, "learning_rate": 2.5724271784598896e-06, "logp/chosen": -334.0, "logp/rejected": -328.0, "loss": 0.4486083984375, "reward/accuracy": 0.75, "reward/chosen": -1.0625, "reward/margin": 2.21875, "reward/rejected": -3.28125, "step": 2206 }, { "approx. KL/chosen": 58.5, "approx. KL/rejected": 656.0, "epoch": 0.6619676064787042, "grad_norm": 2.641075611114502, "learning_rate": 2.568309375457048e-06, "logp/chosen": -366.0, "logp/rejected": -380.0, "loss": 0.28985595703125, "reward/accuracy": 0.875, "reward/chosen": -0.62109375, "reward/margin": 2.3125, "reward/rejected": -2.9375, "step": 2207 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 780.0, "epoch": 0.6622675464907019, "grad_norm": 2.9855148792266846, "learning_rate": 2.5641937315745134e-06, "logp/chosen": -414.0, "logp/rejected": -332.0, "loss": 0.30511474609375, "reward/accuracy": 0.875, "reward/chosen": -1.578125, "reward/margin": 1.8828125, "reward/rejected": -3.46875, "step": 2208 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 716.0, "epoch": 0.6625674865026995, "grad_norm": 4.56935453414917, "learning_rate": 2.560080250466608e-06, "logp/chosen": -344.0, "logp/rejected": -276.0, "loss": 0.381622314453125, "reward/accuracy": 0.8125, "reward/chosen": -1.421875, "reward/margin": 1.96875, "reward/rejected": -3.390625, "step": 2209 }, { "approx. KL/chosen": 245.0, "approx. KL/rejected": 250.0, "epoch": 0.6628674265146971, "grad_norm": 6.683973789215088, "learning_rate": 2.5559689357857266e-06, "logp/chosen": -270.0, "logp/rejected": -318.0, "loss": 0.8450927734375, "reward/accuracy": 0.5625, "reward/chosen": -1.6328125, "reward/margin": 0.205078125, "reward/rejected": -1.8359375, "step": 2210 }, { "approx. KL/chosen": 248.0, "approx. KL/rejected": 356.0, "epoch": 0.6631673665266946, "grad_norm": 4.538270473480225, "learning_rate": 2.551859791182345e-06, "logp/chosen": -294.0, "logp/rejected": -198.0, "loss": 0.54150390625, "reward/accuracy": 0.75, "reward/chosen": -1.34375, "reward/margin": 0.92578125, "reward/rejected": -2.265625, "step": 2211 }, { "approx. KL/chosen": 238.0, "approx. KL/rejected": 1256.0, "epoch": 0.6634673065386922, "grad_norm": 4.682302474975586, "learning_rate": 2.54775282030501e-06, "logp/chosen": -322.0, "logp/rejected": -278.0, "loss": 0.4634056091308594, "reward/accuracy": 0.8125, "reward/chosen": -1.7890625, "reward/margin": 2.6875, "reward/rejected": -4.46875, "step": 2212 }, { "approx. KL/chosen": 209.0, "approx. KL/rejected": 652.0, "epoch": 0.6637672465506899, "grad_norm": 4.653192043304443, "learning_rate": 2.5436480268003373e-06, "logp/chosen": -294.0, "logp/rejected": -406.0, "loss": 0.58642578125, "reward/accuracy": 0.5625, "reward/chosen": -1.5703125, "reward/margin": 1.21875, "reward/rejected": -2.796875, "step": 2213 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 672.0, "epoch": 0.6640671865626875, "grad_norm": 3.546755313873291, "learning_rate": 2.5395454143130094e-06, "logp/chosen": -346.0, "logp/rejected": -356.0, "loss": 0.4073486328125, "reward/accuracy": 0.875, "reward/chosen": -1.78125, "reward/margin": 1.484375, "reward/rejected": -3.265625, "step": 2214 }, { "approx. KL/chosen": 100.0, "approx. KL/rejected": 572.0, "epoch": 0.6643671265746851, "grad_norm": 4.577651023864746, "learning_rate": 2.535444986485777e-06, "logp/chosen": -214.0, "logp/rejected": -286.0, "loss": 0.4893646240234375, "reward/accuracy": 0.6875, "reward/chosen": -0.96875, "reward/margin": 1.7578125, "reward/rejected": -2.734375, "step": 2215 }, { "approx. KL/chosen": 117.5, "approx. KL/rejected": 828.0, "epoch": 0.6646670665866826, "grad_norm": 2.0443406105041504, "learning_rate": 2.5313467469594454e-06, "logp/chosen": -312.0, "logp/rejected": -332.0, "loss": 0.17840576171875, "reward/accuracy": 1.0, "reward/chosen": -1.0078125, "reward/margin": 2.4375, "reward/rejected": -3.453125, "step": 2216 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 528.0, "epoch": 0.6649670065986802, "grad_norm": 4.089347839355469, "learning_rate": 2.5272506993728795e-06, "logp/chosen": -366.0, "logp/rejected": -532.0, "loss": 0.50006103515625, "reward/accuracy": 0.6875, "reward/chosen": -1.0078125, "reward/margin": 1.515625, "reward/rejected": -2.53125, "step": 2217 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 880.0, "epoch": 0.6652669466106779, "grad_norm": 4.275418281555176, "learning_rate": 2.523156847362994e-06, "logp/chosen": -342.0, "logp/rejected": -386.0, "loss": 0.3408203125, "reward/accuracy": 0.875, "reward/chosen": -1.296875, "reward/margin": 2.40625, "reward/rejected": -3.703125, "step": 2218 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 370.0, "epoch": 0.6655668866226755, "grad_norm": 4.9832329750061035, "learning_rate": 2.5190651945647625e-06, "logp/chosen": -354.0, "logp/rejected": -404.0, "loss": 0.556640625, "reward/accuracy": 0.75, "reward/chosen": -1.1328125, "reward/margin": 0.95703125, "reward/rejected": -2.09375, "step": 2219 }, { "approx. KL/chosen": 346.0, "approx. KL/rejected": 988.0, "epoch": 0.6658668266346731, "grad_norm": 5.620014667510986, "learning_rate": 2.514975744611199e-06, "logp/chosen": -252.0, "logp/rejected": -310.0, "loss": 0.7860794067382812, "reward/accuracy": 0.5625, "reward/chosen": -1.921875, "reward/margin": 1.640625, "reward/rejected": -3.5625, "step": 2220 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 780.0, "epoch": 0.6661667666466706, "grad_norm": 2.953900098800659, "learning_rate": 2.510888501133361e-06, "logp/chosen": -280.0, "logp/rejected": -296.0, "loss": 0.3585205078125, "reward/accuracy": 0.8125, "reward/chosen": -0.87109375, "reward/margin": 2.328125, "reward/rejected": -3.1875, "step": 2221 }, { "approx. KL/chosen": 324.0, "approx. KL/rejected": 544.0, "epoch": 0.6664667066586683, "grad_norm": 5.085169792175293, "learning_rate": 2.506803467760354e-06, "logp/chosen": -364.0, "logp/rejected": -282.0, "loss": 0.654296875, "reward/accuracy": 0.75, "reward/chosen": -1.7421875, "reward/margin": 1.078125, "reward/rejected": -2.8125, "step": 2222 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 652.0, "epoch": 0.6667666466706659, "grad_norm": 4.927050590515137, "learning_rate": 2.5027206481193123e-06, "logp/chosen": -426.0, "logp/rejected": -456.0, "loss": 0.424072265625, "reward/accuracy": 0.625, "reward/chosen": -1.1640625, "reward/margin": 1.703125, "reward/rejected": -2.859375, "step": 2223 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 916.0, "epoch": 0.6670665866826635, "grad_norm": 3.6907925605773926, "learning_rate": 2.4986400458354116e-06, "logp/chosen": -390.0, "logp/rejected": -350.0, "loss": 0.484375, "reward/accuracy": 0.75, "reward/chosen": -1.671875, "reward/margin": 1.96875, "reward/rejected": -3.640625, "step": 2224 }, { "approx. KL/chosen": 66.0, "approx. KL/rejected": 488.0, "epoch": 0.6673665266946611, "grad_norm": 4.25689697265625, "learning_rate": 2.494561664531852e-06, "logp/chosen": -386.0, "logp/rejected": -384.0, "loss": 0.3660888671875, "reward/accuracy": 0.75, "reward/chosen": -0.55078125, "reward/margin": 1.9765625, "reward/rejected": -2.53125, "step": 2225 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 912.0, "epoch": 0.6676664667066586, "grad_norm": 4.43934965133667, "learning_rate": 2.4904855078298705e-06, "logp/chosen": -384.0, "logp/rejected": -388.0, "loss": 0.394775390625, "reward/accuracy": 0.8125, "reward/chosen": -1.7421875, "reward/margin": 1.890625, "reward/rejected": -3.625, "step": 2226 }, { "approx. KL/chosen": 86.5, "approx. KL/rejected": 608.0, "epoch": 0.6679664067186563, "grad_norm": 3.105970859527588, "learning_rate": 2.4864115793487216e-06, "logp/chosen": -396.0, "logp/rejected": -492.0, "loss": 0.2830657958984375, "reward/accuracy": 0.8125, "reward/chosen": -0.7578125, "reward/margin": 2.21875, "reward/rejected": -2.96875, "step": 2227 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 1032.0, "epoch": 0.6682663467306539, "grad_norm": 3.2276861667633057, "learning_rate": 2.482339882705681e-06, "logp/chosen": -205.0, "logp/rejected": -354.0, "loss": 0.26592254638671875, "reward/accuracy": 0.875, "reward/chosen": -1.40625, "reward/margin": 2.46875, "reward/rejected": -3.875, "step": 2228 }, { "approx. KL/chosen": 229.0, "approx. KL/rejected": 506.0, "epoch": 0.6685662867426515, "grad_norm": 3.597522020339966, "learning_rate": 2.4782704215160504e-06, "logp/chosen": -316.0, "logp/rejected": -356.0, "loss": 0.39892578125, "reward/accuracy": 0.875, "reward/chosen": -1.2265625, "reward/margin": 1.3671875, "reward/rejected": -2.59375, "step": 2229 }, { "approx. KL/chosen": 125.0, "approx. KL/rejected": 504.0, "epoch": 0.668866226754649, "grad_norm": 3.8870160579681396, "learning_rate": 2.4742031993931383e-06, "logp/chosen": -290.0, "logp/rejected": -368.0, "loss": 0.4442138671875, "reward/accuracy": 0.6875, "reward/chosen": -1.09375, "reward/margin": 1.40625, "reward/rejected": -2.5, "step": 2230 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 740.0, "epoch": 0.6691661667666466, "grad_norm": 2.97121524810791, "learning_rate": 2.470138219948269e-06, "logp/chosen": -219.0, "logp/rejected": -282.0, "loss": 0.2850494384765625, "reward/accuracy": 0.9375, "reward/chosen": -1.5390625, "reward/margin": 1.9296875, "reward/rejected": -3.46875, "step": 2231 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 392.0, "epoch": 0.6694661067786443, "grad_norm": 4.367342948913574, "learning_rate": 2.4660754867907737e-06, "logp/chosen": -233.0, "logp/rejected": -228.0, "loss": 0.610595703125, "reward/accuracy": 0.6875, "reward/chosen": -1.078125, "reward/margin": 1.0, "reward/rejected": -2.078125, "step": 2232 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 464.0, "epoch": 0.6697660467906419, "grad_norm": 5.256435871124268, "learning_rate": 2.4620150035279927e-06, "logp/chosen": -284.0, "logp/rejected": -266.0, "loss": 0.78900146484375, "reward/accuracy": 0.625, "reward/chosen": -1.5859375, "reward/margin": 0.90625, "reward/rejected": -2.5, "step": 2233 }, { "approx. KL/chosen": 82.5, "approx. KL/rejected": 396.0, "epoch": 0.6700659868026395, "grad_norm": 3.957406997680664, "learning_rate": 2.4579567737652653e-06, "logp/chosen": -418.0, "logp/rejected": -318.0, "loss": 0.3321533203125, "reward/accuracy": 0.8125, "reward/chosen": -0.85546875, "reward/margin": 1.59375, "reward/rejected": -2.4375, "step": 2234 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 620.0, "epoch": 0.670365926814637, "grad_norm": 3.7274599075317383, "learning_rate": 2.4539008011059284e-06, "logp/chosen": -330.0, "logp/rejected": -388.0, "loss": 0.33795166015625, "reward/accuracy": 0.75, "reward/chosen": -1.3359375, "reward/margin": 1.6484375, "reward/rejected": -2.984375, "step": 2235 }, { "approx. KL/chosen": 125.5, "approx. KL/rejected": 528.0, "epoch": 0.6706658668266346, "grad_norm": 3.6015396118164062, "learning_rate": 2.449847089151321e-06, "logp/chosen": -256.0, "logp/rejected": -264.0, "loss": 0.46954345703125, "reward/accuracy": 0.875, "reward/chosen": -1.234375, "reward/margin": 1.65625, "reward/rejected": -2.890625, "step": 2236 }, { "approx. KL/chosen": 219.0, "approx. KL/rejected": 516.0, "epoch": 0.6709658068386323, "grad_norm": 5.437499523162842, "learning_rate": 2.4457956415007694e-06, "logp/chosen": -298.0, "logp/rejected": -264.0, "loss": 0.6180267333984375, "reward/accuracy": 0.4375, "reward/chosen": -1.5859375, "reward/margin": 0.89453125, "reward/rejected": -2.484375, "step": 2237 }, { "approx. KL/chosen": 127.5, "approx. KL/rejected": 612.0, "epoch": 0.6712657468506299, "grad_norm": 2.892616033554077, "learning_rate": 2.4417464617515903e-06, "logp/chosen": -320.0, "logp/rejected": -382.0, "loss": 0.301116943359375, "reward/accuracy": 0.875, "reward/chosen": -1.1171875, "reward/margin": 1.7890625, "reward/rejected": -2.90625, "step": 2238 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 752.0, "epoch": 0.6715656868626275, "grad_norm": 4.118109226226807, "learning_rate": 2.437699553499086e-06, "logp/chosen": -466.0, "logp/rejected": -350.0, "loss": 0.497833251953125, "reward/accuracy": 0.625, "reward/chosen": -1.53125, "reward/margin": 1.515625, "reward/rejected": -3.046875, "step": 2239 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 652.0, "epoch": 0.671865626874625, "grad_norm": 3.886809825897217, "learning_rate": 2.4336549203365466e-06, "logp/chosen": -326.0, "logp/rejected": -336.0, "loss": 0.44317626953125, "reward/accuracy": 0.75, "reward/chosen": -1.3125, "reward/margin": 1.625, "reward/rejected": -2.9375, "step": 2240 }, { "approx. KL/chosen": 130.0, "approx. KL/rejected": 380.0, "epoch": 0.6721655668866227, "grad_norm": 4.327596664428711, "learning_rate": 2.429612565855238e-06, "logp/chosen": -302.0, "logp/rejected": -272.0, "loss": 0.5472412109375, "reward/accuracy": 0.6875, "reward/chosen": -1.0546875, "reward/margin": 1.1875, "reward/rejected": -2.25, "step": 2241 }, { "approx. KL/chosen": 123.5, "approx. KL/rejected": 728.0, "epoch": 0.6724655068986203, "grad_norm": 3.3107447624206543, "learning_rate": 2.4255724936444025e-06, "logp/chosen": -352.0, "logp/rejected": -352.0, "loss": 0.27716064453125, "reward/accuracy": 0.75, "reward/chosen": -1.015625, "reward/margin": 2.375, "reward/rejected": -3.390625, "step": 2242 }, { "approx. KL/chosen": 91.0, "approx. KL/rejected": 370.0, "epoch": 0.6727654469106179, "grad_norm": 3.220658779144287, "learning_rate": 2.4215347072912555e-06, "logp/chosen": -220.0, "logp/rejected": -225.0, "loss": 0.4578857421875, "reward/accuracy": 0.75, "reward/chosen": -0.9921875, "reward/margin": 1.2734375, "reward/rejected": -2.265625, "step": 2243 }, { "approx. KL/chosen": 193.0, "approx. KL/rejected": 824.0, "epoch": 0.6730653869226155, "grad_norm": 3.00734281539917, "learning_rate": 2.417499210380988e-06, "logp/chosen": -346.0, "logp/rejected": -272.0, "loss": 0.32269287109375, "reward/accuracy": 0.875, "reward/chosen": -0.90234375, "reward/margin": 2.671875, "reward/rejected": -3.578125, "step": 2244 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 784.0, "epoch": 0.673365326934613, "grad_norm": 3.3961081504821777, "learning_rate": 2.4134660064967518e-06, "logp/chosen": -316.0, "logp/rejected": -338.0, "loss": 0.433990478515625, "reward/accuracy": 0.75, "reward/chosen": -1.1875, "reward/margin": 1.9140625, "reward/rejected": -3.109375, "step": 2245 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 556.0, "epoch": 0.6736652669466107, "grad_norm": 3.210146188735962, "learning_rate": 2.409435099219668e-06, "logp/chosen": -294.0, "logp/rejected": -308.0, "loss": 0.31915283203125, "reward/accuracy": 0.875, "reward/chosen": -1.1953125, "reward/margin": 1.640625, "reward/rejected": -2.828125, "step": 2246 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 322.0, "epoch": 0.6739652069586083, "grad_norm": 4.74835729598999, "learning_rate": 2.4054064921288144e-06, "logp/chosen": -354.0, "logp/rejected": -452.0, "loss": 0.60162353515625, "reward/accuracy": 0.5625, "reward/chosen": -1.03125, "reward/margin": 0.80078125, "reward/rejected": -1.8359375, "step": 2247 }, { "approx. KL/chosen": 107.0, "approx. KL/rejected": 476.0, "epoch": 0.6742651469706059, "grad_norm": 3.6153836250305176, "learning_rate": 2.401380188801229e-06, "logp/chosen": -426.0, "logp/rejected": -422.0, "loss": 0.35784912109375, "reward/accuracy": 0.75, "reward/chosen": -0.8984375, "reward/margin": 1.703125, "reward/rejected": -2.609375, "step": 2248 }, { "approx. KL/chosen": 124.0, "approx. KL/rejected": 504.0, "epoch": 0.6745650869826035, "grad_norm": 4.221826553344727, "learning_rate": 2.3973561928118997e-06, "logp/chosen": -366.0, "logp/rejected": -266.0, "loss": 0.39874267578125, "reward/accuracy": 0.875, "reward/chosen": -0.859375, "reward/margin": 2.03125, "reward/rejected": -2.890625, "step": 2249 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 492.0, "epoch": 0.674865026994601, "grad_norm": 4.248298645019531, "learning_rate": 2.3933345077337744e-06, "logp/chosen": -320.0, "logp/rejected": -344.0, "loss": 0.514434814453125, "reward/accuracy": 0.625, "reward/chosen": -1.2109375, "reward/margin": 1.390625, "reward/rejected": -2.609375, "step": 2250 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 492.0, "epoch": 0.6751649670065987, "grad_norm": 4.283597469329834, "learning_rate": 2.3893151371377423e-06, "logp/chosen": -412.0, "logp/rejected": -352.0, "loss": 0.5677490234375, "reward/accuracy": 0.75, "reward/chosen": -1.578125, "reward/margin": 1.0078125, "reward/rejected": -2.59375, "step": 2251 }, { "approx. KL/chosen": 82.5, "approx. KL/rejected": 684.0, "epoch": 0.6754649070185963, "grad_norm": 3.4990334510803223, "learning_rate": 2.3852980845926377e-06, "logp/chosen": -229.0, "logp/rejected": -242.0, "loss": 0.35198974609375, "reward/accuracy": 0.75, "reward/chosen": -1.0078125, "reward/margin": 1.953125, "reward/rejected": -2.953125, "step": 2252 }, { "approx. KL/chosen": 119.5, "approx. KL/rejected": 728.0, "epoch": 0.6757648470305939, "grad_norm": 3.1588499546051025, "learning_rate": 2.3812833536652373e-06, "logp/chosen": -300.0, "logp/rejected": -390.0, "loss": 0.338104248046875, "reward/accuracy": 0.875, "reward/chosen": -1.1171875, "reward/margin": 1.953125, "reward/rejected": -3.078125, "step": 2253 }, { "approx. KL/chosen": 141.0, "approx. KL/rejected": 462.0, "epoch": 0.6760647870425914, "grad_norm": 4.774737358093262, "learning_rate": 2.3772709479202618e-06, "logp/chosen": -516.0, "logp/rejected": -480.0, "loss": 0.625732421875, "reward/accuracy": 0.625, "reward/chosen": -1.3359375, "reward/margin": 1.0703125, "reward/rejected": -2.40625, "step": 2254 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 912.0, "epoch": 0.676364727054589, "grad_norm": 4.997222423553467, "learning_rate": 2.3732608709203597e-06, "logp/chosen": -388.0, "logp/rejected": -382.0, "loss": 0.6207275390625, "reward/accuracy": 0.625, "reward/chosen": -1.6640625, "reward/margin": 1.4921875, "reward/rejected": -3.15625, "step": 2255 }, { "approx. KL/chosen": 330.0, "approx. KL/rejected": 1216.0, "epoch": 0.6766646670665867, "grad_norm": 3.264848470687866, "learning_rate": 2.3692531262261154e-06, "logp/chosen": -348.0, "logp/rejected": -260.0, "loss": 0.3384857177734375, "reward/accuracy": 0.875, "reward/chosen": -1.9765625, "reward/margin": 2.125, "reward/rejected": -4.09375, "step": 2256 }, { "approx. KL/chosen": 328.0, "approx. KL/rejected": 560.0, "epoch": 0.6769646070785843, "grad_norm": 5.002029895782471, "learning_rate": 2.365247717396041e-06, "logp/chosen": -300.0, "logp/rejected": -320.0, "loss": 0.6446533203125, "reward/accuracy": 0.75, "reward/chosen": -1.984375, "reward/margin": 0.79296875, "reward/rejected": -2.78125, "step": 2257 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 640.0, "epoch": 0.6772645470905819, "grad_norm": 3.910688638687134, "learning_rate": 2.361244647986578e-06, "logp/chosen": -358.0, "logp/rejected": -396.0, "loss": 0.5057373046875, "reward/accuracy": 0.6875, "reward/chosen": -1.4765625, "reward/margin": 1.515625, "reward/rejected": -2.984375, "step": 2258 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 736.0, "epoch": 0.6775644871025794, "grad_norm": 2.9493026733398438, "learning_rate": 2.357243921552086e-06, "logp/chosen": -250.0, "logp/rejected": -294.0, "loss": 0.307708740234375, "reward/accuracy": 0.8125, "reward/chosen": -1.4140625, "reward/margin": 1.9453125, "reward/rejected": -3.359375, "step": 2259 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 808.0, "epoch": 0.6778644271145771, "grad_norm": 3.1525230407714844, "learning_rate": 2.353245541644851e-06, "logp/chosen": -240.0, "logp/rejected": -356.0, "loss": 0.422393798828125, "reward/accuracy": 0.75, "reward/chosen": -1.1484375, "reward/margin": 1.9765625, "reward/rejected": -3.125, "step": 2260 }, { "approx. KL/chosen": 174.0, "approx. KL/rejected": 492.0, "epoch": 0.6781643671265747, "grad_norm": 3.8574438095092773, "learning_rate": 2.349249511815064e-06, "logp/chosen": -225.0, "logp/rejected": -288.0, "loss": 0.5466461181640625, "reward/accuracy": 0.6875, "reward/chosen": -1.4921875, "reward/margin": 0.95703125, "reward/rejected": -2.453125, "step": 2261 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 552.0, "epoch": 0.6784643071385723, "grad_norm": 5.6187639236450195, "learning_rate": 2.3452558356108414e-06, "logp/chosen": -244.0, "logp/rejected": -290.0, "loss": 0.470947265625, "reward/accuracy": 0.75, "reward/chosen": -1.5, "reward/margin": 1.1953125, "reward/rejected": -2.6875, "step": 2262 }, { "approx. KL/chosen": 226.0, "approx. KL/rejected": 500.0, "epoch": 0.6787642471505699, "grad_norm": 4.267927169799805, "learning_rate": 2.3412645165782016e-06, "logp/chosen": -382.0, "logp/rejected": -462.0, "loss": 0.46142578125, "reward/accuracy": 0.75, "reward/chosen": -1.8671875, "reward/margin": 0.921875, "reward/rejected": -2.78125, "step": 2263 }, { "approx. KL/chosen": 380.0, "approx. KL/rejected": 628.0, "epoch": 0.6790641871625674, "grad_norm": 8.583624839782715, "learning_rate": 2.3372755582610773e-06, "logp/chosen": -298.0, "logp/rejected": -322.0, "loss": 0.52978515625, "reward/accuracy": 0.875, "reward/chosen": -1.890625, "reward/margin": 1.28125, "reward/rejected": -3.171875, "step": 2264 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 328.0, "epoch": 0.6793641271745651, "grad_norm": 4.561676979064941, "learning_rate": 2.3332889642012986e-06, "logp/chosen": -342.0, "logp/rejected": -382.0, "loss": 0.5826416015625, "reward/accuracy": 0.6875, "reward/chosen": -1.4609375, "reward/margin": 0.7265625, "reward/rejected": -2.1875, "step": 2265 }, { "approx. KL/chosen": 108.5, "approx. KL/rejected": 760.0, "epoch": 0.6796640671865627, "grad_norm": 2.135019540786743, "learning_rate": 2.3293047379385985e-06, "logp/chosen": -314.0, "logp/rejected": -338.0, "loss": 0.2468719482421875, "reward/accuracy": 0.875, "reward/chosen": -0.8359375, "reward/margin": 2.59375, "reward/rejected": -3.421875, "step": 2266 }, { "approx. KL/chosen": 126.5, "approx. KL/rejected": 496.0, "epoch": 0.6799640071985603, "grad_norm": 4.266119956970215, "learning_rate": 2.3253228830106062e-06, "logp/chosen": -216.0, "logp/rejected": -354.0, "loss": 0.4705810546875, "reward/accuracy": 0.75, "reward/chosen": -1.296875, "reward/margin": 1.4375, "reward/rejected": -2.734375, "step": 2267 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 660.0, "epoch": 0.6802639472105579, "grad_norm": 5.254922866821289, "learning_rate": 2.3213434029528513e-06, "logp/chosen": -322.0, "logp/rejected": -342.0, "loss": 0.68719482421875, "reward/accuracy": 0.6875, "reward/chosen": -1.3828125, "reward/margin": 1.5625, "reward/rejected": -2.953125, "step": 2268 }, { "approx. KL/chosen": 141.0, "approx. KL/rejected": 700.0, "epoch": 0.6805638872225555, "grad_norm": 3.0995419025421143, "learning_rate": 2.317366301298747e-06, "logp/chosen": -241.0, "logp/rejected": -316.0, "loss": 0.3500823974609375, "reward/accuracy": 0.875, "reward/chosen": -1.2890625, "reward/margin": 1.8203125, "reward/rejected": -3.109375, "step": 2269 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 404.0, "epoch": 0.6808638272345531, "grad_norm": 5.596338272094727, "learning_rate": 2.3133915815796026e-06, "logp/chosen": -436.0, "logp/rejected": -400.0, "loss": 0.763031005859375, "reward/accuracy": 0.625, "reward/chosen": -1.3515625, "reward/margin": 0.96484375, "reward/rejected": -2.3125, "step": 2270 }, { "approx. KL/chosen": 364.0, "approx. KL/rejected": 760.0, "epoch": 0.6811637672465507, "grad_norm": 4.499546527862549, "learning_rate": 2.3094192473246007e-06, "logp/chosen": -544.0, "logp/rejected": -382.0, "loss": 0.515960693359375, "reward/accuracy": 0.8125, "reward/chosen": -1.953125, "reward/margin": 1.5078125, "reward/rejected": -3.453125, "step": 2271 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 704.0, "epoch": 0.6814637072585483, "grad_norm": 3.659477472305298, "learning_rate": 2.30544930206082e-06, "logp/chosen": -276.0, "logp/rejected": -276.0, "loss": 0.430908203125, "reward/accuracy": 0.75, "reward/chosen": -1.21875, "reward/margin": 1.6953125, "reward/rejected": -2.90625, "step": 2272 }, { "approx. KL/chosen": 184.0, "approx. KL/rejected": 860.0, "epoch": 0.6817636472705458, "grad_norm": 2.4293034076690674, "learning_rate": 2.301481749313206e-06, "logp/chosen": -372.0, "logp/rejected": -280.0, "loss": 0.33200645446777344, "reward/accuracy": 0.875, "reward/chosen": -1.1640625, "reward/margin": 2.4375, "reward/rejected": -3.609375, "step": 2273 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 516.0, "epoch": 0.6820635872825435, "grad_norm": 5.597037315368652, "learning_rate": 2.297516592604592e-06, "logp/chosen": -422.0, "logp/rejected": -390.0, "loss": 0.56683349609375, "reward/accuracy": 0.6875, "reward/chosen": -1.2890625, "reward/margin": 1.140625, "reward/rejected": -2.421875, "step": 2274 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 728.0, "epoch": 0.6823635272945411, "grad_norm": 3.810091733932495, "learning_rate": 2.293553835455668e-06, "logp/chosen": -276.0, "logp/rejected": -207.0, "loss": 0.700897216796875, "reward/accuracy": 0.75, "reward/chosen": -1.1796875, "reward/margin": 1.7890625, "reward/rejected": -2.96875, "step": 2275 }, { "approx. KL/chosen": 142.0, "approx. KL/rejected": 540.0, "epoch": 0.6826634673065387, "grad_norm": 3.1194677352905273, "learning_rate": 2.289593481385009e-06, "logp/chosen": -406.0, "logp/rejected": -366.0, "loss": 0.3719482421875, "reward/accuracy": 0.8125, "reward/chosen": -1.4296875, "reward/margin": 1.421875, "reward/rejected": -2.859375, "step": 2276 }, { "approx. KL/chosen": 342.0, "approx. KL/rejected": 872.0, "epoch": 0.6829634073185363, "grad_norm": 3.7448716163635254, "learning_rate": 2.285635533909045e-06, "logp/chosen": -300.0, "logp/rejected": -316.0, "loss": 0.4301586151123047, "reward/accuracy": 0.6875, "reward/chosen": -1.8828125, "reward/margin": 1.78125, "reward/rejected": -3.671875, "step": 2277 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 760.0, "epoch": 0.6832633473305338, "grad_norm": 2.95219349861145, "learning_rate": 2.2816799965420778e-06, "logp/chosen": -326.0, "logp/rejected": -314.0, "loss": 0.3748626708984375, "reward/accuracy": 0.8125, "reward/chosen": -1.328125, "reward/margin": 2.140625, "reward/rejected": -3.46875, "step": 2278 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 928.0, "epoch": 0.6835632873425315, "grad_norm": 5.434561252593994, "learning_rate": 2.2777268727962625e-06, "logp/chosen": -332.0, "logp/rejected": -362.0, "loss": 0.4773578643798828, "reward/accuracy": 0.75, "reward/chosen": -1.84375, "reward/margin": 1.765625, "reward/rejected": -3.609375, "step": 2279 }, { "approx. KL/chosen": 91.0, "approx. KL/rejected": 576.0, "epoch": 0.6838632273545291, "grad_norm": 3.36418080329895, "learning_rate": 2.2737761661816156e-06, "logp/chosen": -378.0, "logp/rejected": -374.0, "loss": 0.40796661376953125, "reward/accuracy": 0.8125, "reward/chosen": -1.15625, "reward/margin": 1.6328125, "reward/rejected": -2.78125, "step": 2280 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 708.0, "epoch": 0.6841631673665267, "grad_norm": 2.7743849754333496, "learning_rate": 2.269827880206002e-06, "logp/chosen": -358.0, "logp/rejected": -324.0, "loss": 0.287078857421875, "reward/accuracy": 0.9375, "reward/chosen": -1.4140625, "reward/margin": 1.84375, "reward/rejected": -3.25, "step": 2281 }, { "approx. KL/chosen": 290.0, "approx. KL/rejected": 1136.0, "epoch": 0.6844631073785243, "grad_norm": 3.332695245742798, "learning_rate": 2.265882018375145e-06, "logp/chosen": -243.0, "logp/rejected": -256.0, "loss": 0.340087890625, "reward/accuracy": 0.875, "reward/chosen": -1.921875, "reward/margin": 1.7734375, "reward/rejected": -3.703125, "step": 2282 }, { "approx. KL/chosen": 372.0, "approx. KL/rejected": 732.0, "epoch": 0.6847630473905219, "grad_norm": 5.898489475250244, "learning_rate": 2.261938584192608e-06, "logp/chosen": -318.0, "logp/rejected": -360.0, "loss": 0.71044921875, "reward/accuracy": 0.625, "reward/chosen": -2.171875, "reward/margin": 1.1328125, "reward/rejected": -3.296875, "step": 2283 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 412.0, "epoch": 0.6850629874025195, "grad_norm": 4.619510173797607, "learning_rate": 2.2579975811598074e-06, "logp/chosen": -388.0, "logp/rejected": -422.0, "loss": 0.609039306640625, "reward/accuracy": 0.6875, "reward/chosen": -1.4140625, "reward/margin": 1.0234375, "reward/rejected": -2.4375, "step": 2284 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 928.0, "epoch": 0.6853629274145171, "grad_norm": 3.265467882156372, "learning_rate": 2.2540590127759895e-06, "logp/chosen": -256.0, "logp/rejected": -346.0, "loss": 0.4019012451171875, "reward/accuracy": 0.75, "reward/chosen": -1.421875, "reward/margin": 2.046875, "reward/rejected": -3.46875, "step": 2285 }, { "approx. KL/chosen": 386.0, "approx. KL/rejected": 1004.0, "epoch": 0.6856628674265147, "grad_norm": 3.932183027267456, "learning_rate": 2.250122882538249e-06, "logp/chosen": -312.0, "logp/rejected": -348.0, "loss": 0.542083740234375, "reward/accuracy": 0.625, "reward/chosen": -2.21875, "reward/margin": 1.328125, "reward/rejected": -3.546875, "step": 2286 }, { "approx. KL/chosen": 107.0, "approx. KL/rejected": 1048.0, "epoch": 0.6859628074385123, "grad_norm": 2.8398759365081787, "learning_rate": 2.2461891939415087e-06, "logp/chosen": -338.0, "logp/rejected": -278.0, "loss": 0.188018798828125, "reward/accuracy": 0.9375, "reward/chosen": -1.015625, "reward/margin": 3.0, "reward/rejected": -4.0, "step": 2287 }, { "approx. KL/chosen": 107.5, "approx. KL/rejected": 756.0, "epoch": 0.6862627474505099, "grad_norm": 2.7721896171569824, "learning_rate": 2.2422579504785315e-06, "logp/chosen": -364.0, "logp/rejected": -278.0, "loss": 0.2595367431640625, "reward/accuracy": 0.9375, "reward/chosen": -1.0625, "reward/margin": 2.265625, "reward/rejected": -3.328125, "step": 2288 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 676.0, "epoch": 0.6865626874625075, "grad_norm": 4.554287433624268, "learning_rate": 2.2383291556399006e-06, "logp/chosen": -322.0, "logp/rejected": -308.0, "loss": 0.3941497802734375, "reward/accuracy": 0.75, "reward/chosen": -1.265625, "reward/margin": 2.015625, "reward/rejected": -3.28125, "step": 2289 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 564.0, "epoch": 0.6868626274745051, "grad_norm": 4.704160213470459, "learning_rate": 2.23440281291403e-06, "logp/chosen": -436.0, "logp/rejected": -430.0, "loss": 0.5010986328125, "reward/accuracy": 0.8125, "reward/chosen": -1.5625, "reward/margin": 1.3359375, "reward/rejected": -2.890625, "step": 2290 }, { "approx. KL/chosen": 354.0, "approx. KL/rejected": 744.0, "epoch": 0.6871625674865027, "grad_norm": 4.877643585205078, "learning_rate": 2.2304789257871534e-06, "logp/chosen": -380.0, "logp/rejected": -362.0, "loss": 0.551025390625, "reward/accuracy": 0.6875, "reward/chosen": -1.9453125, "reward/margin": 1.3125, "reward/rejected": -3.25, "step": 2291 }, { "approx. KL/chosen": 124.0, "approx. KL/rejected": 520.0, "epoch": 0.6874625074985004, "grad_norm": 3.3132643699645996, "learning_rate": 2.2265574977433297e-06, "logp/chosen": -334.0, "logp/rejected": -328.0, "loss": 0.34686279296875, "reward/accuracy": 0.75, "reward/chosen": -1.375, "reward/margin": 1.4765625, "reward/rejected": -2.84375, "step": 2292 }, { "approx. KL/chosen": 132.0, "approx. KL/rejected": 688.0, "epoch": 0.6877624475104979, "grad_norm": 3.3476662635803223, "learning_rate": 2.222638532264426e-06, "logp/chosen": -306.0, "logp/rejected": -310.0, "loss": 0.35724639892578125, "reward/accuracy": 0.875, "reward/chosen": -1.2734375, "reward/margin": 1.8828125, "reward/rejected": -3.15625, "step": 2293 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 564.0, "epoch": 0.6880623875224955, "grad_norm": 3.60683274269104, "learning_rate": 2.2187220328301346e-06, "logp/chosen": -358.0, "logp/rejected": -332.0, "loss": 0.377593994140625, "reward/accuracy": 0.75, "reward/chosen": -1.4453125, "reward/margin": 1.46875, "reward/rejected": -2.921875, "step": 2294 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 1144.0, "epoch": 0.6883623275344931, "grad_norm": 3.2672860622406006, "learning_rate": 2.214808002917943e-06, "logp/chosen": -364.0, "logp/rejected": -378.0, "loss": 0.1958770751953125, "reward/accuracy": 0.875, "reward/chosen": -2.015625, "reward/margin": 2.5, "reward/rejected": -4.5, "step": 2295 }, { "approx. KL/chosen": 91.5, "approx. KL/rejected": 422.0, "epoch": 0.6886622675464907, "grad_norm": 4.016824245452881, "learning_rate": 2.210896446003159e-06, "logp/chosen": -332.0, "logp/rejected": -235.0, "loss": 0.4219970703125, "reward/accuracy": 0.875, "reward/chosen": -0.8515625, "reward/margin": 1.453125, "reward/rejected": -2.3125, "step": 2296 }, { "approx. KL/chosen": 142.0, "approx. KL/rejected": 520.0, "epoch": 0.6889622075584882, "grad_norm": 2.438030242919922, "learning_rate": 2.2069873655588876e-06, "logp/chosen": -224.0, "logp/rejected": -231.0, "loss": 0.30462646484375, "reward/accuracy": 0.9375, "reward/chosen": -1.1875, "reward/margin": 1.8515625, "reward/rejected": -3.03125, "step": 2297 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 956.0, "epoch": 0.6892621475704859, "grad_norm": 3.557417631149292, "learning_rate": 2.2030807650560403e-06, "logp/chosen": -356.0, "logp/rejected": -302.0, "loss": 0.312347412109375, "reward/accuracy": 0.75, "reward/chosen": -1.6015625, "reward/margin": 2.453125, "reward/rejected": -4.0625, "step": 2298 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 992.0, "epoch": 0.6895620875824835, "grad_norm": 4.303969860076904, "learning_rate": 2.199176647963318e-06, "logp/chosen": -294.0, "logp/rejected": -358.0, "loss": 0.398590087890625, "reward/accuracy": 0.75, "reward/chosen": -1.4765625, "reward/margin": 2.25, "reward/rejected": -3.734375, "step": 2299 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 488.0, "epoch": 0.6898620275944811, "grad_norm": 5.685421943664551, "learning_rate": 2.195275017747227e-06, "logp/chosen": -324.0, "logp/rejected": -312.0, "loss": 0.87310791015625, "reward/accuracy": 0.4375, "reward/chosen": -2.015625, "reward/margin": 0.462890625, "reward/rejected": -2.46875, "step": 2300 }, { "approx. KL/chosen": 344.0, "approx. KL/rejected": 668.0, "epoch": 0.6901619676064787, "grad_norm": 4.326655387878418, "learning_rate": 2.1913758778720564e-06, "logp/chosen": -342.0, "logp/rejected": -324.0, "loss": 0.5584716796875, "reward/accuracy": 0.75, "reward/chosen": -1.9765625, "reward/margin": 0.94921875, "reward/rejected": -2.921875, "step": 2301 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 460.0, "epoch": 0.6904619076184763, "grad_norm": 3.1601955890655518, "learning_rate": 2.1874792317998913e-06, "logp/chosen": -278.0, "logp/rejected": -318.0, "loss": 0.4136810302734375, "reward/accuracy": 0.8125, "reward/chosen": -0.98046875, "reward/margin": 1.2578125, "reward/rejected": -2.234375, "step": 2302 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 652.0, "epoch": 0.6907618476304739, "grad_norm": 3.3774094581604004, "learning_rate": 2.183585082990598e-06, "logp/chosen": -478.0, "logp/rejected": -420.0, "loss": 0.4287109375, "reward/accuracy": 0.8125, "reward/chosen": -1.2265625, "reward/margin": 1.8828125, "reward/rejected": -3.109375, "step": 2303 }, { "approx. KL/chosen": 94.0, "approx. KL/rejected": 424.0, "epoch": 0.6910617876424715, "grad_norm": 3.431450605392456, "learning_rate": 2.179693434901827e-06, "logp/chosen": -286.0, "logp/rejected": -450.0, "loss": 0.43365478515625, "reward/accuracy": 0.8125, "reward/chosen": -1.0703125, "reward/margin": 1.3359375, "reward/rejected": -2.40625, "step": 2304 }, { "approx. KL/chosen": 127.5, "approx. KL/rejected": 1024.0, "epoch": 0.6913617276544691, "grad_norm": 2.132664918899536, "learning_rate": 2.175804290989006e-06, "logp/chosen": -478.0, "logp/rejected": -460.0, "loss": 0.16473388671875, "reward/accuracy": 0.9375, "reward/chosen": -0.953125, "reward/margin": 2.984375, "reward/rejected": -3.9375, "step": 2305 }, { "approx. KL/chosen": 434.0, "approx. KL/rejected": 450.0, "epoch": 0.6916616676664668, "grad_norm": 8.038228988647461, "learning_rate": 2.171917654705345e-06, "logp/chosen": -532.0, "logp/rejected": -572.0, "loss": 1.160888671875, "reward/accuracy": 0.625, "reward/chosen": -2.15625, "reward/margin": 0.30078125, "reward/rejected": -2.453125, "step": 2306 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 780.0, "epoch": 0.6919616076784643, "grad_norm": 3.932034492492676, "learning_rate": 2.1680335295018207e-06, "logp/chosen": -304.0, "logp/rejected": -278.0, "loss": 0.45660400390625, "reward/accuracy": 0.875, "reward/chosen": -1.1796875, "reward/margin": 2.140625, "reward/rejected": -3.3125, "step": 2307 }, { "approx. KL/chosen": 138.0, "approx. KL/rejected": 330.0, "epoch": 0.6922615476904619, "grad_norm": 4.604526042938232, "learning_rate": 2.164151918827188e-06, "logp/chosen": -270.0, "logp/rejected": -314.0, "loss": 0.64996337890625, "reward/accuracy": 0.625, "reward/chosen": -1.2578125, "reward/margin": 0.80078125, "reward/rejected": -2.046875, "step": 2308 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 1184.0, "epoch": 0.6925614877024595, "grad_norm": 2.1155543327331543, "learning_rate": 2.160272826127958e-06, "logp/chosen": -342.0, "logp/rejected": -314.0, "loss": 0.16387939453125, "reward/accuracy": 1.0, "reward/chosen": -1.484375, "reward/margin": 2.671875, "reward/rejected": -4.15625, "step": 2309 }, { "approx. KL/chosen": 118.5, "approx. KL/rejected": 1064.0, "epoch": 0.6928614277144571, "grad_norm": 3.0413923263549805, "learning_rate": 2.1563962548484174e-06, "logp/chosen": -232.0, "logp/rejected": -306.0, "loss": 0.3081817626953125, "reward/accuracy": 0.875, "reward/chosen": -1.140625, "reward/margin": 2.546875, "reward/rejected": -3.6875, "step": 2310 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 652.0, "epoch": 0.6931613677264548, "grad_norm": 2.8997280597686768, "learning_rate": 2.1525222084306064e-06, "logp/chosen": -280.0, "logp/rejected": -234.0, "loss": 0.3035888671875, "reward/accuracy": 0.875, "reward/chosen": -1.4453125, "reward/margin": 1.7734375, "reward/rejected": -3.21875, "step": 2311 }, { "approx. KL/chosen": 229.0, "approx. KL/rejected": 744.0, "epoch": 0.6934613077384523, "grad_norm": 3.1582791805267334, "learning_rate": 2.1486506903143288e-06, "logp/chosen": -274.0, "logp/rejected": -310.0, "loss": 0.373046875, "reward/accuracy": 0.8125, "reward/chosen": -1.4296875, "reward/margin": 1.9140625, "reward/rejected": -3.34375, "step": 2312 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 908.0, "epoch": 0.6937612477504499, "grad_norm": 2.8029847145080566, "learning_rate": 2.1447817039371395e-06, "logp/chosen": -344.0, "logp/rejected": -342.0, "loss": 0.2664794921875, "reward/accuracy": 0.875, "reward/chosen": -1.4453125, "reward/margin": 2.328125, "reward/rejected": -3.78125, "step": 2313 }, { "approx. KL/chosen": 223.0, "approx. KL/rejected": 764.0, "epoch": 0.6940611877624475, "grad_norm": 4.428816795349121, "learning_rate": 2.140915252734347e-06, "logp/chosen": -360.0, "logp/rejected": -386.0, "loss": 0.517578125, "reward/accuracy": 0.625, "reward/chosen": -1.6796875, "reward/margin": 1.7109375, "reward/rejected": -3.390625, "step": 2314 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 664.0, "epoch": 0.6943611277744451, "grad_norm": 3.197171449661255, "learning_rate": 2.1370513401390065e-06, "logp/chosen": -440.0, "logp/rejected": -376.0, "loss": 0.376129150390625, "reward/accuracy": 0.8125, "reward/chosen": -1.234375, "reward/margin": 1.8046875, "reward/rejected": -3.03125, "step": 2315 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 1000.0, "epoch": 0.6946610677864428, "grad_norm": 2.350628137588501, "learning_rate": 2.1331899695819253e-06, "logp/chosen": -326.0, "logp/rejected": -332.0, "loss": 0.22308349609375, "reward/accuracy": 0.9375, "reward/chosen": -1.1875, "reward/margin": 2.625, "reward/rejected": -3.8125, "step": 2316 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 704.0, "epoch": 0.6949610077984403, "grad_norm": 3.688316822052002, "learning_rate": 2.129331144491646e-06, "logp/chosen": -213.0, "logp/rejected": -314.0, "loss": 0.5149688720703125, "reward/accuracy": 0.625, "reward/chosen": -1.515625, "reward/margin": 1.3125, "reward/rejected": -2.828125, "step": 2317 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 398.0, "epoch": 0.6952609478104379, "grad_norm": 3.4436330795288086, "learning_rate": 2.1254748682944593e-06, "logp/chosen": -388.0, "logp/rejected": -362.0, "loss": 0.410400390625, "reward/accuracy": 0.8125, "reward/chosen": -1.2890625, "reward/margin": 1.15625, "reward/rejected": -2.453125, "step": 2318 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 600.0, "epoch": 0.6955608878224355, "grad_norm": 3.9315006732940674, "learning_rate": 2.1216211444143814e-06, "logp/chosen": -286.0, "logp/rejected": -368.0, "loss": 0.3546142578125, "reward/accuracy": 0.75, "reward/chosen": -1.4140625, "reward/margin": 1.6484375, "reward/rejected": -3.0625, "step": 2319 }, { "approx. KL/chosen": 167.0, "approx. KL/rejected": 372.0, "epoch": 0.6958608278344331, "grad_norm": 4.8353657722473145, "learning_rate": 2.117769976273174e-06, "logp/chosen": -284.0, "logp/rejected": -334.0, "loss": 0.553497314453125, "reward/accuracy": 0.8125, "reward/chosen": -1.2890625, "reward/margin": 1.1796875, "reward/rejected": -2.46875, "step": 2320 }, { "approx. KL/chosen": 145.0, "approx. KL/rejected": 740.0, "epoch": 0.6961607678464307, "grad_norm": 4.383206844329834, "learning_rate": 2.1139213672903213e-06, "logp/chosen": -404.0, "logp/rejected": -468.0, "loss": 0.491424560546875, "reward/accuracy": 0.8125, "reward/chosen": -1.140625, "reward/margin": 2.046875, "reward/rejected": -3.203125, "step": 2321 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 804.0, "epoch": 0.6964607078584283, "grad_norm": 2.8667752742767334, "learning_rate": 2.110075320883044e-06, "logp/chosen": -324.0, "logp/rejected": -320.0, "loss": 0.2972412109375, "reward/accuracy": 0.8125, "reward/chosen": -1.625, "reward/margin": 1.9921875, "reward/rejected": -3.609375, "step": 2322 }, { "approx. KL/chosen": 137.0, "approx. KL/rejected": 556.0, "epoch": 0.6967606478704259, "grad_norm": 3.997741222381592, "learning_rate": 2.1062318404662736e-06, "logp/chosen": -528.0, "logp/rejected": -478.0, "loss": 0.4835205078125, "reward/accuracy": 0.6875, "reward/chosen": -1.1796875, "reward/margin": 1.6640625, "reward/rejected": -2.84375, "step": 2323 }, { "approx. KL/chosen": 132.0, "approx. KL/rejected": 1160.0, "epoch": 0.6970605878824235, "grad_norm": 3.0034360885620117, "learning_rate": 2.1023909294526773e-06, "logp/chosen": -326.0, "logp/rejected": -312.0, "loss": 0.29364776611328125, "reward/accuracy": 0.875, "reward/chosen": -1.015625, "reward/margin": 3.09375, "reward/rejected": -4.09375, "step": 2324 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 692.0, "epoch": 0.6973605278944212, "grad_norm": 3.4759714603424072, "learning_rate": 2.0985525912526324e-06, "logp/chosen": -264.0, "logp/rejected": -280.0, "loss": 0.3662109375, "reward/accuracy": 0.8125, "reward/chosen": -1.6796875, "reward/margin": 1.625, "reward/rejected": -3.296875, "step": 2325 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 804.0, "epoch": 0.6976604679064187, "grad_norm": 3.8472208976745605, "learning_rate": 2.0947168292742366e-06, "logp/chosen": -233.0, "logp/rejected": -272.0, "loss": 0.50885009765625, "reward/accuracy": 0.6875, "reward/chosen": -2.046875, "reward/margin": 1.484375, "reward/rejected": -3.53125, "step": 2326 }, { "approx. KL/chosen": 58.5, "approx. KL/rejected": 980.0, "epoch": 0.6979604079184163, "grad_norm": 1.4707850217819214, "learning_rate": 2.0908836469232968e-06, "logp/chosen": -219.0, "logp/rejected": -256.0, "loss": 0.16649627685546875, "reward/accuracy": 0.9375, "reward/chosen": -0.76953125, "reward/margin": 3.078125, "reward/rejected": -3.84375, "step": 2327 }, { "approx. KL/chosen": 94.5, "approx. KL/rejected": 354.0, "epoch": 0.6982603479304139, "grad_norm": 3.2976489067077637, "learning_rate": 2.087053047603331e-06, "logp/chosen": -350.0, "logp/rejected": -316.0, "loss": 0.420654296875, "reward/accuracy": 0.8125, "reward/chosen": -1.078125, "reward/margin": 1.1796875, "reward/rejected": -2.25, "step": 2328 }, { "approx. KL/chosen": 84.0, "approx. KL/rejected": 370.0, "epoch": 0.6985602879424115, "grad_norm": 2.6325674057006836, "learning_rate": 2.083225034715561e-06, "logp/chosen": -328.0, "logp/rejected": -336.0, "loss": 0.34033203125, "reward/accuracy": 0.875, "reward/chosen": -0.7578125, "reward/margin": 1.6953125, "reward/rejected": -2.453125, "step": 2329 }, { "approx. KL/chosen": 129.0, "approx. KL/rejected": 656.0, "epoch": 0.6988602279544092, "grad_norm": 3.0673861503601074, "learning_rate": 2.0793996116589183e-06, "logp/chosen": -276.0, "logp/rejected": -253.0, "loss": 0.36913299560546875, "reward/accuracy": 0.8125, "reward/chosen": -1.1171875, "reward/margin": 1.7890625, "reward/rejected": -2.90625, "step": 2330 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 632.0, "epoch": 0.6991601679664067, "grad_norm": 4.468791961669922, "learning_rate": 2.0755767818300267e-06, "logp/chosen": -402.0, "logp/rejected": -354.0, "loss": 0.45965576171875, "reward/accuracy": 0.8125, "reward/chosen": -1.5859375, "reward/margin": 1.4375, "reward/rejected": -3.015625, "step": 2331 }, { "approx. KL/chosen": 244.0, "approx. KL/rejected": 2864.0, "epoch": 0.6994601079784043, "grad_norm": 5.329221248626709, "learning_rate": 2.0717565486232178e-06, "logp/chosen": -260.0, "logp/rejected": -556.0, "loss": 0.44671630859375, "reward/accuracy": 0.6875, "reward/chosen": -1.71875, "reward/margin": 3.25, "reward/rejected": -4.96875, "step": 2332 }, { "approx. KL/chosen": 137.0, "approx. KL/rejected": 844.0, "epoch": 0.6997600479904019, "grad_norm": 2.766786575317383, "learning_rate": 2.0679389154305045e-06, "logp/chosen": -348.0, "logp/rejected": -388.0, "loss": 0.367584228515625, "reward/accuracy": 0.8125, "reward/chosen": -1.078125, "reward/margin": 2.234375, "reward/rejected": -3.3125, "step": 2333 }, { "approx. KL/chosen": 79.5, "approx. KL/rejected": 608.0, "epoch": 0.7000599880023995, "grad_norm": 2.4655632972717285, "learning_rate": 2.064123885641603e-06, "logp/chosen": -342.0, "logp/rejected": -364.0, "loss": 0.284698486328125, "reward/accuracy": 0.9375, "reward/chosen": -0.75390625, "reward/margin": 2.140625, "reward/rejected": -2.890625, "step": 2334 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 748.0, "epoch": 0.7003599280143972, "grad_norm": 2.9129607677459717, "learning_rate": 2.0603114626439096e-06, "logp/chosen": -390.0, "logp/rejected": -400.0, "loss": 0.29766845703125, "reward/accuracy": 0.8125, "reward/chosen": -1.578125, "reward/margin": 1.6796875, "reward/rejected": -3.25, "step": 2335 }, { "approx. KL/chosen": 172.0, "approx. KL/rejected": 632.0, "epoch": 0.7006598680263947, "grad_norm": 3.4068784713745117, "learning_rate": 2.056501649822514e-06, "logp/chosen": -318.0, "logp/rejected": -360.0, "loss": 0.305389404296875, "reward/accuracy": 0.875, "reward/chosen": -1.5, "reward/margin": 1.7421875, "reward/rejected": -3.25, "step": 2336 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 552.0, "epoch": 0.7009598080383923, "grad_norm": 3.1736412048339844, "learning_rate": 2.0526944505601804e-06, "logp/chosen": -446.0, "logp/rejected": -368.0, "loss": 0.4344482421875, "reward/accuracy": 0.875, "reward/chosen": -1.171875, "reward/margin": 1.453125, "reward/rejected": -2.625, "step": 2337 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 676.0, "epoch": 0.7012597480503899, "grad_norm": 4.479524612426758, "learning_rate": 2.0488898682373565e-06, "logp/chosen": -366.0, "logp/rejected": -392.0, "loss": 0.4779052734375, "reward/accuracy": 0.6875, "reward/chosen": -1.6953125, "reward/margin": 1.53125, "reward/rejected": -3.234375, "step": 2338 }, { "approx. KL/chosen": 342.0, "approx. KL/rejected": 932.0, "epoch": 0.7015596880623876, "grad_norm": 4.924284934997559, "learning_rate": 2.045087906232164e-06, "logp/chosen": -370.0, "logp/rejected": -346.0, "loss": 0.59967041015625, "reward/accuracy": 0.75, "reward/chosen": -2.1875, "reward/margin": 1.6953125, "reward/rejected": -3.875, "step": 2339 }, { "approx. KL/chosen": 218.0, "approx. KL/rejected": 446.0, "epoch": 0.7018596280743852, "grad_norm": 4.109692096710205, "learning_rate": 2.0412885679204038e-06, "logp/chosen": -312.0, "logp/rejected": -338.0, "loss": 0.57281494140625, "reward/accuracy": 0.625, "reward/chosen": -1.328125, "reward/margin": 1.3125, "reward/rejected": -2.640625, "step": 2340 }, { "approx. KL/chosen": 108.5, "approx. KL/rejected": 1240.0, "epoch": 0.7021595680863827, "grad_norm": 2.4100492000579834, "learning_rate": 2.0374918566755385e-06, "logp/chosen": -290.0, "logp/rejected": -388.0, "loss": 0.21216583251953125, "reward/accuracy": 0.875, "reward/chosen": -1.1015625, "reward/margin": 3.15625, "reward/rejected": -4.25, "step": 2341 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 928.0, "epoch": 0.7024595080983803, "grad_norm": 5.016402721405029, "learning_rate": 2.0336977758687087e-06, "logp/chosen": -274.0, "logp/rejected": -300.0, "loss": 0.6031494140625, "reward/accuracy": 0.6875, "reward/chosen": -2.015625, "reward/margin": 1.609375, "reward/rejected": -3.625, "step": 2342 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 490.0, "epoch": 0.7027594481103779, "grad_norm": 3.63985276222229, "learning_rate": 2.029906328868706e-06, "logp/chosen": -330.0, "logp/rejected": -292.0, "loss": 0.5501708984375, "reward/accuracy": 0.6875, "reward/chosen": -1.2578125, "reward/margin": 1.2109375, "reward/rejected": -2.46875, "step": 2343 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 434.0, "epoch": 0.7030593881223756, "grad_norm": 4.897317886352539, "learning_rate": 2.0261175190419964e-06, "logp/chosen": -362.0, "logp/rejected": -290.0, "loss": 0.46502685546875, "reward/accuracy": 0.75, "reward/chosen": -1.0703125, "reward/margin": 1.3984375, "reward/rejected": -2.46875, "step": 2344 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 752.0, "epoch": 0.7033593281343731, "grad_norm": 3.0711255073547363, "learning_rate": 2.022331349752695e-06, "logp/chosen": -338.0, "logp/rejected": -239.0, "loss": 0.341766357421875, "reward/accuracy": 0.8125, "reward/chosen": -1.71875, "reward/margin": 1.734375, "reward/rejected": -3.453125, "step": 2345 }, { "approx. KL/chosen": 446.0, "approx. KL/rejected": 776.0, "epoch": 0.7036592681463707, "grad_norm": 4.857806205749512, "learning_rate": 2.018547824362581e-06, "logp/chosen": -292.0, "logp/rejected": -360.0, "loss": 0.4757080078125, "reward/accuracy": 0.8125, "reward/chosen": -2.078125, "reward/margin": 1.3671875, "reward/rejected": -3.4375, "step": 2346 }, { "approx. KL/chosen": 225.0, "approx. KL/rejected": 462.0, "epoch": 0.7039592081583683, "grad_norm": 4.326657772064209, "learning_rate": 2.0147669462310747e-06, "logp/chosen": -372.0, "logp/rejected": -378.0, "loss": 0.572509765625, "reward/accuracy": 0.75, "reward/chosen": -1.5859375, "reward/margin": 0.9609375, "reward/rejected": -2.546875, "step": 2347 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 936.0, "epoch": 0.704259148170366, "grad_norm": 1.919390320777893, "learning_rate": 2.0109887187152577e-06, "logp/chosen": -330.0, "logp/rejected": -330.0, "loss": 0.219940185546875, "reward/accuracy": 0.875, "reward/chosen": -1.46875, "reward/margin": 2.40625, "reward/rejected": -3.875, "step": 2348 }, { "approx. KL/chosen": 418.0, "approx. KL/rejected": 916.0, "epoch": 0.7045590881823636, "grad_norm": 3.38525390625, "learning_rate": 2.0072131451698485e-06, "logp/chosen": -290.0, "logp/rejected": -296.0, "loss": 0.365386962890625, "reward/accuracy": 0.8125, "reward/chosen": -2.375, "reward/margin": 1.375, "reward/rejected": -3.75, "step": 2349 }, { "approx. KL/chosen": 108.0, "approx. KL/rejected": 988.0, "epoch": 0.7048590281943611, "grad_norm": 2.106619119644165, "learning_rate": 2.003440228947217e-06, "logp/chosen": -300.0, "logp/rejected": -326.0, "loss": 0.2314453125, "reward/accuracy": 0.9375, "reward/chosen": -1.1171875, "reward/margin": 2.609375, "reward/rejected": -3.734375, "step": 2350 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 812.0, "epoch": 0.7051589682063587, "grad_norm": 2.7239155769348145, "learning_rate": 1.9996699733973685e-06, "logp/chosen": -386.0, "logp/rejected": -412.0, "loss": 0.2723846435546875, "reward/accuracy": 0.875, "reward/chosen": -1.3046875, "reward/margin": 2.296875, "reward/rejected": -3.609375, "step": 2351 }, { "approx. KL/chosen": 304.0, "approx. KL/rejected": 700.0, "epoch": 0.7054589082183563, "grad_norm": 5.134279251098633, "learning_rate": 1.9959023818679473e-06, "logp/chosen": -336.0, "logp/rejected": -386.0, "loss": 0.592498779296875, "reward/accuracy": 0.625, "reward/chosen": -2.03125, "reward/margin": 1.171875, "reward/rejected": -3.203125, "step": 2352 }, { "approx. KL/chosen": 130.0, "approx. KL/rejected": 684.0, "epoch": 0.705758848230354, "grad_norm": 2.837064743041992, "learning_rate": 1.9921374577042306e-06, "logp/chosen": -262.0, "logp/rejected": -252.0, "loss": 0.30712890625, "reward/accuracy": 0.875, "reward/chosen": -1.3515625, "reward/margin": 1.828125, "reward/rejected": -3.1875, "step": 2353 }, { "approx. KL/chosen": 248.0, "approx. KL/rejected": 892.0, "epoch": 0.7060587882423516, "grad_norm": 2.100398540496826, "learning_rate": 1.988375204249133e-06, "logp/chosen": -198.0, "logp/rejected": -168.0, "loss": 0.211517333984375, "reward/accuracy": 0.875, "reward/chosen": -1.359375, "reward/margin": 2.3125, "reward/rejected": -3.671875, "step": 2354 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 588.0, "epoch": 0.7063587282543491, "grad_norm": 3.6635541915893555, "learning_rate": 1.9846156248431886e-06, "logp/chosen": -344.0, "logp/rejected": -274.0, "loss": 0.37835693359375, "reward/accuracy": 0.8125, "reward/chosen": -1.5546875, "reward/margin": 1.46875, "reward/rejected": -3.015625, "step": 2355 }, { "approx. KL/chosen": 362.0, "approx. KL/rejected": 676.0, "epoch": 0.7066586682663467, "grad_norm": 5.851841926574707, "learning_rate": 1.980858722824569e-06, "logp/chosen": -330.0, "logp/rejected": -556.0, "loss": 0.816619873046875, "reward/accuracy": 0.625, "reward/chosen": -2.125, "reward/margin": 0.91796875, "reward/rejected": -3.046875, "step": 2356 }, { "approx. KL/chosen": 108.5, "approx. KL/rejected": 716.0, "epoch": 0.7069586082783443, "grad_norm": 2.474489450454712, "learning_rate": 1.977104501529054e-06, "logp/chosen": -484.0, "logp/rejected": -432.0, "loss": 0.2773447036743164, "reward/accuracy": 0.875, "reward/chosen": -0.99609375, "reward/margin": 2.28125, "reward/rejected": -3.28125, "step": 2357 }, { "approx. KL/chosen": 246.0, "approx. KL/rejected": 996.0, "epoch": 0.707258548290342, "grad_norm": 3.4192564487457275, "learning_rate": 1.9733529642900568e-06, "logp/chosen": -372.0, "logp/rejected": -424.0, "loss": 0.2452392578125, "reward/accuracy": 0.875, "reward/chosen": -1.90625, "reward/margin": 2.0625, "reward/rejected": -3.96875, "step": 2358 }, { "approx. KL/chosen": 168.0, "approx. KL/rejected": 740.0, "epoch": 0.7075584883023396, "grad_norm": 3.870555877685547, "learning_rate": 1.9696041144385975e-06, "logp/chosen": -312.0, "logp/rejected": -432.0, "loss": 0.4715576171875, "reward/accuracy": 0.8125, "reward/chosen": -1.046875, "reward/margin": 1.8828125, "reward/rejected": -2.921875, "step": 2359 }, { "approx. KL/chosen": 185.0, "approx. KL/rejected": 644.0, "epoch": 0.7078584283143371, "grad_norm": 4.4236674308776855, "learning_rate": 1.965857955303318e-06, "logp/chosen": -398.0, "logp/rejected": -404.0, "loss": 0.46630859375, "reward/accuracy": 0.75, "reward/chosen": -1.4453125, "reward/margin": 1.7421875, "reward/rejected": -3.1875, "step": 2360 }, { "approx. KL/chosen": 294.0, "approx. KL/rejected": 1032.0, "epoch": 0.7081583683263347, "grad_norm": 8.157963752746582, "learning_rate": 1.9621144902104653e-06, "logp/chosen": -436.0, "logp/rejected": -584.0, "loss": 0.277862548828125, "reward/accuracy": 0.875, "reward/chosen": -1.1015625, "reward/margin": 2.6875, "reward/rejected": -3.796875, "step": 2361 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 496.0, "epoch": 0.7084583083383323, "grad_norm": 4.567842483520508, "learning_rate": 1.9583737224838967e-06, "logp/chosen": -258.0, "logp/rejected": -262.0, "loss": 0.591064453125, "reward/accuracy": 0.625, "reward/chosen": -1.421875, "reward/margin": 1.203125, "reward/rejected": -2.625, "step": 2362 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 736.0, "epoch": 0.70875824835033, "grad_norm": 4.488089084625244, "learning_rate": 1.954635655445071e-06, "logp/chosen": -416.0, "logp/rejected": -348.0, "loss": 0.51055908203125, "reward/accuracy": 0.625, "reward/chosen": -1.4921875, "reward/margin": 1.5390625, "reward/rejected": -3.03125, "step": 2363 }, { "approx. KL/chosen": 233.0, "approx. KL/rejected": 708.0, "epoch": 0.7090581883623275, "grad_norm": 4.79164457321167, "learning_rate": 1.9509002924130578e-06, "logp/chosen": -386.0, "logp/rejected": -296.0, "loss": 0.47479248046875, "reward/accuracy": 0.875, "reward/chosen": -1.5078125, "reward/margin": 1.7265625, "reward/rejected": -3.234375, "step": 2364 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 568.0, "epoch": 0.7093581283743251, "grad_norm": 4.534909725189209, "learning_rate": 1.9471676367045155e-06, "logp/chosen": -424.0, "logp/rejected": -394.0, "loss": 0.69989013671875, "reward/accuracy": 0.6875, "reward/chosen": -1.640625, "reward/margin": 1.1171875, "reward/rejected": -2.765625, "step": 2365 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 1020.0, "epoch": 0.7096580683863227, "grad_norm": 3.2487614154815674, "learning_rate": 1.9434376916337055e-06, "logp/chosen": -364.0, "logp/rejected": -344.0, "loss": 0.27056884765625, "reward/accuracy": 0.875, "reward/chosen": -0.98828125, "reward/margin": 2.6875, "reward/rejected": -3.6875, "step": 2366 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 1128.0, "epoch": 0.7099580083983204, "grad_norm": 3.0909910202026367, "learning_rate": 1.9397104605124775e-06, "logp/chosen": -320.0, "logp/rejected": -298.0, "loss": 0.3513145446777344, "reward/accuracy": 0.875, "reward/chosen": -1.6953125, "reward/margin": 2.59375, "reward/rejected": -4.28125, "step": 2367 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 664.0, "epoch": 0.710257948410318, "grad_norm": 3.3500983715057373, "learning_rate": 1.9359859466502767e-06, "logp/chosen": -294.0, "logp/rejected": -300.0, "loss": 0.38055419921875, "reward/accuracy": 0.8125, "reward/chosen": -1.046875, "reward/margin": 2.0, "reward/rejected": -3.046875, "step": 2368 }, { "approx. KL/chosen": 340.0, "approx. KL/rejected": 712.0, "epoch": 0.7105578884223155, "grad_norm": 6.228593349456787, "learning_rate": 1.932264153354131e-06, "logp/chosen": -336.0, "logp/rejected": -322.0, "loss": 0.8203125, "reward/accuracy": 0.75, "reward/chosen": -1.8203125, "reward/margin": 1.1015625, "reward/rejected": -2.921875, "step": 2369 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 620.0, "epoch": 0.7108578284343131, "grad_norm": 6.867370128631592, "learning_rate": 1.928545083928659e-06, "logp/chosen": -408.0, "logp/rejected": -322.0, "loss": 0.9161376953125, "reward/accuracy": 0.625, "reward/chosen": -2.109375, "reward/margin": 0.67578125, "reward/rejected": -2.78125, "step": 2370 }, { "approx. KL/chosen": 226.0, "approx. KL/rejected": 740.0, "epoch": 0.7111577684463107, "grad_norm": 3.7350072860717773, "learning_rate": 1.9248287416760493e-06, "logp/chosen": -320.0, "logp/rejected": -358.0, "loss": 0.423828125, "reward/accuracy": 0.6875, "reward/chosen": -1.6015625, "reward/margin": 1.65625, "reward/rejected": -3.265625, "step": 2371 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 680.0, "epoch": 0.7114577084583084, "grad_norm": 4.460913181304932, "learning_rate": 1.921115129896082e-06, "logp/chosen": -253.0, "logp/rejected": -222.0, "loss": 0.51654052734375, "reward/accuracy": 0.75, "reward/chosen": -1.3984375, "reward/margin": 1.5390625, "reward/rejected": -2.9375, "step": 2372 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 644.0, "epoch": 0.711757648470306, "grad_norm": 5.956287384033203, "learning_rate": 1.9174042518861023e-06, "logp/chosen": -320.0, "logp/rejected": -324.0, "loss": 0.57867431640625, "reward/accuracy": 0.5625, "reward/chosen": -1.7578125, "reward/margin": 0.953125, "reward/rejected": -2.703125, "step": 2373 }, { "approx. KL/chosen": 171.0, "approx. KL/rejected": 428.0, "epoch": 0.7120575884823035, "grad_norm": 4.449361801147461, "learning_rate": 1.913696110941038e-06, "logp/chosen": -241.0, "logp/rejected": -272.0, "loss": 0.668304443359375, "reward/accuracy": 0.625, "reward/chosen": -1.4609375, "reward/margin": 0.93359375, "reward/rejected": -2.40625, "step": 2374 }, { "approx. KL/chosen": 254.0, "approx. KL/rejected": 968.0, "epoch": 0.7123575284943011, "grad_norm": 4.148759841918945, "learning_rate": 1.909990710353378e-06, "logp/chosen": -390.0, "logp/rejected": -492.0, "loss": 0.48779296875, "reward/accuracy": 0.75, "reward/chosen": -1.875, "reward/margin": 1.9921875, "reward/rejected": -3.875, "step": 2375 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 680.0, "epoch": 0.7126574685062987, "grad_norm": 3.7395827770233154, "learning_rate": 1.9062880534131812e-06, "logp/chosen": -342.0, "logp/rejected": -306.0, "loss": 0.40032958984375, "reward/accuracy": 0.8125, "reward/chosen": -1.625, "reward/margin": 1.765625, "reward/rejected": -3.390625, "step": 2376 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 852.0, "epoch": 0.7129574085182964, "grad_norm": 4.2745771408081055, "learning_rate": 1.90258814340807e-06, "logp/chosen": -306.0, "logp/rejected": -360.0, "loss": 0.46673583984375, "reward/accuracy": 0.8125, "reward/chosen": -1.734375, "reward/margin": 1.703125, "reward/rejected": -3.4375, "step": 2377 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 692.0, "epoch": 0.713257348530294, "grad_norm": 4.385919570922852, "learning_rate": 1.8988909836232305e-06, "logp/chosen": -398.0, "logp/rejected": -398.0, "loss": 0.4146728515625, "reward/accuracy": 0.8125, "reward/chosen": -1.578125, "reward/margin": 1.6875, "reward/rejected": -3.265625, "step": 2378 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 1020.0, "epoch": 0.7135572885422915, "grad_norm": 3.0964581966400146, "learning_rate": 1.8951965773414043e-06, "logp/chosen": -340.0, "logp/rejected": -376.0, "loss": 0.3177490234375, "reward/accuracy": 0.8125, "reward/chosen": -1.515625, "reward/margin": 2.234375, "reward/rejected": -3.75, "step": 2379 }, { "approx. KL/chosen": 87.0, "approx. KL/rejected": 728.0, "epoch": 0.7138572285542891, "grad_norm": 3.559011459350586, "learning_rate": 1.8915049278428871e-06, "logp/chosen": -264.0, "logp/rejected": -308.0, "loss": 0.299957275390625, "reward/accuracy": 0.875, "reward/chosen": -0.6328125, "reward/margin": 2.71875, "reward/rejected": -3.34375, "step": 2380 }, { "approx. KL/chosen": 193.0, "approx. KL/rejected": 494.0, "epoch": 0.7141571685662867, "grad_norm": 4.277538776397705, "learning_rate": 1.887816038405527e-06, "logp/chosen": -310.0, "logp/rejected": -322.0, "loss": 0.55157470703125, "reward/accuracy": 0.8125, "reward/chosen": -1.640625, "reward/margin": 1.078125, "reward/rejected": -2.71875, "step": 2381 }, { "approx. KL/chosen": 168.0, "approx. KL/rejected": 820.0, "epoch": 0.7144571085782844, "grad_norm": 2.7598466873168945, "learning_rate": 1.8841299123047268e-06, "logp/chosen": -262.0, "logp/rejected": -258.0, "loss": 0.27032470703125, "reward/accuracy": 0.9375, "reward/chosen": -1.4453125, "reward/margin": 2.0625, "reward/rejected": -3.515625, "step": 2382 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 720.0, "epoch": 0.714757048590282, "grad_norm": 3.420590400695801, "learning_rate": 1.8804465528134268e-06, "logp/chosen": -272.0, "logp/rejected": -374.0, "loss": 0.3176727294921875, "reward/accuracy": 0.9375, "reward/chosen": -1.671875, "reward/margin": 1.6875, "reward/rejected": -3.359375, "step": 2383 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 908.0, "epoch": 0.7150569886022795, "grad_norm": 3.0761096477508545, "learning_rate": 1.8767659632021195e-06, "logp/chosen": -348.0, "logp/rejected": -422.0, "loss": 0.25537109375, "reward/accuracy": 0.8125, "reward/chosen": -1.5546875, "reward/margin": 2.34375, "reward/rejected": -3.890625, "step": 2384 }, { "approx. KL/chosen": 400.0, "approx. KL/rejected": 784.0, "epoch": 0.7153569286142771, "grad_norm": 6.188187599182129, "learning_rate": 1.8730881467388328e-06, "logp/chosen": -454.0, "logp/rejected": -426.0, "loss": 0.65972900390625, "reward/accuracy": 0.8125, "reward/chosen": -1.8828125, "reward/margin": 1.671875, "reward/rejected": -3.5625, "step": 2385 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 828.0, "epoch": 0.7156568686262748, "grad_norm": 1.8765206336975098, "learning_rate": 1.869413106689133e-06, "logp/chosen": -402.0, "logp/rejected": -290.0, "loss": 0.20697021484375, "reward/accuracy": 0.9375, "reward/chosen": -1.3046875, "reward/margin": 2.375, "reward/rejected": -3.6875, "step": 2386 }, { "approx. KL/chosen": 42.75, "approx. KL/rejected": 968.0, "epoch": 0.7159568086382724, "grad_norm": 2.7821013927459717, "learning_rate": 1.8657408463161198e-06, "logp/chosen": -186.0, "logp/rejected": -298.0, "loss": 0.23682785034179688, "reward/accuracy": 0.875, "reward/chosen": -0.63671875, "reward/margin": 3.0625, "reward/rejected": -3.703125, "step": 2387 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 780.0, "epoch": 0.7162567486502699, "grad_norm": 2.2779359817504883, "learning_rate": 1.8620713688804304e-06, "logp/chosen": -214.0, "logp/rejected": -292.0, "loss": 0.200927734375, "reward/accuracy": 0.9375, "reward/chosen": -1.46875, "reward/margin": 2.21875, "reward/rejected": -3.6875, "step": 2388 }, { "approx. KL/chosen": 237.0, "approx. KL/rejected": 656.0, "epoch": 0.7165566886622675, "grad_norm": 3.896143913269043, "learning_rate": 1.8584046776402258e-06, "logp/chosen": -388.0, "logp/rejected": -398.0, "loss": 0.46514892578125, "reward/accuracy": 0.6875, "reward/chosen": -1.6328125, "reward/margin": 1.34375, "reward/rejected": -2.96875, "step": 2389 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 648.0, "epoch": 0.7168566286742651, "grad_norm": 4.987519264221191, "learning_rate": 1.8547407758511948e-06, "logp/chosen": -368.0, "logp/rejected": -346.0, "loss": 0.4051513671875, "reward/accuracy": 0.875, "reward/chosen": -1.7578125, "reward/margin": 1.3671875, "reward/rejected": -3.125, "step": 2390 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 656.0, "epoch": 0.7171565686862628, "grad_norm": 5.2059478759765625, "learning_rate": 1.8510796667665465e-06, "logp/chosen": -382.0, "logp/rejected": -392.0, "loss": 0.599609375, "reward/accuracy": 0.6875, "reward/chosen": -1.78125, "reward/margin": 1.15625, "reward/rejected": -2.9375, "step": 2391 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 464.0, "epoch": 0.7174565086982604, "grad_norm": 3.790555477142334, "learning_rate": 1.8474213536370178e-06, "logp/chosen": -246.0, "logp/rejected": -207.0, "loss": 0.4216346740722656, "reward/accuracy": 0.75, "reward/chosen": -1.2109375, "reward/margin": 1.40625, "reward/rejected": -2.609375, "step": 2392 }, { "approx. KL/chosen": 223.0, "approx. KL/rejected": 632.0, "epoch": 0.7177564487102579, "grad_norm": 4.226360321044922, "learning_rate": 1.843765839710856e-06, "logp/chosen": -262.0, "logp/rejected": -330.0, "loss": 0.409423828125, "reward/accuracy": 0.75, "reward/chosen": -1.703125, "reward/margin": 1.3203125, "reward/rejected": -3.015625, "step": 2393 }, { "approx. KL/chosen": 243.0, "approx. KL/rejected": 482.0, "epoch": 0.7180563887222555, "grad_norm": 3.4853579998016357, "learning_rate": 1.8401131282338253e-06, "logp/chosen": -416.0, "logp/rejected": -370.0, "loss": 0.45172119140625, "reward/accuracy": 0.8125, "reward/chosen": -1.28125, "reward/margin": 1.375, "reward/rejected": -2.65625, "step": 2394 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 438.0, "epoch": 0.7183563287342531, "grad_norm": 5.0251665115356445, "learning_rate": 1.8364632224491997e-06, "logp/chosen": -302.0, "logp/rejected": -292.0, "loss": 0.680694580078125, "reward/accuracy": 0.625, "reward/chosen": -1.9375, "reward/margin": 0.69140625, "reward/rejected": -2.625, "step": 2395 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 688.0, "epoch": 0.7186562687462508, "grad_norm": 4.678326606750488, "learning_rate": 1.8328161255977679e-06, "logp/chosen": -252.0, "logp/rejected": -338.0, "loss": 0.4470367431640625, "reward/accuracy": 0.625, "reward/chosen": -1.421875, "reward/margin": 1.6171875, "reward/rejected": -3.046875, "step": 2396 }, { "approx. KL/chosen": 234.0, "approx. KL/rejected": 536.0, "epoch": 0.7189562087582484, "grad_norm": 4.90498161315918, "learning_rate": 1.8291718409178177e-06, "logp/chosen": -456.0, "logp/rejected": -372.0, "loss": 0.500732421875, "reward/accuracy": 0.6875, "reward/chosen": -1.65625, "reward/margin": 1.265625, "reward/rejected": -2.921875, "step": 2397 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 712.0, "epoch": 0.7192561487702459, "grad_norm": 5.648049831390381, "learning_rate": 1.8255303716451423e-06, "logp/chosen": -240.0, "logp/rejected": -300.0, "loss": 0.9564476013183594, "reward/accuracy": 0.625, "reward/chosen": -1.8984375, "reward/margin": 1.1171875, "reward/rejected": -3.015625, "step": 2398 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 680.0, "epoch": 0.7195560887822435, "grad_norm": 3.7965447902679443, "learning_rate": 1.8218917210130382e-06, "logp/chosen": -308.0, "logp/rejected": -266.0, "loss": 0.360076904296875, "reward/accuracy": 0.8125, "reward/chosen": -0.9921875, "reward/margin": 2.25, "reward/rejected": -3.234375, "step": 2399 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 996.0, "epoch": 0.7198560287942412, "grad_norm": 4.200425624847412, "learning_rate": 1.8182558922522948e-06, "logp/chosen": -394.0, "logp/rejected": -376.0, "loss": 0.57952880859375, "reward/accuracy": 0.6875, "reward/chosen": -1.703125, "reward/margin": 1.7890625, "reward/rejected": -3.5, "step": 2400 }, { "approx. KL/chosen": 294.0, "approx. KL/rejected": 700.0, "epoch": 0.7201559688062388, "grad_norm": 5.664307594299316, "learning_rate": 1.814622888591196e-06, "logp/chosen": -346.0, "logp/rejected": -362.0, "loss": 0.465118408203125, "reward/accuracy": 0.75, "reward/chosen": -1.734375, "reward/margin": 1.6015625, "reward/rejected": -3.328125, "step": 2401 }, { "approx. KL/chosen": 121.0, "approx. KL/rejected": 888.0, "epoch": 0.7204559088182364, "grad_norm": 2.4893906116485596, "learning_rate": 1.8109927132555228e-06, "logp/chosen": -318.0, "logp/rejected": -352.0, "loss": 0.1808013916015625, "reward/accuracy": 0.9375, "reward/chosen": -1.2578125, "reward/margin": 2.53125, "reward/rejected": -3.78125, "step": 2402 }, { "approx. KL/chosen": 147.0, "approx. KL/rejected": 756.0, "epoch": 0.7207558488302339, "grad_norm": 3.1783289909362793, "learning_rate": 1.807365369468539e-06, "logp/chosen": -290.0, "logp/rejected": -312.0, "loss": 0.332916259765625, "reward/accuracy": 0.75, "reward/chosen": -1.3125, "reward/margin": 1.9609375, "reward/rejected": -3.265625, "step": 2403 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 668.0, "epoch": 0.7210557888422315, "grad_norm": 2.596327066421509, "learning_rate": 1.8037408604509959e-06, "logp/chosen": -288.0, "logp/rejected": -348.0, "loss": 0.2960205078125, "reward/accuracy": 0.875, "reward/chosen": -1.3203125, "reward/margin": 2.0, "reward/rejected": -3.328125, "step": 2404 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 1012.0, "epoch": 0.7213557288542292, "grad_norm": 4.117245674133301, "learning_rate": 1.8001191894211274e-06, "logp/chosen": -388.0, "logp/rejected": -266.0, "loss": 0.37762451171875, "reward/accuracy": 0.75, "reward/chosen": -1.8046875, "reward/margin": 1.921875, "reward/rejected": -3.734375, "step": 2405 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 564.0, "epoch": 0.7216556688662268, "grad_norm": 2.5299439430236816, "learning_rate": 1.7965003595946506e-06, "logp/chosen": -510.0, "logp/rejected": -418.0, "loss": 0.23236083984375, "reward/accuracy": 1.0, "reward/chosen": -0.7890625, "reward/margin": 1.890625, "reward/rejected": -2.6875, "step": 2406 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 948.0, "epoch": 0.7219556088782244, "grad_norm": 5.211880683898926, "learning_rate": 1.7928843741847562e-06, "logp/chosen": -348.0, "logp/rejected": -320.0, "loss": 0.3775444030761719, "reward/accuracy": 0.6875, "reward/chosen": -1.4140625, "reward/margin": 2.359375, "reward/rejected": -3.765625, "step": 2407 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 696.0, "epoch": 0.7222555488902219, "grad_norm": 3.647502899169922, "learning_rate": 1.7892712364021099e-06, "logp/chosen": -322.0, "logp/rejected": -454.0, "loss": 0.34429931640625, "reward/accuracy": 0.875, "reward/chosen": -1.4375, "reward/margin": 1.75, "reward/rejected": -3.1875, "step": 2408 }, { "approx. KL/chosen": 452.0, "approx. KL/rejected": 536.0, "epoch": 0.7225554889022195, "grad_norm": 5.665175437927246, "learning_rate": 1.7856609494548516e-06, "logp/chosen": -292.0, "logp/rejected": -304.0, "loss": 0.9312744140625, "reward/accuracy": 0.625, "reward/chosen": -2.265625, "reward/margin": 0.55078125, "reward/rejected": -2.8125, "step": 2409 }, { "approx. KL/chosen": 101.5, "approx. KL/rejected": 584.0, "epoch": 0.7228554289142172, "grad_norm": 2.7474632263183594, "learning_rate": 1.782053516548587e-06, "logp/chosen": -238.0, "logp/rejected": -292.0, "loss": 0.34814453125, "reward/accuracy": 0.875, "reward/chosen": -1.1953125, "reward/margin": 1.6015625, "reward/rejected": -2.796875, "step": 2410 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 800.0, "epoch": 0.7231553689262148, "grad_norm": 4.458169460296631, "learning_rate": 1.7784489408863891e-06, "logp/chosen": -218.0, "logp/rejected": -286.0, "loss": 0.485931396484375, "reward/accuracy": 0.5625, "reward/chosen": -1.9140625, "reward/margin": 1.609375, "reward/rejected": -3.53125, "step": 2411 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 920.0, "epoch": 0.7234553089382123, "grad_norm": 4.939310073852539, "learning_rate": 1.7748472256687916e-06, "logp/chosen": -255.0, "logp/rejected": -334.0, "loss": 0.5797882080078125, "reward/accuracy": 0.625, "reward/chosen": -1.4140625, "reward/margin": 1.8984375, "reward/rejected": -3.3125, "step": 2412 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 368.0, "epoch": 0.7237552489502099, "grad_norm": 4.232978820800781, "learning_rate": 1.771248374093793e-06, "logp/chosen": -402.0, "logp/rejected": -382.0, "loss": 0.5128173828125, "reward/accuracy": 0.75, "reward/chosen": -1.3515625, "reward/margin": 1.0, "reward/rejected": -2.359375, "step": 2413 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 652.0, "epoch": 0.7240551889622076, "grad_norm": 2.868563652038574, "learning_rate": 1.7676523893568459e-06, "logp/chosen": -350.0, "logp/rejected": -394.0, "loss": 0.300689697265625, "reward/accuracy": 0.9375, "reward/chosen": -1.3828125, "reward/margin": 1.953125, "reward/rejected": -3.34375, "step": 2414 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 836.0, "epoch": 0.7243551289742052, "grad_norm": 4.311501502990723, "learning_rate": 1.7640592746508578e-06, "logp/chosen": -486.0, "logp/rejected": -350.0, "loss": 0.6041259765625, "reward/accuracy": 0.75, "reward/chosen": -1.578125, "reward/margin": 1.8828125, "reward/rejected": -3.453125, "step": 2415 }, { "approx. KL/chosen": 330.0, "approx. KL/rejected": 984.0, "epoch": 0.7246550689862028, "grad_norm": 4.079535484313965, "learning_rate": 1.7604690331661855e-06, "logp/chosen": -330.0, "logp/rejected": -348.0, "loss": 0.4852447509765625, "reward/accuracy": 0.6875, "reward/chosen": -2.15625, "reward/margin": 1.5625, "reward/rejected": -3.71875, "step": 2416 }, { "approx. KL/chosen": 226.0, "approx. KL/rejected": 1088.0, "epoch": 0.7249550089982003, "grad_norm": 3.291985034942627, "learning_rate": 1.7568816680906415e-06, "logp/chosen": -272.0, "logp/rejected": -322.0, "loss": 0.28253173828125, "reward/accuracy": 0.75, "reward/chosen": -1.578125, "reward/margin": 2.4375, "reward/rejected": -4.03125, "step": 2417 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 1112.0, "epoch": 0.7252549490101979, "grad_norm": 2.7611422538757324, "learning_rate": 1.753297182609478e-06, "logp/chosen": -398.0, "logp/rejected": -348.0, "loss": 0.21850204467773438, "reward/accuracy": 0.9375, "reward/chosen": -1.703125, "reward/margin": 2.578125, "reward/rejected": -4.28125, "step": 2418 }, { "approx. KL/chosen": 177.0, "approx. KL/rejected": 792.0, "epoch": 0.7255548890221956, "grad_norm": 3.8239872455596924, "learning_rate": 1.7497155799053906e-06, "logp/chosen": -348.0, "logp/rejected": -386.0, "loss": 0.5228271484375, "reward/accuracy": 0.8125, "reward/chosen": -1.421875, "reward/margin": 1.7421875, "reward/rejected": -3.171875, "step": 2419 }, { "approx. KL/chosen": 294.0, "approx. KL/rejected": 1020.0, "epoch": 0.7258548290341932, "grad_norm": 2.468245506286621, "learning_rate": 1.7461368631585206e-06, "logp/chosen": -364.0, "logp/rejected": -318.0, "loss": 0.229278564453125, "reward/accuracy": 0.9375, "reward/chosen": -1.59375, "reward/margin": 2.359375, "reward/rejected": -3.96875, "step": 2420 }, { "approx. KL/chosen": 141.0, "approx. KL/rejected": 480.0, "epoch": 0.7261547690461908, "grad_norm": 3.65785813331604, "learning_rate": 1.7425610355464418e-06, "logp/chosen": -294.0, "logp/rejected": -374.0, "loss": 0.44970703125, "reward/accuracy": 0.625, "reward/chosen": -1.3203125, "reward/margin": 1.3046875, "reward/rejected": -2.625, "step": 2421 }, { "approx. KL/chosen": 114.5, "approx. KL/rejected": 720.0, "epoch": 0.7264547090581883, "grad_norm": 2.4409103393554688, "learning_rate": 1.7389881002441617e-06, "logp/chosen": -304.0, "logp/rejected": -262.0, "loss": 0.246826171875, "reward/accuracy": 0.9375, "reward/chosen": -1.140625, "reward/margin": 2.265625, "reward/rejected": -3.40625, "step": 2422 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 640.0, "epoch": 0.7267546490701859, "grad_norm": 4.060361385345459, "learning_rate": 1.7354180604241261e-06, "logp/chosen": -476.0, "logp/rejected": -440.0, "loss": 0.493621826171875, "reward/accuracy": 0.75, "reward/chosen": -1.953125, "reward/margin": 1.1953125, "reward/rejected": -3.15625, "step": 2423 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 1012.0, "epoch": 0.7270545890821836, "grad_norm": 3.760946750640869, "learning_rate": 1.7318509192562038e-06, "logp/chosen": -284.0, "logp/rejected": -372.0, "loss": 0.3159637451171875, "reward/accuracy": 0.9375, "reward/chosen": -1.8359375, "reward/margin": 2.25, "reward/rejected": -4.09375, "step": 2424 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 768.0, "epoch": 0.7273545290941812, "grad_norm": 2.8989720344543457, "learning_rate": 1.728286679907693e-06, "logp/chosen": -324.0, "logp/rejected": -272.0, "loss": 0.348114013671875, "reward/accuracy": 0.75, "reward/chosen": -1.4140625, "reward/margin": 2.078125, "reward/rejected": -3.484375, "step": 2425 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 788.0, "epoch": 0.7276544691061788, "grad_norm": 4.321732044219971, "learning_rate": 1.7247253455433123e-06, "logp/chosen": -306.0, "logp/rejected": -330.0, "loss": 0.46112060546875, "reward/accuracy": 0.8125, "reward/chosen": -1.5546875, "reward/margin": 1.84375, "reward/rejected": -3.40625, "step": 2426 }, { "approx. KL/chosen": 171.0, "approx. KL/rejected": 636.0, "epoch": 0.7279544091181763, "grad_norm": 2.7450270652770996, "learning_rate": 1.7211669193252057e-06, "logp/chosen": -312.0, "logp/rejected": -330.0, "loss": 0.299713134765625, "reward/accuracy": 0.8125, "reward/chosen": -1.5546875, "reward/margin": 1.6953125, "reward/rejected": -3.25, "step": 2427 }, { "approx. KL/chosen": 130.0, "approx. KL/rejected": 600.0, "epoch": 0.728254349130174, "grad_norm": 3.5038838386535645, "learning_rate": 1.7176114044129317e-06, "logp/chosen": -402.0, "logp/rejected": -436.0, "loss": 0.40625, "reward/accuracy": 0.6875, "reward/chosen": -1.46875, "reward/margin": 1.453125, "reward/rejected": -2.921875, "step": 2428 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 576.0, "epoch": 0.7285542891421716, "grad_norm": 4.567799091339111, "learning_rate": 1.7140588039634631e-06, "logp/chosen": -400.0, "logp/rejected": -320.0, "loss": 0.54815673828125, "reward/accuracy": 0.5625, "reward/chosen": -1.546875, "reward/margin": 1.2890625, "reward/rejected": -2.828125, "step": 2429 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 624.0, "epoch": 0.7288542291541692, "grad_norm": 3.644737482070923, "learning_rate": 1.7105091211311858e-06, "logp/chosen": -334.0, "logp/rejected": -276.0, "loss": 0.55645751953125, "reward/accuracy": 0.6875, "reward/chosen": -1.4765625, "reward/margin": 1.3984375, "reward/rejected": -2.875, "step": 2430 }, { "approx. KL/chosen": 484.0, "approx. KL/rejected": 1544.0, "epoch": 0.7291541691661668, "grad_norm": 3.954684019088745, "learning_rate": 1.7069623590678984e-06, "logp/chosen": -346.0, "logp/rejected": -332.0, "loss": 0.33319091796875, "reward/accuracy": 0.875, "reward/chosen": -2.5, "reward/margin": 2.40625, "reward/rejected": -4.90625, "step": 2431 }, { "approx. KL/chosen": 378.0, "approx. KL/rejected": 436.0, "epoch": 0.7294541091781643, "grad_norm": 5.513082504272461, "learning_rate": 1.7034185209228004e-06, "logp/chosen": -306.0, "logp/rejected": -378.0, "loss": 1.031982421875, "reward/accuracy": 0.625, "reward/chosen": -2.15625, "reward/margin": 0.265625, "reward/rejected": -2.421875, "step": 2432 }, { "approx. KL/chosen": 556.0, "approx. KL/rejected": 904.0, "epoch": 0.729754049190162, "grad_norm": 5.8748040199279785, "learning_rate": 1.6998776098425024e-06, "logp/chosen": -290.0, "logp/rejected": -232.0, "loss": 0.673095703125, "reward/accuracy": 0.625, "reward/chosen": -2.5, "reward/margin": 0.90625, "reward/rejected": -3.40625, "step": 2433 }, { "approx. KL/chosen": 212.0, "approx. KL/rejected": 636.0, "epoch": 0.7300539892021596, "grad_norm": 4.797735214233398, "learning_rate": 1.696339628971006e-06, "logp/chosen": -416.0, "logp/rejected": -358.0, "loss": 0.5595703125, "reward/accuracy": 0.6875, "reward/chosen": -1.6328125, "reward/margin": 1.46875, "reward/rejected": -3.09375, "step": 2434 }, { "approx. KL/chosen": 185.0, "approx. KL/rejected": 1056.0, "epoch": 0.7303539292141572, "grad_norm": 2.326383113861084, "learning_rate": 1.6928045814497206e-06, "logp/chosen": -400.0, "logp/rejected": -504.0, "loss": 0.23150634765625, "reward/accuracy": 0.875, "reward/chosen": -1.3828125, "reward/margin": 2.625, "reward/rejected": -4.0, "step": 2435 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 700.0, "epoch": 0.7306538692261547, "grad_norm": 3.5721828937530518, "learning_rate": 1.689272470417444e-06, "logp/chosen": -210.0, "logp/rejected": -296.0, "loss": 0.38006591796875, "reward/accuracy": 0.875, "reward/chosen": -1.9609375, "reward/margin": 1.3125, "reward/rejected": -3.28125, "step": 2436 }, { "approx. KL/chosen": 104.5, "approx. KL/rejected": 500.0, "epoch": 0.7309538092381523, "grad_norm": 3.7668559551239014, "learning_rate": 1.6857432990103745e-06, "logp/chosen": -308.0, "logp/rejected": -346.0, "loss": 0.4393310546875, "reward/accuracy": 0.8125, "reward/chosen": -1.015625, "reward/margin": 1.453125, "reward/rejected": -2.46875, "step": 2437 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 1008.0, "epoch": 0.73125374925015, "grad_norm": 3.99070405960083, "learning_rate": 1.6822170703620922e-06, "logp/chosen": -308.0, "logp/rejected": -298.0, "loss": 0.543792724609375, "reward/accuracy": 0.75, "reward/chosen": -1.8125, "reward/margin": 1.84375, "reward/rejected": -3.65625, "step": 2438 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 956.0, "epoch": 0.7315536892621476, "grad_norm": 3.460453510284424, "learning_rate": 1.6786937876035685e-06, "logp/chosen": -412.0, "logp/rejected": -304.0, "loss": 0.3256340026855469, "reward/accuracy": 0.6875, "reward/chosen": -1.453125, "reward/margin": 2.125, "reward/rejected": -3.578125, "step": 2439 }, { "approx. KL/chosen": 106.5, "approx. KL/rejected": 748.0, "epoch": 0.7318536292741452, "grad_norm": 3.122157096862793, "learning_rate": 1.675173453863157e-06, "logp/chosen": -288.0, "logp/rejected": -244.0, "loss": 0.29044342041015625, "reward/accuracy": 0.8125, "reward/chosen": -1.0859375, "reward/margin": 2.34375, "reward/rejected": -3.421875, "step": 2440 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 520.0, "epoch": 0.7321535692861427, "grad_norm": 4.9358978271484375, "learning_rate": 1.671656072266597e-06, "logp/chosen": -338.0, "logp/rejected": -372.0, "loss": 0.57647705078125, "reward/accuracy": 0.75, "reward/chosen": -1.5703125, "reward/margin": 1.1484375, "reward/rejected": -2.71875, "step": 2441 }, { "approx. KL/chosen": 148.0, "approx. KL/rejected": 544.0, "epoch": 0.7324535092981403, "grad_norm": 3.870187997817993, "learning_rate": 1.668141645937002e-06, "logp/chosen": -237.0, "logp/rejected": -214.0, "loss": 0.677154541015625, "reward/accuracy": 0.5625, "reward/chosen": -1.3125, "reward/margin": 1.046875, "reward/rejected": -2.359375, "step": 2442 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 1024.0, "epoch": 0.732753449310138, "grad_norm": 4.489999771118164, "learning_rate": 1.6646301779948637e-06, "logp/chosen": -462.0, "logp/rejected": -438.0, "loss": 0.5176925659179688, "reward/accuracy": 0.8125, "reward/chosen": -1.8515625, "reward/margin": 2.203125, "reward/rejected": -4.0625, "step": 2443 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 498.0, "epoch": 0.7330533893221356, "grad_norm": 3.4462273120880127, "learning_rate": 1.6611216715580448e-06, "logp/chosen": -276.0, "logp/rejected": -288.0, "loss": 0.464324951171875, "reward/accuracy": 0.75, "reward/chosen": -1.4765625, "reward/margin": 1.3359375, "reward/rejected": -2.8125, "step": 2444 }, { "approx. KL/chosen": 114.5, "approx. KL/rejected": 452.0, "epoch": 0.7333533293341332, "grad_norm": 4.119115829467773, "learning_rate": 1.6576161297417832e-06, "logp/chosen": -406.0, "logp/rejected": -350.0, "loss": 0.5283203125, "reward/accuracy": 0.6875, "reward/chosen": -0.89453125, "reward/margin": 1.1484375, "reward/rejected": -2.046875, "step": 2445 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 1272.0, "epoch": 0.7336532693461307, "grad_norm": 4.382141590118408, "learning_rate": 1.654113555658678e-06, "logp/chosen": -302.0, "logp/rejected": -408.0, "loss": 0.488189697265625, "reward/accuracy": 0.8125, "reward/chosen": -1.59375, "reward/margin": 2.171875, "reward/rejected": -3.765625, "step": 2446 }, { "approx. KL/chosen": 440.0, "approx. KL/rejected": 948.0, "epoch": 0.7339532093581284, "grad_norm": 4.351387977600098, "learning_rate": 1.6506139524187025e-06, "logp/chosen": -352.0, "logp/rejected": -334.0, "loss": 0.396636962890625, "reward/accuracy": 0.75, "reward/chosen": -2.390625, "reward/margin": 1.4375, "reward/rejected": -3.828125, "step": 2447 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 588.0, "epoch": 0.734253149370126, "grad_norm": 4.5756635665893555, "learning_rate": 1.6471173231291788e-06, "logp/chosen": -288.0, "logp/rejected": -292.0, "loss": 0.49713134765625, "reward/accuracy": 0.8125, "reward/chosen": -1.703125, "reward/margin": 1.25, "reward/rejected": -2.953125, "step": 2448 }, { "approx. KL/chosen": 476.0, "approx. KL/rejected": 748.0, "epoch": 0.7345530893821236, "grad_norm": 9.198992729187012, "learning_rate": 1.6436236708948017e-06, "logp/chosen": -312.0, "logp/rejected": -256.0, "loss": 0.804931640625, "reward/accuracy": 0.6875, "reward/chosen": -2.53125, "reward/margin": 0.6171875, "reward/rejected": -3.15625, "step": 2449 }, { "approx. KL/chosen": 338.0, "approx. KL/rejected": 980.0, "epoch": 0.7348530293941212, "grad_norm": 3.459073543548584, "learning_rate": 1.6401329988176134e-06, "logp/chosen": -328.0, "logp/rejected": -308.0, "loss": 0.324188232421875, "reward/accuracy": 0.8125, "reward/chosen": -2.125, "reward/margin": 1.9609375, "reward/rejected": -4.09375, "step": 2450 }, { "approx. KL/chosen": 245.0, "approx. KL/rejected": 636.0, "epoch": 0.7351529694061187, "grad_norm": 4.948724746704102, "learning_rate": 1.6366453099970165e-06, "logp/chosen": -346.0, "logp/rejected": -342.0, "loss": 0.5713958740234375, "reward/accuracy": 0.75, "reward/chosen": -1.59375, "reward/margin": 1.5234375, "reward/rejected": -3.125, "step": 2451 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 840.0, "epoch": 0.7354529094181164, "grad_norm": 4.884936809539795, "learning_rate": 1.6331606075297607e-06, "logp/chosen": -394.0, "logp/rejected": -402.0, "loss": 0.637451171875, "reward/accuracy": 0.5625, "reward/chosen": -1.7265625, "reward/margin": 1.4765625, "reward/rejected": -3.203125, "step": 2452 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 1120.0, "epoch": 0.735752849430114, "grad_norm": 3.0890355110168457, "learning_rate": 1.6296788945099445e-06, "logp/chosen": -392.0, "logp/rejected": -412.0, "loss": 0.3319664001464844, "reward/accuracy": 0.8125, "reward/chosen": -1.96875, "reward/margin": 1.9609375, "reward/rejected": -3.921875, "step": 2453 }, { "approx. KL/chosen": 149.0, "approx. KL/rejected": 856.0, "epoch": 0.7360527894421116, "grad_norm": 2.264528512954712, "learning_rate": 1.6262001740290113e-06, "logp/chosen": -446.0, "logp/rejected": -364.0, "loss": 0.20287322998046875, "reward/accuracy": 0.9375, "reward/chosen": -1.390625, "reward/margin": 2.296875, "reward/rejected": -3.6875, "step": 2454 }, { "approx. KL/chosen": 171.0, "approx. KL/rejected": 680.0, "epoch": 0.7363527294541091, "grad_norm": 3.7352488040924072, "learning_rate": 1.6227244491757521e-06, "logp/chosen": -354.0, "logp/rejected": -400.0, "loss": 0.460174560546875, "reward/accuracy": 0.75, "reward/chosen": -1.5, "reward/margin": 1.6796875, "reward/rejected": -3.171875, "step": 2455 }, { "approx. KL/chosen": 254.0, "approx. KL/rejected": 672.0, "epoch": 0.7366526694661067, "grad_norm": 3.9798836708068848, "learning_rate": 1.619251723036293e-06, "logp/chosen": -237.0, "logp/rejected": -286.0, "loss": 0.470703125, "reward/accuracy": 0.8125, "reward/chosen": -1.8984375, "reward/margin": 1.3671875, "reward/rejected": -3.265625, "step": 2456 }, { "approx. KL/chosen": 167.0, "approx. KL/rejected": 648.0, "epoch": 0.7369526094781044, "grad_norm": 3.7526440620422363, "learning_rate": 1.6157819986940992e-06, "logp/chosen": -398.0, "logp/rejected": -386.0, "loss": 0.325408935546875, "reward/accuracy": 0.8125, "reward/chosen": -1.3984375, "reward/margin": 1.6875, "reward/rejected": -3.09375, "step": 2457 }, { "approx. KL/chosen": 564.0, "approx. KL/rejected": 560.0, "epoch": 0.737252549490102, "grad_norm": 9.368180274963379, "learning_rate": 1.6123152792299695e-06, "logp/chosen": -258.0, "logp/rejected": -201.0, "loss": 0.933349609375, "reward/accuracy": 0.5, "reward/chosen": -2.53125, "reward/margin": 0.341796875, "reward/rejected": -2.875, "step": 2458 }, { "approx. KL/chosen": 304.0, "approx. KL/rejected": 980.0, "epoch": 0.7375524895020996, "grad_norm": 3.675523281097412, "learning_rate": 1.6088515677220383e-06, "logp/chosen": -390.0, "logp/rejected": -498.0, "loss": 0.439544677734375, "reward/accuracy": 0.875, "reward/chosen": -2.015625, "reward/margin": 1.6640625, "reward/rejected": -3.6875, "step": 2459 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 692.0, "epoch": 0.7378524295140971, "grad_norm": 4.97573709487915, "learning_rate": 1.6053908672457642e-06, "logp/chosen": -336.0, "logp/rejected": -404.0, "loss": 0.3981475830078125, "reward/accuracy": 0.8125, "reward/chosen": -1.2890625, "reward/margin": 1.921875, "reward/rejected": -3.21875, "step": 2460 }, { "approx. KL/chosen": 180.0, "approx. KL/rejected": 780.0, "epoch": 0.7381523695260948, "grad_norm": 2.881666421890259, "learning_rate": 1.6019331808739385e-06, "logp/chosen": -268.0, "logp/rejected": -332.0, "loss": 0.31268310546875, "reward/accuracy": 0.75, "reward/chosen": -1.53125, "reward/margin": 1.890625, "reward/rejected": -3.421875, "step": 2461 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 760.0, "epoch": 0.7384523095380924, "grad_norm": 4.110766887664795, "learning_rate": 1.5984785116766676e-06, "logp/chosen": -284.0, "logp/rejected": -308.0, "loss": 0.55255126953125, "reward/accuracy": 0.75, "reward/chosen": -1.5078125, "reward/margin": 1.3671875, "reward/rejected": -2.875, "step": 2462 }, { "approx. KL/chosen": 114.5, "approx. KL/rejected": 596.0, "epoch": 0.73875224955009, "grad_norm": 3.1725692749023438, "learning_rate": 1.5950268627213873e-06, "logp/chosen": -418.0, "logp/rejected": -358.0, "loss": 0.3482666015625, "reward/accuracy": 0.875, "reward/chosen": -1.109375, "reward/margin": 1.7890625, "reward/rejected": -2.890625, "step": 2463 }, { "approx. KL/chosen": 129.0, "approx. KL/rejected": 828.0, "epoch": 0.7390521895620876, "grad_norm": 3.3924434185028076, "learning_rate": 1.5915782370728455e-06, "logp/chosen": -338.0, "logp/rejected": -334.0, "loss": 0.342041015625, "reward/accuracy": 0.875, "reward/chosen": -1.140625, "reward/margin": 2.265625, "reward/rejected": -3.390625, "step": 2464 }, { "approx. KL/chosen": 168.0, "approx. KL/rejected": 724.0, "epoch": 0.7393521295740851, "grad_norm": 4.006190776824951, "learning_rate": 1.5881326377931117e-06, "logp/chosen": -368.0, "logp/rejected": -324.0, "loss": 0.320068359375, "reward/accuracy": 0.875, "reward/chosen": -1.328125, "reward/margin": 2.125, "reward/rejected": -3.453125, "step": 2465 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 306.0, "epoch": 0.7396520695860828, "grad_norm": 5.377490520477295, "learning_rate": 1.5846900679415628e-06, "logp/chosen": -270.0, "logp/rejected": -330.0, "loss": 0.7322998046875, "reward/accuracy": 0.5625, "reward/chosen": -1.46875, "reward/margin": 0.57421875, "reward/rejected": -2.046875, "step": 2466 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 1080.0, "epoch": 0.7399520095980804, "grad_norm": 3.168931245803833, "learning_rate": 1.5812505305748888e-06, "logp/chosen": -344.0, "logp/rejected": -332.0, "loss": 0.4343719482421875, "reward/accuracy": 0.875, "reward/chosen": -1.46875, "reward/margin": 2.625, "reward/rejected": -4.09375, "step": 2467 }, { "approx. KL/chosen": 132.0, "approx. KL/rejected": 968.0, "epoch": 0.740251949610078, "grad_norm": 4.083230018615723, "learning_rate": 1.5778140287470843e-06, "logp/chosen": -296.0, "logp/rejected": -308.0, "loss": 0.4040985107421875, "reward/accuracy": 0.8125, "reward/chosen": -1.2265625, "reward/margin": 2.390625, "reward/rejected": -3.625, "step": 2468 }, { "approx. KL/chosen": 382.0, "approx. KL/rejected": 784.0, "epoch": 0.7405518896220756, "grad_norm": 4.711359024047852, "learning_rate": 1.5743805655094541e-06, "logp/chosen": -406.0, "logp/rejected": -310.0, "loss": 0.591400146484375, "reward/accuracy": 0.75, "reward/chosen": -2.125, "reward/margin": 1.53125, "reward/rejected": -3.65625, "step": 2469 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 740.0, "epoch": 0.7408518296340731, "grad_norm": 4.83462381362915, "learning_rate": 1.5709501439105985e-06, "logp/chosen": -302.0, "logp/rejected": -278.0, "loss": 0.41650390625, "reward/accuracy": 0.625, "reward/chosen": -1.6171875, "reward/margin": 1.5703125, "reward/rejected": -3.1875, "step": 2470 }, { "approx. KL/chosen": 362.0, "approx. KL/rejected": 856.0, "epoch": 0.7411517696460708, "grad_norm": 5.883728504180908, "learning_rate": 1.5675227669964244e-06, "logp/chosen": -424.0, "logp/rejected": -392.0, "loss": 0.784271240234375, "reward/accuracy": 0.75, "reward/chosen": -2.046875, "reward/margin": 1.3359375, "reward/rejected": -3.375, "step": 2471 }, { "approx. KL/chosen": 96.5, "approx. KL/rejected": 584.0, "epoch": 0.7414517096580684, "grad_norm": 3.577576160430908, "learning_rate": 1.5640984378101249e-06, "logp/chosen": -284.0, "logp/rejected": -282.0, "loss": 0.48181915283203125, "reward/accuracy": 0.8125, "reward/chosen": -0.69921875, "reward/margin": 1.7734375, "reward/rejected": -2.46875, "step": 2472 }, { "approx. KL/chosen": 232.0, "approx. KL/rejected": 752.0, "epoch": 0.741751649670066, "grad_norm": 3.6288723945617676, "learning_rate": 1.5606771593921976e-06, "logp/chosen": -328.0, "logp/rejected": -366.0, "loss": 0.4033203125, "reward/accuracy": 0.875, "reward/chosen": -1.4921875, "reward/margin": 1.953125, "reward/rejected": -3.4375, "step": 2473 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 680.0, "epoch": 0.7420515896820636, "grad_norm": 3.9055042266845703, "learning_rate": 1.5572589347804246e-06, "logp/chosen": -274.0, "logp/rejected": -223.0, "loss": 0.4769287109375, "reward/accuracy": 0.75, "reward/chosen": -1.796875, "reward/margin": 1.453125, "reward/rejected": -3.25, "step": 2474 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 896.0, "epoch": 0.7423515296940612, "grad_norm": 2.8688926696777344, "learning_rate": 1.553843767009881e-06, "logp/chosen": -378.0, "logp/rejected": -386.0, "loss": 0.330047607421875, "reward/accuracy": 0.8125, "reward/chosen": -1.3203125, "reward/margin": 2.296875, "reward/rejected": -3.625, "step": 2475 }, { "approx. KL/chosen": 254.0, "approx. KL/rejected": 564.0, "epoch": 0.7426514697060588, "grad_norm": 4.901071071624756, "learning_rate": 1.5504316591129247e-06, "logp/chosen": -452.0, "logp/rejected": -420.0, "loss": 0.44482421875, "reward/accuracy": 0.875, "reward/chosen": -1.578125, "reward/margin": 1.2890625, "reward/rejected": -2.859375, "step": 2476 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 508.0, "epoch": 0.7429514097180564, "grad_norm": 2.7303149700164795, "learning_rate": 1.5470226141191969e-06, "logp/chosen": -346.0, "logp/rejected": -358.0, "loss": 0.38311767578125, "reward/accuracy": 0.8125, "reward/chosen": -1.3671875, "reward/margin": 1.296875, "reward/rejected": -2.671875, "step": 2477 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 428.0, "epoch": 0.743251349730054, "grad_norm": 3.695356845855713, "learning_rate": 1.5436166350556181e-06, "logp/chosen": -372.0, "logp/rejected": -332.0, "loss": 0.468170166015625, "reward/accuracy": 0.75, "reward/chosen": -1.34375, "reward/margin": 1.2109375, "reward/rejected": -2.5625, "step": 2478 }, { "approx. KL/chosen": 138.0, "approx. KL/rejected": 812.0, "epoch": 0.7435512897420515, "grad_norm": 3.695773124694824, "learning_rate": 1.5402137249463917e-06, "logp/chosen": -354.0, "logp/rejected": -454.0, "loss": 0.40386962890625, "reward/accuracy": 0.8125, "reward/chosen": -1.40625, "reward/margin": 1.9453125, "reward/rejected": -3.359375, "step": 2479 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 1560.0, "epoch": 0.7438512297540492, "grad_norm": 3.6227848529815674, "learning_rate": 1.5368138868129905e-06, "logp/chosen": -302.0, "logp/rejected": -432.0, "loss": 0.26076793670654297, "reward/accuracy": 0.9375, "reward/chosen": -1.3125, "reward/margin": 3.609375, "reward/rejected": -4.9375, "step": 2480 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 394.0, "epoch": 0.7441511697660468, "grad_norm": 4.635890007019043, "learning_rate": 1.5334171236741624e-06, "logp/chosen": -213.0, "logp/rejected": -328.0, "loss": 0.767364501953125, "reward/accuracy": 0.5, "reward/chosen": -1.6015625, "reward/margin": 0.494140625, "reward/rejected": -2.09375, "step": 2481 }, { "approx. KL/chosen": 124.0, "approx. KL/rejected": 668.0, "epoch": 0.7444511097780444, "grad_norm": 3.80383038520813, "learning_rate": 1.5300234385459228e-06, "logp/chosen": -332.0, "logp/rejected": -366.0, "loss": 0.4462890625, "reward/accuracy": 0.6875, "reward/chosen": -1.203125, "reward/margin": 1.7109375, "reward/rejected": -2.921875, "step": 2482 }, { "approx. KL/chosen": 138.0, "approx. KL/rejected": 944.0, "epoch": 0.744751049790042, "grad_norm": 2.4800944328308105, "learning_rate": 1.5266328344415587e-06, "logp/chosen": -304.0, "logp/rejected": -316.0, "loss": 0.193359375, "reward/accuracy": 0.9375, "reward/chosen": -1.21875, "reward/margin": 2.734375, "reward/rejected": -3.953125, "step": 2483 }, { "approx. KL/chosen": 125.5, "approx. KL/rejected": 704.0, "epoch": 0.7450509898020395, "grad_norm": 3.076301097869873, "learning_rate": 1.5232453143716157e-06, "logp/chosen": -364.0, "logp/rejected": -374.0, "loss": 0.3160400390625, "reward/accuracy": 0.8125, "reward/chosen": -0.96484375, "reward/margin": 2.265625, "reward/rejected": -3.234375, "step": 2484 }, { "approx. KL/chosen": 123.5, "approx. KL/rejected": 1208.0, "epoch": 0.7453509298140372, "grad_norm": 3.0185904502868652, "learning_rate": 1.519860881343908e-06, "logp/chosen": -272.0, "logp/rejected": -344.0, "loss": 0.316925048828125, "reward/accuracy": 0.9375, "reward/chosen": -1.0859375, "reward/margin": 2.78125, "reward/rejected": -3.859375, "step": 2485 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 740.0, "epoch": 0.7456508698260348, "grad_norm": 5.280910968780518, "learning_rate": 1.5164795383634984e-06, "logp/chosen": -288.0, "logp/rejected": -340.0, "loss": 0.5557651519775391, "reward/accuracy": 0.75, "reward/chosen": -1.890625, "reward/margin": 1.515625, "reward/rejected": -3.40625, "step": 2486 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 648.0, "epoch": 0.7459508098380324, "grad_norm": 4.947169780731201, "learning_rate": 1.5131012884327167e-06, "logp/chosen": -238.0, "logp/rejected": -312.0, "loss": 0.5457763671875, "reward/accuracy": 0.625, "reward/chosen": -1.484375, "reward/margin": 1.515625, "reward/rejected": -3.0, "step": 2487 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 656.0, "epoch": 0.74625074985003, "grad_norm": 5.36045503616333, "learning_rate": 1.5097261345511394e-06, "logp/chosen": -424.0, "logp/rejected": -442.0, "loss": 0.47802734375, "reward/accuracy": 0.75, "reward/chosen": -2.03125, "reward/margin": 1.265625, "reward/rejected": -3.296875, "step": 2488 }, { "approx. KL/chosen": 67.0, "approx. KL/rejected": 652.0, "epoch": 0.7465506898620275, "grad_norm": 2.2975473403930664, "learning_rate": 1.5063540797155995e-06, "logp/chosen": -438.0, "logp/rejected": -474.0, "loss": 0.25469970703125, "reward/accuracy": 0.9375, "reward/chosen": -0.64453125, "reward/margin": 2.25, "reward/rejected": -2.890625, "step": 2489 }, { "approx. KL/chosen": 238.0, "approx. KL/rejected": 800.0, "epoch": 0.7468506298740252, "grad_norm": 3.389472246170044, "learning_rate": 1.5029851269201729e-06, "logp/chosen": -322.0, "logp/rejected": -314.0, "loss": 0.32965087890625, "reward/accuracy": 0.8125, "reward/chosen": -1.6171875, "reward/margin": 1.7734375, "reward/rejected": -3.390625, "step": 2490 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 584.0, "epoch": 0.7471505698860228, "grad_norm": 4.774756908416748, "learning_rate": 1.4996192791561841e-06, "logp/chosen": -270.0, "logp/rejected": -278.0, "loss": 0.5009918212890625, "reward/accuracy": 0.625, "reward/chosen": -1.3125, "reward/margin": 1.6640625, "reward/rejected": -2.984375, "step": 2491 }, { "approx. KL/chosen": 188.0, "approx. KL/rejected": 884.0, "epoch": 0.7474505098980204, "grad_norm": 3.7378363609313965, "learning_rate": 1.4962565394121986e-06, "logp/chosen": -372.0, "logp/rejected": -276.0, "loss": 0.36328125, "reward/accuracy": 0.8125, "reward/chosen": -1.546875, "reward/margin": 2.078125, "reward/rejected": -3.625, "step": 2492 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 588.0, "epoch": 0.747750449910018, "grad_norm": 5.069326400756836, "learning_rate": 1.4928969106740266e-06, "logp/chosen": -278.0, "logp/rejected": -274.0, "loss": 0.68939208984375, "reward/accuracy": 0.75, "reward/chosen": -1.6953125, "reward/margin": 1.1015625, "reward/rejected": -2.796875, "step": 2493 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 422.0, "epoch": 0.7480503899220156, "grad_norm": 7.066447734832764, "learning_rate": 1.48954039592471e-06, "logp/chosen": -262.0, "logp/rejected": -324.0, "loss": 0.802947998046875, "reward/accuracy": 0.5625, "reward/chosen": -1.609375, "reward/margin": 0.78515625, "reward/rejected": -2.390625, "step": 2494 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 768.0, "epoch": 0.7483503299340132, "grad_norm": 4.006478309631348, "learning_rate": 1.4861869981445342e-06, "logp/chosen": -366.0, "logp/rejected": -262.0, "loss": 0.5794677734375, "reward/accuracy": 0.8125, "reward/chosen": -1.6796875, "reward/margin": 1.6015625, "reward/rejected": -3.28125, "step": 2495 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 824.0, "epoch": 0.7486502699460108, "grad_norm": 2.190577268600464, "learning_rate": 1.4828367203110045e-06, "logp/chosen": -418.0, "logp/rejected": -322.0, "loss": 0.2862548828125, "reward/accuracy": 0.8125, "reward/chosen": -1.328125, "reward/margin": 2.1875, "reward/rejected": -3.515625, "step": 2496 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 428.0, "epoch": 0.7489502099580084, "grad_norm": 5.094438552856445, "learning_rate": 1.4794895653988688e-06, "logp/chosen": -336.0, "logp/rejected": -344.0, "loss": 0.7279052734375, "reward/accuracy": 0.5625, "reward/chosen": -1.671875, "reward/margin": 0.734375, "reward/rejected": -2.40625, "step": 2497 }, { "approx. KL/chosen": 132.0, "approx. KL/rejected": 392.0, "epoch": 0.749250149970006, "grad_norm": 4.695434093475342, "learning_rate": 1.4761455363800924e-06, "logp/chosen": -370.0, "logp/rejected": -360.0, "loss": 0.542999267578125, "reward/accuracy": 0.75, "reward/chosen": -1.1640625, "reward/margin": 1.0546875, "reward/rejected": -2.21875, "step": 2498 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 796.0, "epoch": 0.7495500899820036, "grad_norm": 2.5955257415771484, "learning_rate": 1.472804636223873e-06, "logp/chosen": -418.0, "logp/rejected": -412.0, "loss": 0.2567901611328125, "reward/accuracy": 0.875, "reward/chosen": -1.3515625, "reward/margin": 2.234375, "reward/rejected": -3.578125, "step": 2499 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 648.0, "epoch": 0.7498500299940012, "grad_norm": 3.368382453918457, "learning_rate": 1.469466867896625e-06, "logp/chosen": -280.0, "logp/rejected": -272.0, "loss": 0.32861328125, "reward/accuracy": 0.9375, "reward/chosen": -1.8125, "reward/margin": 1.3203125, "reward/rejected": -3.125, "step": 2500 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 1072.0, "epoch": 0.7501499700059988, "grad_norm": 2.7620131969451904, "learning_rate": 1.4661322343619828e-06, "logp/chosen": -316.0, "logp/rejected": -440.0, "loss": 0.27010345458984375, "reward/accuracy": 0.8125, "reward/chosen": -1.828125, "reward/margin": 2.296875, "reward/rejected": -4.125, "step": 2501 }, { "approx. KL/chosen": 172.0, "approx. KL/rejected": 1004.0, "epoch": 0.7504499100179964, "grad_norm": 3.6475579738616943, "learning_rate": 1.4628007385807962e-06, "logp/chosen": -296.0, "logp/rejected": -418.0, "loss": 0.3563232421875, "reward/accuracy": 0.875, "reward/chosen": -1.4453125, "reward/margin": 2.296875, "reward/rejected": -3.75, "step": 2502 }, { "approx. KL/chosen": 145.0, "approx. KL/rejected": 792.0, "epoch": 0.750749850029994, "grad_norm": 3.1292073726654053, "learning_rate": 1.4594723835111347e-06, "logp/chosen": -324.0, "logp/rejected": -338.0, "loss": 0.287109375, "reward/accuracy": 0.875, "reward/chosen": -1.1015625, "reward/margin": 2.5, "reward/rejected": -3.59375, "step": 2503 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 804.0, "epoch": 0.7510497900419916, "grad_norm": 3.682478427886963, "learning_rate": 1.4561471721082732e-06, "logp/chosen": -213.0, "logp/rejected": -256.0, "loss": 0.493560791015625, "reward/accuracy": 0.625, "reward/chosen": -1.28125, "reward/margin": 2.03125, "reward/rejected": -3.296875, "step": 2504 }, { "approx. KL/chosen": 362.0, "approx. KL/rejected": 680.0, "epoch": 0.7513497300539892, "grad_norm": 4.519556045532227, "learning_rate": 1.4528251073246967e-06, "logp/chosen": -452.0, "logp/rejected": -416.0, "loss": 0.5791015625, "reward/accuracy": 0.6875, "reward/chosen": -2.25, "reward/margin": 0.8203125, "reward/rejected": -3.0625, "step": 2505 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 864.0, "epoch": 0.7516496700659868, "grad_norm": 7.303446292877197, "learning_rate": 1.4495061921100968e-06, "logp/chosen": -406.0, "logp/rejected": -414.0, "loss": 0.75787353515625, "reward/accuracy": 0.75, "reward/chosen": -2.046875, "reward/margin": 1.3125, "reward/rejected": -3.359375, "step": 2506 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 900.0, "epoch": 0.7519496100779844, "grad_norm": 5.860036373138428, "learning_rate": 1.44619042941137e-06, "logp/chosen": -440.0, "logp/rejected": -328.0, "loss": 0.5557403564453125, "reward/accuracy": 0.6875, "reward/chosen": -1.5, "reward/margin": 1.4921875, "reward/rejected": -2.984375, "step": 2507 }, { "approx. KL/chosen": 222.0, "approx. KL/rejected": 932.0, "epoch": 0.752249550089982, "grad_norm": 4.43947172164917, "learning_rate": 1.4428778221726103e-06, "logp/chosen": -422.0, "logp/rejected": -438.0, "loss": 0.6096954345703125, "reward/accuracy": 0.6875, "reward/chosen": -1.8125, "reward/margin": 1.5625, "reward/rejected": -3.375, "step": 2508 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 720.0, "epoch": 0.7525494901019796, "grad_norm": 3.8941869735717773, "learning_rate": 1.4395683733351158e-06, "logp/chosen": -308.0, "logp/rejected": -247.0, "loss": 0.408660888671875, "reward/accuracy": 0.75, "reward/chosen": -1.4765625, "reward/margin": 1.9609375, "reward/rejected": -3.4375, "step": 2509 }, { "approx. KL/chosen": 760.0, "approx. KL/rejected": 972.0, "epoch": 0.7528494301139772, "grad_norm": 8.150148391723633, "learning_rate": 1.4362620858373705e-06, "logp/chosen": -296.0, "logp/rejected": -314.0, "loss": 0.76043701171875, "reward/accuracy": 0.75, "reward/chosen": -2.5, "reward/margin": 1.1796875, "reward/rejected": -3.6875, "step": 2510 }, { "approx. KL/chosen": 115.0, "approx. KL/rejected": 1048.0, "epoch": 0.7531493701259748, "grad_norm": 2.937276840209961, "learning_rate": 1.4329589626150625e-06, "logp/chosen": -210.0, "logp/rejected": -394.0, "loss": 0.32210445404052734, "reward/accuracy": 0.8125, "reward/chosen": -1.015625, "reward/margin": 2.703125, "reward/rejected": -3.71875, "step": 2511 }, { "approx. KL/chosen": 121.5, "approx. KL/rejected": 744.0, "epoch": 0.7534493101379725, "grad_norm": 2.6756229400634766, "learning_rate": 1.4296590066010614e-06, "logp/chosen": -280.0, "logp/rejected": -223.0, "loss": 0.368499755859375, "reward/accuracy": 0.875, "reward/chosen": -1.1171875, "reward/margin": 2.046875, "reward/rejected": -3.171875, "step": 2512 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 450.0, "epoch": 0.75374925014997, "grad_norm": 4.330255508422852, "learning_rate": 1.4263622207254308e-06, "logp/chosen": -412.0, "logp/rejected": -358.0, "loss": 0.52386474609375, "reward/accuracy": 0.6875, "reward/chosen": -1.5546875, "reward/margin": 1.1875, "reward/rejected": -2.734375, "step": 2513 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 1200.0, "epoch": 0.7540491901619676, "grad_norm": 3.258575201034546, "learning_rate": 1.4230686079154166e-06, "logp/chosen": -241.0, "logp/rejected": -378.0, "loss": 0.4185676574707031, "reward/accuracy": 0.75, "reward/chosen": -1.734375, "reward/margin": 1.7734375, "reward/rejected": -3.5, "step": 2514 }, { "approx. KL/chosen": 338.0, "approx. KL/rejected": 688.0, "epoch": 0.7543491301739652, "grad_norm": 4.457991123199463, "learning_rate": 1.4197781710954467e-06, "logp/chosen": -340.0, "logp/rejected": -350.0, "loss": 0.46484375, "reward/accuracy": 0.75, "reward/chosen": -1.953125, "reward/margin": 1.34375, "reward/rejected": -3.296875, "step": 2515 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 796.0, "epoch": 0.7546490701859628, "grad_norm": 3.8077073097229004, "learning_rate": 1.4164909131871291e-06, "logp/chosen": -352.0, "logp/rejected": -412.0, "loss": 0.2753448486328125, "reward/accuracy": 0.875, "reward/chosen": -1.28125, "reward/margin": 2.234375, "reward/rejected": -3.5, "step": 2516 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 824.0, "epoch": 0.7549490101979605, "grad_norm": 4.629327774047852, "learning_rate": 1.4132068371092533e-06, "logp/chosen": -336.0, "logp/rejected": -404.0, "loss": 0.568115234375, "reward/accuracy": 0.6875, "reward/chosen": -1.4609375, "reward/margin": 1.3515625, "reward/rejected": -2.8125, "step": 2517 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 1256.0, "epoch": 0.755248950209958, "grad_norm": 2.9707815647125244, "learning_rate": 1.4099259457777764e-06, "logp/chosen": -386.0, "logp/rejected": -520.0, "loss": 0.2611255645751953, "reward/accuracy": 0.875, "reward/chosen": -1.796875, "reward/margin": 2.5625, "reward/rejected": -4.34375, "step": 2518 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 528.0, "epoch": 0.7555488902219556, "grad_norm": 3.087674140930176, "learning_rate": 1.4066482421058375e-06, "logp/chosen": -286.0, "logp/rejected": -290.0, "loss": 0.387908935546875, "reward/accuracy": 0.875, "reward/chosen": -1.390625, "reward/margin": 1.3515625, "reward/rejected": -2.734375, "step": 2519 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 580.0, "epoch": 0.7558488302339532, "grad_norm": 5.5659403800964355, "learning_rate": 1.4033737290037325e-06, "logp/chosen": -384.0, "logp/rejected": -388.0, "loss": 0.554290771484375, "reward/accuracy": 0.75, "reward/chosen": -1.5625, "reward/margin": 1.3203125, "reward/rejected": -2.875, "step": 2520 }, { "approx. KL/chosen": 255.0, "approx. KL/rejected": 1248.0, "epoch": 0.7561487702459508, "grad_norm": 3.5391829013824463, "learning_rate": 1.4001024093789372e-06, "logp/chosen": -358.0, "logp/rejected": -462.0, "loss": 0.2647857666015625, "reward/accuracy": 0.875, "reward/chosen": -1.7109375, "reward/margin": 2.515625, "reward/rejected": -4.21875, "step": 2521 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 828.0, "epoch": 0.7564487102579485, "grad_norm": 5.653744220733643, "learning_rate": 1.3968342861360824e-06, "logp/chosen": -372.0, "logp/rejected": -316.0, "loss": 0.4894256591796875, "reward/accuracy": 0.75, "reward/chosen": -1.6875, "reward/margin": 1.8203125, "reward/rejected": -3.515625, "step": 2522 }, { "approx. KL/chosen": 384.0, "approx. KL/rejected": 768.0, "epoch": 0.756748650269946, "grad_norm": 4.093700408935547, "learning_rate": 1.393569362176967e-06, "logp/chosen": -206.0, "logp/rejected": -358.0, "loss": 0.4212646484375, "reward/accuracy": 0.875, "reward/chosen": -1.8359375, "reward/margin": 1.5390625, "reward/rejected": -3.375, "step": 2523 }, { "approx. KL/chosen": 225.0, "approx. KL/rejected": 1256.0, "epoch": 0.7570485902819436, "grad_norm": 3.496495485305786, "learning_rate": 1.3903076404005467e-06, "logp/chosen": -350.0, "logp/rejected": -470.0, "loss": 0.44483184814453125, "reward/accuracy": 0.8125, "reward/chosen": -1.6953125, "reward/margin": 1.921875, "reward/rejected": -3.609375, "step": 2524 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 1176.0, "epoch": 0.7573485302939412, "grad_norm": 3.974156141281128, "learning_rate": 1.3870491237029332e-06, "logp/chosen": -418.0, "logp/rejected": -364.0, "loss": 0.3887360095977783, "reward/accuracy": 0.8125, "reward/chosen": -1.7421875, "reward/margin": 2.359375, "reward/rejected": -4.09375, "step": 2525 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 868.0, "epoch": 0.7576484703059388, "grad_norm": 5.44366455078125, "learning_rate": 1.3837938149773915e-06, "logp/chosen": -288.0, "logp/rejected": -338.0, "loss": 0.5787887573242188, "reward/accuracy": 0.75, "reward/chosen": -1.6171875, "reward/margin": 1.84375, "reward/rejected": -3.46875, "step": 2526 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 928.0, "epoch": 0.7579484103179364, "grad_norm": 3.0567128658294678, "learning_rate": 1.3805417171143437e-06, "logp/chosen": -604.0, "logp/rejected": -528.0, "loss": 0.37945556640625, "reward/accuracy": 0.8125, "reward/chosen": -1.5390625, "reward/margin": 2.0625, "reward/rejected": -3.59375, "step": 2527 }, { "approx. KL/chosen": 188.0, "approx. KL/rejected": 884.0, "epoch": 0.758248350329934, "grad_norm": 4.244494438171387, "learning_rate": 1.3772928330013551e-06, "logp/chosen": -244.0, "logp/rejected": -334.0, "loss": 0.45404052734375, "reward/accuracy": 0.75, "reward/chosen": -1.6015625, "reward/margin": 2.078125, "reward/rejected": -3.6875, "step": 2528 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 584.0, "epoch": 0.7585482903419316, "grad_norm": 4.528592586517334, "learning_rate": 1.3740471655231396e-06, "logp/chosen": -290.0, "logp/rejected": -354.0, "loss": 0.4788818359375, "reward/accuracy": 0.8125, "reward/chosen": -1.6796875, "reward/margin": 1.2578125, "reward/rejected": -2.9375, "step": 2529 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 504.0, "epoch": 0.7588482303539292, "grad_norm": 3.9950101375579834, "learning_rate": 1.3708047175615542e-06, "logp/chosen": -432.0, "logp/rejected": -334.0, "loss": 0.50421142578125, "reward/accuracy": 0.75, "reward/chosen": -1.6171875, "reward/margin": 1.2578125, "reward/rejected": -2.875, "step": 2530 }, { "approx. KL/chosen": 74.0, "approx. KL/rejected": 410.0, "epoch": 0.7591481703659269, "grad_norm": 2.622417449951172, "learning_rate": 1.3675654919955999e-06, "logp/chosen": -344.0, "logp/rejected": -314.0, "loss": 0.3135986328125, "reward/accuracy": 0.875, "reward/chosen": -0.69921875, "reward/margin": 1.546875, "reward/rejected": -2.25, "step": 2531 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 792.0, "epoch": 0.7594481103779244, "grad_norm": 5.41845703125, "learning_rate": 1.3643294917014123e-06, "logp/chosen": -342.0, "logp/rejected": -338.0, "loss": 0.7109756469726562, "reward/accuracy": 0.625, "reward/chosen": -1.546875, "reward/margin": 1.9140625, "reward/rejected": -3.453125, "step": 2532 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 728.0, "epoch": 0.759748050389922, "grad_norm": 4.614269256591797, "learning_rate": 1.3610967195522706e-06, "logp/chosen": -280.0, "logp/rejected": -326.0, "loss": 0.72607421875, "reward/accuracy": 0.625, "reward/chosen": -1.96875, "reward/margin": 1.1328125, "reward/rejected": -3.109375, "step": 2533 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 840.0, "epoch": 0.7600479904019196, "grad_norm": 4.211655139923096, "learning_rate": 1.3578671784185754e-06, "logp/chosen": -324.0, "logp/rejected": -340.0, "loss": 0.40976715087890625, "reward/accuracy": 0.8125, "reward/chosen": -1.34375, "reward/margin": 2.046875, "reward/rejected": -3.390625, "step": 2534 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 584.0, "epoch": 0.7603479304139172, "grad_norm": 2.9985640048980713, "learning_rate": 1.3546408711678716e-06, "logp/chosen": -340.0, "logp/rejected": -314.0, "loss": 0.350128173828125, "reward/accuracy": 0.875, "reward/chosen": -1.234375, "reward/margin": 1.46875, "reward/rejected": -2.703125, "step": 2535 }, { "approx. KL/chosen": 332.0, "approx. KL/rejected": 868.0, "epoch": 0.7606478704259149, "grad_norm": 5.7545647621154785, "learning_rate": 1.3514178006648231e-06, "logp/chosen": -255.0, "logp/rejected": -310.0, "loss": 0.652557373046875, "reward/accuracy": 0.75, "reward/chosen": -2.015625, "reward/margin": 1.6484375, "reward/rejected": -3.65625, "step": 2536 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 466.0, "epoch": 0.7609478104379124, "grad_norm": 4.322669982910156, "learning_rate": 1.348197969771227e-06, "logp/chosen": -254.0, "logp/rejected": -227.0, "loss": 0.6263427734375, "reward/accuracy": 0.6875, "reward/chosen": -1.8515625, "reward/margin": 0.7578125, "reward/rejected": -2.609375, "step": 2537 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 836.0, "epoch": 0.76124775044991, "grad_norm": 5.8876447677612305, "learning_rate": 1.344981381345999e-06, "logp/chosen": -388.0, "logp/rejected": -438.0, "loss": 0.61199951171875, "reward/accuracy": 0.6875, "reward/chosen": -1.4609375, "reward/margin": 1.703125, "reward/rejected": -3.171875, "step": 2538 }, { "approx. KL/chosen": 376.0, "approx. KL/rejected": 1104.0, "epoch": 0.7615476904619076, "grad_norm": 3.4164998531341553, "learning_rate": 1.341768038245177e-06, "logp/chosen": -354.0, "logp/rejected": -460.0, "loss": 0.297271728515625, "reward/accuracy": 0.875, "reward/chosen": -2.328125, "reward/margin": 2.109375, "reward/rejected": -4.4375, "step": 2539 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 636.0, "epoch": 0.7618476304739052, "grad_norm": 3.5665230751037598, "learning_rate": 1.3385579433219166e-06, "logp/chosen": -364.0, "logp/rejected": -300.0, "loss": 0.4268798828125, "reward/accuracy": 0.8125, "reward/chosen": -1.5625, "reward/margin": 1.421875, "reward/rejected": -2.984375, "step": 2540 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 1176.0, "epoch": 0.7621475704859029, "grad_norm": 4.636402130126953, "learning_rate": 1.3353510994264929e-06, "logp/chosen": -300.0, "logp/rejected": -302.0, "loss": 0.478271484375, "reward/accuracy": 0.625, "reward/chosen": -1.8125, "reward/margin": 2.203125, "reward/rejected": -4.03125, "step": 2541 }, { "approx. KL/chosen": 129.0, "approx. KL/rejected": 552.0, "epoch": 0.7624475104979004, "grad_norm": 3.641508102416992, "learning_rate": 1.332147509406289e-06, "logp/chosen": -378.0, "logp/rejected": -302.0, "loss": 0.37481689453125, "reward/accuracy": 0.8125, "reward/chosen": -1.21875, "reward/margin": 1.6796875, "reward/rejected": -2.90625, "step": 2542 }, { "approx. KL/chosen": 496.0, "approx. KL/rejected": 544.0, "epoch": 0.762747450509898, "grad_norm": 8.691228866577148, "learning_rate": 1.328947176105807e-06, "logp/chosen": -350.0, "logp/rejected": -396.0, "loss": 0.835784912109375, "reward/accuracy": 0.75, "reward/chosen": -2.078125, "reward/margin": 0.7890625, "reward/rejected": -2.859375, "step": 2543 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 656.0, "epoch": 0.7630473905218956, "grad_norm": 4.738447666168213, "learning_rate": 1.3257501023666453e-06, "logp/chosen": -360.0, "logp/rejected": -326.0, "loss": 0.56573486328125, "reward/accuracy": 0.6875, "reward/chosen": -1.7265625, "reward/margin": 1.4453125, "reward/rejected": -3.171875, "step": 2544 }, { "approx. KL/chosen": 76.5, "approx. KL/rejected": 700.0, "epoch": 0.7633473305338933, "grad_norm": 3.166135311126709, "learning_rate": 1.3225562910275197e-06, "logp/chosen": -233.0, "logp/rejected": -312.0, "loss": 0.321929931640625, "reward/accuracy": 0.875, "reward/chosen": -0.81640625, "reward/margin": 2.265625, "reward/rejected": -3.078125, "step": 2545 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 1320.0, "epoch": 0.7636472705458909, "grad_norm": 3.370778799057007, "learning_rate": 1.319365744924241e-06, "logp/chosen": -318.0, "logp/rejected": -380.0, "loss": 0.30239105224609375, "reward/accuracy": 0.8125, "reward/chosen": -1.2734375, "reward/margin": 2.90625, "reward/rejected": -4.1875, "step": 2546 }, { "approx. KL/chosen": 113.0, "approx. KL/rejected": 382.0, "epoch": 0.7639472105578884, "grad_norm": 3.9713306427001953, "learning_rate": 1.316178466889727e-06, "logp/chosen": -446.0, "logp/rejected": -426.0, "loss": 0.494537353515625, "reward/accuracy": 0.75, "reward/chosen": -1.0, "reward/margin": 1.1015625, "reward/rejected": -2.109375, "step": 2547 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 816.0, "epoch": 0.764247150569886, "grad_norm": 3.721125841140747, "learning_rate": 1.3129944597539907e-06, "logp/chosen": -310.0, "logp/rejected": -322.0, "loss": 0.26363372802734375, "reward/accuracy": 0.8125, "reward/chosen": -1.0703125, "reward/margin": 2.375, "reward/rejected": -3.4375, "step": 2548 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 804.0, "epoch": 0.7645470905818836, "grad_norm": 4.519914627075195, "learning_rate": 1.3098137263441401e-06, "logp/chosen": -376.0, "logp/rejected": -286.0, "loss": 0.5802001953125, "reward/accuracy": 0.625, "reward/chosen": -1.15625, "reward/margin": 1.8671875, "reward/rejected": -3.015625, "step": 2549 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 696.0, "epoch": 0.7648470305938813, "grad_norm": 4.130340576171875, "learning_rate": 1.3066362694843765e-06, "logp/chosen": -238.0, "logp/rejected": -264.0, "loss": 0.4219970703125, "reward/accuracy": 0.75, "reward/chosen": -1.4765625, "reward/margin": 1.6953125, "reward/rejected": -3.171875, "step": 2550 }, { "approx. KL/chosen": 145.0, "approx. KL/rejected": 644.0, "epoch": 0.7651469706058788, "grad_norm": 3.225148916244507, "learning_rate": 1.3034620919959957e-06, "logp/chosen": -276.0, "logp/rejected": -280.0, "loss": 0.391510009765625, "reward/accuracy": 0.875, "reward/chosen": -1.15625, "reward/margin": 1.8203125, "reward/rejected": -2.984375, "step": 2551 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 612.0, "epoch": 0.7654469106178764, "grad_norm": 5.179178237915039, "learning_rate": 1.300291196697377e-06, "logp/chosen": -314.0, "logp/rejected": -348.0, "loss": 0.378692626953125, "reward/accuracy": 0.8125, "reward/chosen": -1.515625, "reward/margin": 1.71875, "reward/rejected": -3.234375, "step": 2552 }, { "approx. KL/chosen": 464.0, "approx. KL/rejected": 444.0, "epoch": 0.765746850629874, "grad_norm": 7.957759380340576, "learning_rate": 1.2971235864039871e-06, "logp/chosen": -358.0, "logp/rejected": -356.0, "loss": 0.736328125, "reward/accuracy": 0.625, "reward/chosen": -1.8125, "reward/margin": 0.703125, "reward/rejected": -2.515625, "step": 2553 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 792.0, "epoch": 0.7660467906418716, "grad_norm": 2.962073564529419, "learning_rate": 1.2939592639283754e-06, "logp/chosen": -308.0, "logp/rejected": -354.0, "loss": 0.296783447265625, "reward/accuracy": 0.8125, "reward/chosen": -1.421875, "reward/margin": 1.9375, "reward/rejected": -3.359375, "step": 2554 }, { "approx. KL/chosen": 99.5, "approx. KL/rejected": 764.0, "epoch": 0.7663467306538693, "grad_norm": 2.618561267852783, "learning_rate": 1.2907982320801754e-06, "logp/chosen": -290.0, "logp/rejected": -382.0, "loss": 0.2353515625, "reward/accuracy": 0.9375, "reward/chosen": -1.0625, "reward/margin": 2.375, "reward/rejected": -3.453125, "step": 2555 }, { "approx. KL/chosen": 118.5, "approx. KL/rejected": 470.0, "epoch": 0.7666466706658668, "grad_norm": 3.2954843044281006, "learning_rate": 1.2876404936660936e-06, "logp/chosen": -282.0, "logp/rejected": -298.0, "loss": 0.42041015625, "reward/accuracy": 0.875, "reward/chosen": -1.2265625, "reward/margin": 1.3046875, "reward/rejected": -2.53125, "step": 2556 }, { "approx. KL/chosen": 238.0, "approx. KL/rejected": 740.0, "epoch": 0.7669466106778644, "grad_norm": 4.894970417022705, "learning_rate": 1.2844860514899182e-06, "logp/chosen": -195.0, "logp/rejected": -282.0, "loss": 0.5235595703125, "reward/accuracy": 0.8125, "reward/chosen": -1.6796875, "reward/margin": 1.6953125, "reward/rejected": -3.375, "step": 2557 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 908.0, "epoch": 0.767246550689862, "grad_norm": 6.886438846588135, "learning_rate": 1.2813349083525018e-06, "logp/chosen": -308.0, "logp/rejected": -378.0, "loss": 0.357818603515625, "reward/accuracy": 0.875, "reward/chosen": -1.5234375, "reward/margin": 2.28125, "reward/rejected": -3.796875, "step": 2558 }, { "approx. KL/chosen": 148.0, "approx. KL/rejected": 684.0, "epoch": 0.7675464907018597, "grad_norm": 5.1254377365112305, "learning_rate": 1.2781870670517777e-06, "logp/chosen": -179.0, "logp/rejected": -384.0, "loss": 0.5738296508789062, "reward/accuracy": 0.5625, "reward/chosen": -1.2421875, "reward/margin": 1.625, "reward/rejected": -2.875, "step": 2559 }, { "approx. KL/chosen": 506.0, "approx. KL/rejected": 784.0, "epoch": 0.7678464307138573, "grad_norm": 5.7271881103515625, "learning_rate": 1.27504253038274e-06, "logp/chosen": -420.0, "logp/rejected": -298.0, "loss": 0.5295944213867188, "reward/accuracy": 0.75, "reward/chosen": -2.28125, "reward/margin": 1.28125, "reward/rejected": -3.578125, "step": 2560 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 1248.0, "epoch": 0.7681463707258548, "grad_norm": 2.8794593811035156, "learning_rate": 1.2719013011374548e-06, "logp/chosen": -500.0, "logp/rejected": -636.0, "loss": 0.2263660430908203, "reward/accuracy": 0.9375, "reward/chosen": -1.515625, "reward/margin": 2.796875, "reward/rejected": -4.3125, "step": 2561 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 540.0, "epoch": 0.7684463107378524, "grad_norm": 4.620761871337891, "learning_rate": 1.268763382105046e-06, "logp/chosen": -266.0, "logp/rejected": -237.0, "loss": 0.588134765625, "reward/accuracy": 0.6875, "reward/chosen": -1.484375, "reward/margin": 1.265625, "reward/rejected": -2.75, "step": 2562 }, { "approx. KL/chosen": 146.0, "approx. KL/rejected": 644.0, "epoch": 0.76874625074985, "grad_norm": 3.863664388656616, "learning_rate": 1.2656287760717007e-06, "logp/chosen": -382.0, "logp/rejected": -458.0, "loss": 0.3272705078125, "reward/accuracy": 0.8125, "reward/chosen": -1.328125, "reward/margin": 1.796875, "reward/rejected": -3.125, "step": 2563 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 836.0, "epoch": 0.7690461907618477, "grad_norm": 4.046519756317139, "learning_rate": 1.262497485820663e-06, "logp/chosen": -348.0, "logp/rejected": -318.0, "loss": 0.3404541015625, "reward/accuracy": 0.875, "reward/chosen": -1.6328125, "reward/margin": 1.984375, "reward/rejected": -3.609375, "step": 2564 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 568.0, "epoch": 0.7693461307738453, "grad_norm": 5.1028008460998535, "learning_rate": 1.259369514132237e-06, "logp/chosen": -350.0, "logp/rejected": -316.0, "loss": 0.7930908203125, "reward/accuracy": 0.625, "reward/chosen": -1.75, "reward/margin": 0.75390625, "reward/rejected": -2.5, "step": 2565 }, { "approx. KL/chosen": 548.0, "approx. KL/rejected": 2032.0, "epoch": 0.7696460707858428, "grad_norm": 5.150111198425293, "learning_rate": 1.2562448637837765e-06, "logp/chosen": -450.0, "logp/rejected": -410.0, "loss": 0.3763306140899658, "reward/accuracy": 0.8125, "reward/chosen": -2.09375, "reward/margin": 3.40625, "reward/rejected": -5.5, "step": 2566 }, { "approx. KL/chosen": 162.0, "approx. KL/rejected": 588.0, "epoch": 0.7699460107978404, "grad_norm": 3.3051328659057617, "learning_rate": 1.253123537549686e-06, "logp/chosen": -280.0, "logp/rejected": -356.0, "loss": 0.38232421875, "reward/accuracy": 0.75, "reward/chosen": -1.3984375, "reward/margin": 1.5234375, "reward/rejected": -2.921875, "step": 2567 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 784.0, "epoch": 0.770245950809838, "grad_norm": 2.9941229820251465, "learning_rate": 1.2500055382014182e-06, "logp/chosen": -268.0, "logp/rejected": -356.0, "loss": 0.280059814453125, "reward/accuracy": 0.9375, "reward/chosen": -1.359375, "reward/margin": 2.046875, "reward/rejected": -3.421875, "step": 2568 }, { "approx. KL/chosen": 150.0, "approx. KL/rejected": 688.0, "epoch": 0.7705458908218357, "grad_norm": 3.024456024169922, "learning_rate": 1.2468908685074765e-06, "logp/chosen": -264.0, "logp/rejected": -204.0, "loss": 0.3796539306640625, "reward/accuracy": 0.875, "reward/chosen": -1.28125, "reward/margin": 2.046875, "reward/rejected": -3.328125, "step": 2569 }, { "approx. KL/chosen": 366.0, "approx. KL/rejected": 584.0, "epoch": 0.7708458308338332, "grad_norm": 6.525261878967285, "learning_rate": 1.2437795312334023e-06, "logp/chosen": -382.0, "logp/rejected": -278.0, "loss": 0.950775146484375, "reward/accuracy": 0.5625, "reward/chosen": -2.109375, "reward/margin": 0.439453125, "reward/rejected": -2.546875, "step": 2570 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 828.0, "epoch": 0.7711457708458308, "grad_norm": 4.056014537811279, "learning_rate": 1.2406715291417838e-06, "logp/chosen": -442.0, "logp/rejected": -442.0, "loss": 0.372314453125, "reward/accuracy": 0.8125, "reward/chosen": -1.3515625, "reward/margin": 2.015625, "reward/rejected": -3.375, "step": 2571 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 652.0, "epoch": 0.7714457108578284, "grad_norm": 3.3195064067840576, "learning_rate": 1.2375668649922384e-06, "logp/chosen": -360.0, "logp/rejected": -388.0, "loss": 0.35650634765625, "reward/accuracy": 0.8125, "reward/chosen": -1.546875, "reward/margin": 1.5859375, "reward/rejected": -3.125, "step": 2572 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 936.0, "epoch": 0.771745650869826, "grad_norm": 4.361300945281982, "learning_rate": 1.234465541541431e-06, "logp/chosen": -278.0, "logp/rejected": -332.0, "loss": 0.454132080078125, "reward/accuracy": 0.75, "reward/chosen": -1.8046875, "reward/margin": 1.828125, "reward/rejected": -3.640625, "step": 2573 }, { "approx. KL/chosen": 235.0, "approx. KL/rejected": 896.0, "epoch": 0.7720455908818237, "grad_norm": 2.745629072189331, "learning_rate": 1.2313675615430514e-06, "logp/chosen": -524.0, "logp/rejected": -412.0, "loss": 0.2058868408203125, "reward/accuracy": 0.9375, "reward/chosen": -1.7421875, "reward/margin": 2.296875, "reward/rejected": -4.03125, "step": 2574 }, { "approx. KL/chosen": 454.0, "approx. KL/rejected": 696.0, "epoch": 0.7723455308938212, "grad_norm": 6.648643970489502, "learning_rate": 1.2282729277478273e-06, "logp/chosen": -344.0, "logp/rejected": -322.0, "loss": 0.6297607421875, "reward/accuracy": 0.75, "reward/chosen": -2.09375, "reward/margin": 0.96875, "reward/rejected": -3.0625, "step": 2575 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 772.0, "epoch": 0.7726454709058188, "grad_norm": 3.3120131492614746, "learning_rate": 1.2251816429035112e-06, "logp/chosen": -298.0, "logp/rejected": -221.0, "loss": 0.362213134765625, "reward/accuracy": 0.8125, "reward/chosen": -1.3828125, "reward/margin": 2.03125, "reward/rejected": -3.421875, "step": 2576 }, { "approx. KL/chosen": 108.0, "approx. KL/rejected": 884.0, "epoch": 0.7729454109178164, "grad_norm": 2.9958112239837646, "learning_rate": 1.2220937097548835e-06, "logp/chosen": -458.0, "logp/rejected": -536.0, "loss": 0.315643310546875, "reward/accuracy": 0.8125, "reward/chosen": -1.25, "reward/margin": 2.4375, "reward/rejected": -3.6875, "step": 2577 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 768.0, "epoch": 0.7732453509298141, "grad_norm": 4.2159504890441895, "learning_rate": 1.2190091310437457e-06, "logp/chosen": -370.0, "logp/rejected": -352.0, "loss": 0.4352226257324219, "reward/accuracy": 0.8125, "reward/chosen": -1.203125, "reward/margin": 2.09375, "reward/rejected": -3.296875, "step": 2578 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 940.0, "epoch": 0.7735452909418117, "grad_norm": 4.834174156188965, "learning_rate": 1.2159279095089277e-06, "logp/chosen": -312.0, "logp/rejected": -366.0, "loss": 0.4608306884765625, "reward/accuracy": 0.75, "reward/chosen": -1.8125, "reward/margin": 2.234375, "reward/rejected": -4.0625, "step": 2579 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 516.0, "epoch": 0.7738452309538092, "grad_norm": 4.797194004058838, "learning_rate": 1.2128500478862716e-06, "logp/chosen": -322.0, "logp/rejected": -358.0, "loss": 0.6678466796875, "reward/accuracy": 0.5, "reward/chosen": -1.3359375, "reward/margin": 1.1484375, "reward/rejected": -2.484375, "step": 2580 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 1376.0, "epoch": 0.7741451709658068, "grad_norm": 2.269270896911621, "learning_rate": 1.2097755489086388e-06, "logp/chosen": -368.0, "logp/rejected": -322.0, "loss": 0.2576122283935547, "reward/accuracy": 0.8125, "reward/chosen": -1.4375, "reward/margin": 2.921875, "reward/rejected": -4.375, "step": 2581 }, { "approx. KL/chosen": 148.0, "approx. KL/rejected": 640.0, "epoch": 0.7744451109778044, "grad_norm": 3.402961015701294, "learning_rate": 1.2067044153059043e-06, "logp/chosen": -430.0, "logp/rejected": -416.0, "loss": 0.38720703125, "reward/accuracy": 0.75, "reward/chosen": -1.4296875, "reward/margin": 1.734375, "reward/rejected": -3.171875, "step": 2582 }, { "approx. KL/chosen": 177.0, "approx. KL/rejected": 548.0, "epoch": 0.7747450509898021, "grad_norm": 4.104834079742432, "learning_rate": 1.2036366498049583e-06, "logp/chosen": -320.0, "logp/rejected": -278.0, "loss": 0.5006103515625, "reward/accuracy": 0.8125, "reward/chosen": -1.3671875, "reward/margin": 1.5, "reward/rejected": -2.875, "step": 2583 }, { "approx. KL/chosen": 294.0, "approx. KL/rejected": 908.0, "epoch": 0.7750449910017997, "grad_norm": 3.812187671661377, "learning_rate": 1.200572255129696e-06, "logp/chosen": -356.0, "logp/rejected": -374.0, "loss": 0.387451171875, "reward/accuracy": 0.6875, "reward/chosen": -1.7421875, "reward/margin": 1.7265625, "reward/rejected": -3.46875, "step": 2584 }, { "approx. KL/chosen": 121.0, "approx. KL/rejected": 644.0, "epoch": 0.7753449310137972, "grad_norm": 4.037410736083984, "learning_rate": 1.197511234001021e-06, "logp/chosen": -312.0, "logp/rejected": -340.0, "loss": 0.4292449951171875, "reward/accuracy": 0.75, "reward/chosen": -1.2890625, "reward/margin": 1.703125, "reward/rejected": -3.0, "step": 2585 }, { "approx. KL/chosen": 245.0, "approx. KL/rejected": 430.0, "epoch": 0.7756448710257948, "grad_norm": 4.1776885986328125, "learning_rate": 1.194453589136843e-06, "logp/chosen": -326.0, "logp/rejected": -280.0, "loss": 0.5137939453125, "reward/accuracy": 0.8125, "reward/chosen": -1.390625, "reward/margin": 1.125, "reward/rejected": -2.515625, "step": 2586 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 576.0, "epoch": 0.7759448110377924, "grad_norm": 3.2406129837036133, "learning_rate": 1.1913993232520731e-06, "logp/chosen": -192.0, "logp/rejected": -196.0, "loss": 0.380615234375, "reward/accuracy": 0.75, "reward/chosen": -1.234375, "reward/margin": 1.6171875, "reward/rejected": -2.84375, "step": 2587 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 552.0, "epoch": 0.7762447510497901, "grad_norm": 5.019537925720215, "learning_rate": 1.1883484390586197e-06, "logp/chosen": -314.0, "logp/rejected": -316.0, "loss": 0.63671875, "reward/accuracy": 0.625, "reward/chosen": -1.796875, "reward/margin": 0.75, "reward/rejected": -2.546875, "step": 2588 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 1280.0, "epoch": 0.7765446910617877, "grad_norm": 2.627580165863037, "learning_rate": 1.1853009392653937e-06, "logp/chosen": -296.0, "logp/rejected": -344.0, "loss": 0.1732940673828125, "reward/accuracy": 0.9375, "reward/chosen": -1.2890625, "reward/margin": 3.265625, "reward/rejected": -4.53125, "step": 2589 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 948.0, "epoch": 0.7768446310737852, "grad_norm": 4.488485813140869, "learning_rate": 1.1822568265782963e-06, "logp/chosen": -253.0, "logp/rejected": -300.0, "loss": 0.48384857177734375, "reward/accuracy": 0.75, "reward/chosen": -1.3203125, "reward/margin": 2.5, "reward/rejected": -3.828125, "step": 2590 }, { "approx. KL/chosen": 104.5, "approx. KL/rejected": 644.0, "epoch": 0.7771445710857828, "grad_norm": 3.9666223526000977, "learning_rate": 1.1792161037002237e-06, "logp/chosen": -374.0, "logp/rejected": -520.0, "loss": 0.4534454345703125, "reward/accuracy": 0.75, "reward/chosen": -1.203125, "reward/margin": 1.6953125, "reward/rejected": -2.90625, "step": 2591 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 1048.0, "epoch": 0.7774445110977805, "grad_norm": 2.6375532150268555, "learning_rate": 1.1761787733310593e-06, "logp/chosen": -322.0, "logp/rejected": -340.0, "loss": 0.2668609619140625, "reward/accuracy": 0.875, "reward/chosen": -1.234375, "reward/margin": 2.828125, "reward/rejected": -4.0625, "step": 2592 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 812.0, "epoch": 0.7777444511097781, "grad_norm": 3.1458120346069336, "learning_rate": 1.1731448381676791e-06, "logp/chosen": -252.0, "logp/rejected": -292.0, "loss": 0.24404144287109375, "reward/accuracy": 0.9375, "reward/chosen": -1.25, "reward/margin": 2.4375, "reward/rejected": -3.6875, "step": 2593 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 516.0, "epoch": 0.7780443911217756, "grad_norm": 5.469702243804932, "learning_rate": 1.170114300903941e-06, "logp/chosen": -256.0, "logp/rejected": -340.0, "loss": 0.6158447265625, "reward/accuracy": 0.75, "reward/chosen": -1.4296875, "reward/margin": 1.2734375, "reward/rejected": -2.703125, "step": 2594 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 612.0, "epoch": 0.7783443311337732, "grad_norm": 4.450825214385986, "learning_rate": 1.167087164230687e-06, "logp/chosen": -492.0, "logp/rejected": -366.0, "loss": 0.470001220703125, "reward/accuracy": 0.75, "reward/chosen": -1.296875, "reward/margin": 1.59375, "reward/rejected": -2.890625, "step": 2595 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 628.0, "epoch": 0.7786442711457708, "grad_norm": 4.921121597290039, "learning_rate": 1.1640634308357375e-06, "logp/chosen": -512.0, "logp/rejected": -378.0, "loss": 0.3782958984375, "reward/accuracy": 0.75, "reward/chosen": -0.89453125, "reward/margin": 2.046875, "reward/rejected": -2.9375, "step": 2596 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 1096.0, "epoch": 0.7789442111577685, "grad_norm": 2.5462706089019775, "learning_rate": 1.161043103403896e-06, "logp/chosen": -370.0, "logp/rejected": -406.0, "loss": 0.2338714599609375, "reward/accuracy": 0.9375, "reward/chosen": -1.25, "reward/margin": 2.625, "reward/rejected": -3.875, "step": 2597 }, { "approx. KL/chosen": 168.0, "approx. KL/rejected": 516.0, "epoch": 0.7792441511697661, "grad_norm": 3.8902297019958496, "learning_rate": 1.1580261846169372e-06, "logp/chosen": -294.0, "logp/rejected": -286.0, "loss": 0.5069580078125, "reward/accuracy": 0.75, "reward/chosen": -1.28125, "reward/margin": 1.6015625, "reward/rejected": -2.890625, "step": 2598 }, { "approx. KL/chosen": 376.0, "approx. KL/rejected": 720.0, "epoch": 0.7795440911817636, "grad_norm": 5.155867576599121, "learning_rate": 1.155012677153609e-06, "logp/chosen": -332.0, "logp/rejected": -344.0, "loss": 0.6273193359375, "reward/accuracy": 0.625, "reward/chosen": -2.171875, "reward/margin": 0.93359375, "reward/rejected": -3.09375, "step": 2599 }, { "approx. KL/chosen": 243.0, "approx. KL/rejected": 628.0, "epoch": 0.7798440311937612, "grad_norm": 4.1136040687561035, "learning_rate": 1.1520025836896365e-06, "logp/chosen": -254.0, "logp/rejected": -322.0, "loss": 0.4080810546875, "reward/accuracy": 0.8125, "reward/chosen": -1.5625, "reward/margin": 1.3515625, "reward/rejected": -2.921875, "step": 2600 }, { "approx. KL/chosen": 137.0, "approx. KL/rejected": 620.0, "epoch": 0.7801439712057588, "grad_norm": 2.904374837875366, "learning_rate": 1.1489959068977057e-06, "logp/chosen": -330.0, "logp/rejected": -368.0, "loss": 0.35662841796875, "reward/accuracy": 0.8125, "reward/chosen": -1.3359375, "reward/margin": 1.5390625, "reward/rejected": -2.875, "step": 2601 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 608.0, "epoch": 0.7804439112177565, "grad_norm": 4.999896049499512, "learning_rate": 1.1459926494474744e-06, "logp/chosen": -298.0, "logp/rejected": -282.0, "loss": 0.7076263427734375, "reward/accuracy": 0.875, "reward/chosen": -1.7734375, "reward/margin": 1.28125, "reward/rejected": -3.046875, "step": 2602 }, { "approx. KL/chosen": 132.0, "approx. KL/rejected": 384.0, "epoch": 0.7807438512297541, "grad_norm": 4.4282402992248535, "learning_rate": 1.1429928140055596e-06, "logp/chosen": -280.0, "logp/rejected": -306.0, "loss": 0.6015625, "reward/accuracy": 0.6875, "reward/chosen": -1.15625, "reward/margin": 1.0625, "reward/rejected": -2.21875, "step": 2603 }, { "approx. KL/chosen": 432.0, "approx. KL/rejected": 1080.0, "epoch": 0.7810437912417516, "grad_norm": 5.135715007781982, "learning_rate": 1.1399964032355453e-06, "logp/chosen": -440.0, "logp/rejected": -342.0, "loss": 0.5477142333984375, "reward/accuracy": 0.8125, "reward/chosen": -2.078125, "reward/margin": 1.9453125, "reward/rejected": -4.0, "step": 2604 }, { "approx. KL/chosen": 420.0, "approx. KL/rejected": 720.0, "epoch": 0.7813437312537492, "grad_norm": 6.836088180541992, "learning_rate": 1.1370034197979718e-06, "logp/chosen": -328.0, "logp/rejected": -374.0, "loss": 0.53125, "reward/accuracy": 0.8125, "reward/chosen": -2.296875, "reward/margin": 1.1640625, "reward/rejected": -3.46875, "step": 2605 }, { "approx. KL/chosen": 344.0, "approx. KL/rejected": 716.0, "epoch": 0.7816436712657469, "grad_norm": 4.466447830200195, "learning_rate": 1.134013866350334e-06, "logp/chosen": -258.0, "logp/rejected": -252.0, "loss": 0.53485107421875, "reward/accuracy": 0.875, "reward/chosen": -2.125, "reward/margin": 1.3828125, "reward/rejected": -3.5, "step": 2606 }, { "approx. KL/chosen": 240.0, "approx. KL/rejected": 864.0, "epoch": 0.7819436112777445, "grad_norm": 3.293266534805298, "learning_rate": 1.131027745547087e-06, "logp/chosen": -228.0, "logp/rejected": -358.0, "loss": 0.4051513671875, "reward/accuracy": 0.6875, "reward/chosen": -1.609375, "reward/margin": 1.8046875, "reward/rejected": -3.421875, "step": 2607 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 1040.0, "epoch": 0.7822435512897421, "grad_norm": 3.2877705097198486, "learning_rate": 1.1280450600396325e-06, "logp/chosen": -356.0, "logp/rejected": -354.0, "loss": 0.18682861328125, "reward/accuracy": 0.9375, "reward/chosen": -1.1640625, "reward/margin": 2.828125, "reward/rejected": -4.0, "step": 2608 }, { "approx. KL/chosen": 350.0, "approx. KL/rejected": 456.0, "epoch": 0.7825434913017396, "grad_norm": 7.049473285675049, "learning_rate": 1.1250658124763242e-06, "logp/chosen": -498.0, "logp/rejected": -430.0, "loss": 0.5411376953125, "reward/accuracy": 0.6875, "reward/chosen": -1.5859375, "reward/margin": 0.78515625, "reward/rejected": -2.375, "step": 2609 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 426.0, "epoch": 0.7828434313137372, "grad_norm": 6.9772257804870605, "learning_rate": 1.1220900055024647e-06, "logp/chosen": -354.0, "logp/rejected": -386.0, "loss": 0.80322265625, "reward/accuracy": 0.6875, "reward/chosen": -1.890625, "reward/margin": 0.62109375, "reward/rejected": -2.515625, "step": 2610 }, { "approx. KL/chosen": 234.0, "approx. KL/rejected": 564.0, "epoch": 0.7831433713257349, "grad_norm": 4.115499496459961, "learning_rate": 1.1191176417602995e-06, "logp/chosen": -424.0, "logp/rejected": -376.0, "loss": 0.4820709228515625, "reward/accuracy": 0.6875, "reward/chosen": -1.5234375, "reward/margin": 1.296875, "reward/rejected": -2.828125, "step": 2611 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 904.0, "epoch": 0.7834433113377325, "grad_norm": 5.622288703918457, "learning_rate": 1.1161487238890168e-06, "logp/chosen": -348.0, "logp/rejected": -364.0, "loss": 0.6524162292480469, "reward/accuracy": 0.75, "reward/chosen": -2.015625, "reward/margin": 1.734375, "reward/rejected": -3.75, "step": 2612 }, { "approx. KL/chosen": 137.0, "approx. KL/rejected": 664.0, "epoch": 0.7837432513497301, "grad_norm": 2.7601749897003174, "learning_rate": 1.1131832545247456e-06, "logp/chosen": -260.0, "logp/rejected": -310.0, "loss": 0.23138427734375, "reward/accuracy": 0.9375, "reward/chosen": -1.234375, "reward/margin": 2.171875, "reward/rejected": -3.40625, "step": 2613 }, { "approx. KL/chosen": 62.25, "approx. KL/rejected": 612.0, "epoch": 0.7840431913617276, "grad_norm": 2.438464641571045, "learning_rate": 1.110221236300555e-06, "logp/chosen": -404.0, "logp/rejected": -294.0, "loss": 0.2330322265625, "reward/accuracy": 0.9375, "reward/chosen": -0.72265625, "reward/margin": 2.21875, "reward/rejected": -2.9375, "step": 2614 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 712.0, "epoch": 0.7843431313737252, "grad_norm": 3.1139705181121826, "learning_rate": 1.1072626718464474e-06, "logp/chosen": -330.0, "logp/rejected": -450.0, "loss": 0.328369140625, "reward/accuracy": 0.8125, "reward/chosen": -1.3828125, "reward/margin": 1.7734375, "reward/rejected": -3.15625, "step": 2615 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 816.0, "epoch": 0.7846430713857229, "grad_norm": 3.703336477279663, "learning_rate": 1.1043075637893587e-06, "logp/chosen": -234.0, "logp/rejected": -316.0, "loss": 0.48272705078125, "reward/accuracy": 0.6875, "reward/chosen": -1.8671875, "reward/margin": 1.4375, "reward/rejected": -3.296875, "step": 2616 }, { "approx. KL/chosen": 468.0, "approx. KL/rejected": 612.0, "epoch": 0.7849430113977205, "grad_norm": 5.161707401275635, "learning_rate": 1.101355914753155e-06, "logp/chosen": -378.0, "logp/rejected": -368.0, "loss": 0.5784912109375, "reward/accuracy": 0.75, "reward/chosen": -2.40625, "reward/margin": 0.7890625, "reward/rejected": -3.1875, "step": 2617 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 840.0, "epoch": 0.785242951409718, "grad_norm": 3.56712007522583, "learning_rate": 1.0984077273586358e-06, "logp/chosen": -304.0, "logp/rejected": -342.0, "loss": 0.454864501953125, "reward/accuracy": 0.75, "reward/chosen": -1.8984375, "reward/margin": 1.5078125, "reward/rejected": -3.40625, "step": 2618 }, { "approx. KL/chosen": 221.0, "approx. KL/rejected": 672.0, "epoch": 0.7855428914217156, "grad_norm": 3.6295816898345947, "learning_rate": 1.095463004223521e-06, "logp/chosen": -228.0, "logp/rejected": -328.0, "loss": 0.4190673828125, "reward/accuracy": 0.6875, "reward/chosen": -1.484375, "reward/margin": 1.359375, "reward/rejected": -2.84375, "step": 2619 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 764.0, "epoch": 0.7858428314337133, "grad_norm": 2.526435613632202, "learning_rate": 1.0925217479624583e-06, "logp/chosen": -460.0, "logp/rejected": -356.0, "loss": 0.21051025390625, "reward/accuracy": 0.9375, "reward/chosen": -1.390625, "reward/margin": 2.125, "reward/rejected": -3.5, "step": 2620 }, { "approx. KL/chosen": 206.0, "approx. KL/rejected": 828.0, "epoch": 0.7861427714457109, "grad_norm": 3.549398422241211, "learning_rate": 1.0895839611870141e-06, "logp/chosen": -304.0, "logp/rejected": -336.0, "loss": 0.3572959899902344, "reward/accuracy": 0.875, "reward/chosen": -1.6015625, "reward/margin": 1.9296875, "reward/rejected": -3.53125, "step": 2621 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 844.0, "epoch": 0.7864427114577085, "grad_norm": 4.922927379608154, "learning_rate": 1.086649646505679e-06, "logp/chosen": -410.0, "logp/rejected": -376.0, "loss": 0.3159637451171875, "reward/accuracy": 0.8125, "reward/chosen": -1.7890625, "reward/margin": 1.765625, "reward/rejected": -3.546875, "step": 2622 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 366.0, "epoch": 0.786742651469706, "grad_norm": 4.06554651260376, "learning_rate": 1.0837188065238546e-06, "logp/chosen": -384.0, "logp/rejected": -384.0, "loss": 0.5367431640625, "reward/accuracy": 0.75, "reward/chosen": -1.484375, "reward/margin": 0.84765625, "reward/rejected": -2.328125, "step": 2623 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 552.0, "epoch": 0.7870425914817036, "grad_norm": 4.0127692222595215, "learning_rate": 1.0807914438438643e-06, "logp/chosen": -428.0, "logp/rejected": -548.0, "loss": 0.4541015625, "reward/accuracy": 0.625, "reward/chosen": -1.28125, "reward/margin": 1.421875, "reward/rejected": -2.703125, "step": 2624 }, { "approx. KL/chosen": 209.0, "approx. KL/rejected": 1004.0, "epoch": 0.7873425314937013, "grad_norm": 3.4839060306549072, "learning_rate": 1.0778675610649374e-06, "logp/chosen": -276.0, "logp/rejected": -374.0, "loss": 0.380218505859375, "reward/accuracy": 0.6875, "reward/chosen": -1.5, "reward/margin": 1.9765625, "reward/rejected": -3.46875, "step": 2625 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 932.0, "epoch": 0.7876424715056989, "grad_norm": 7.276942729949951, "learning_rate": 1.0749471607832173e-06, "logp/chosen": -316.0, "logp/rejected": -400.0, "loss": 0.7305755615234375, "reward/accuracy": 0.625, "reward/chosen": -1.734375, "reward/margin": 1.921875, "reward/rejected": -3.65625, "step": 2626 }, { "approx. KL/chosen": 205.0, "approx. KL/rejected": 632.0, "epoch": 0.7879424115176965, "grad_norm": 3.0205376148223877, "learning_rate": 1.072030245591752e-06, "logp/chosen": -390.0, "logp/rejected": -316.0, "loss": 0.38238525390625, "reward/accuracy": 0.8125, "reward/chosen": -1.5078125, "reward/margin": 1.5234375, "reward/rejected": -3.03125, "step": 2627 }, { "approx. KL/chosen": 308.0, "approx. KL/rejected": 808.0, "epoch": 0.788242351529694, "grad_norm": 3.6585497856140137, "learning_rate": 1.0691168180805006e-06, "logp/chosen": -230.0, "logp/rejected": -286.0, "loss": 0.310302734375, "reward/accuracy": 0.875, "reward/chosen": -1.6640625, "reward/margin": 1.875, "reward/rejected": -3.53125, "step": 2628 }, { "approx. KL/chosen": 198.0, "approx. KL/rejected": 560.0, "epoch": 0.7885422915416916, "grad_norm": 3.626523733139038, "learning_rate": 1.0662068808363197e-06, "logp/chosen": -362.0, "logp/rejected": -356.0, "loss": 0.472412109375, "reward/accuracy": 0.6875, "reward/chosen": -1.28125, "reward/margin": 1.4765625, "reward/rejected": -2.765625, "step": 2629 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 692.0, "epoch": 0.7888422315536893, "grad_norm": 3.7319419384002686, "learning_rate": 1.0633004364429694e-06, "logp/chosen": -374.0, "logp/rejected": -350.0, "loss": 0.5535888671875, "reward/accuracy": 0.8125, "reward/chosen": -1.140625, "reward/margin": 2.015625, "reward/rejected": -3.15625, "step": 2630 }, { "approx. KL/chosen": 167.0, "approx. KL/rejected": 1080.0, "epoch": 0.7891421715656869, "grad_norm": 2.7570340633392334, "learning_rate": 1.0603974874811074e-06, "logp/chosen": -342.0, "logp/rejected": -432.0, "loss": 0.25839996337890625, "reward/accuracy": 0.875, "reward/chosen": -1.34375, "reward/margin": 2.765625, "reward/rejected": -4.125, "step": 2631 }, { "approx. KL/chosen": 112.0, "approx. KL/rejected": 576.0, "epoch": 0.7894421115776845, "grad_norm": 3.848653554916382, "learning_rate": 1.0574980365282905e-06, "logp/chosen": -370.0, "logp/rejected": -356.0, "loss": 0.27032470703125, "reward/accuracy": 0.9375, "reward/chosen": -0.75390625, "reward/margin": 2.09375, "reward/rejected": -2.84375, "step": 2632 }, { "approx. KL/chosen": 171.0, "approx. KL/rejected": 728.0, "epoch": 0.789742051589682, "grad_norm": 4.406620025634766, "learning_rate": 1.0546020861589656e-06, "logp/chosen": -436.0, "logp/rejected": -410.0, "loss": 0.3671417236328125, "reward/accuracy": 0.8125, "reward/chosen": -1.515625, "reward/margin": 1.84375, "reward/rejected": -3.359375, "step": 2633 }, { "approx. KL/chosen": 406.0, "approx. KL/rejected": 952.0, "epoch": 0.7900419916016796, "grad_norm": 4.310000896453857, "learning_rate": 1.0517096389444776e-06, "logp/chosen": -219.0, "logp/rejected": -292.0, "loss": 0.74359130859375, "reward/accuracy": 0.625, "reward/chosen": -2.34375, "reward/margin": 1.21875, "reward/rejected": -3.5625, "step": 2634 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 660.0, "epoch": 0.7903419316136773, "grad_norm": 3.4904282093048096, "learning_rate": 1.0488206974530513e-06, "logp/chosen": -330.0, "logp/rejected": -318.0, "loss": 0.415191650390625, "reward/accuracy": 0.75, "reward/chosen": -1.4609375, "reward/margin": 1.421875, "reward/rejected": -2.890625, "step": 2635 }, { "approx. KL/chosen": 106.0, "approx. KL/rejected": 700.0, "epoch": 0.7906418716256749, "grad_norm": 3.86879563331604, "learning_rate": 1.0459352642498078e-06, "logp/chosen": -342.0, "logp/rejected": -358.0, "loss": 0.5305023193359375, "reward/accuracy": 0.625, "reward/chosen": -1.0546875, "reward/margin": 1.7734375, "reward/rejected": -2.828125, "step": 2636 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 1200.0, "epoch": 0.7909418116376725, "grad_norm": 3.4748268127441406, "learning_rate": 1.043053341896747e-06, "logp/chosen": -330.0, "logp/rejected": -504.0, "loss": 0.4081001281738281, "reward/accuracy": 0.8125, "reward/chosen": -1.5546875, "reward/margin": 2.34375, "reward/rejected": -3.890625, "step": 2637 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 430.0, "epoch": 0.79124175164967, "grad_norm": 4.719532012939453, "learning_rate": 1.0401749329527572e-06, "logp/chosen": -240.0, "logp/rejected": -245.0, "loss": 0.6683349609375, "reward/accuracy": 0.6875, "reward/chosen": -1.4921875, "reward/margin": 0.85546875, "reward/rejected": -2.34375, "step": 2638 }, { "approx. KL/chosen": 147.0, "approx. KL/rejected": 692.0, "epoch": 0.7915416916616677, "grad_norm": 2.7479004859924316, "learning_rate": 1.0373000399736022e-06, "logp/chosen": -302.0, "logp/rejected": -298.0, "loss": 0.231414794921875, "reward/accuracy": 0.9375, "reward/chosen": -0.90625, "reward/margin": 2.265625, "reward/rejected": -3.171875, "step": 2639 }, { "approx. KL/chosen": 111.5, "approx. KL/rejected": 788.0, "epoch": 0.7918416316736653, "grad_norm": 3.382598400115967, "learning_rate": 1.0344286655119257e-06, "logp/chosen": -284.0, "logp/rejected": -334.0, "loss": 0.36766815185546875, "reward/accuracy": 0.8125, "reward/chosen": -1.1484375, "reward/margin": 2.1875, "reward/rejected": -3.34375, "step": 2640 }, { "approx. KL/chosen": 218.0, "approx. KL/rejected": 920.0, "epoch": 0.7921415716856629, "grad_norm": 4.531986713409424, "learning_rate": 1.0315608121172467e-06, "logp/chosen": -320.0, "logp/rejected": -356.0, "loss": 0.47537994384765625, "reward/accuracy": 0.75, "reward/chosen": -1.4921875, "reward/margin": 2.359375, "reward/rejected": -3.84375, "step": 2641 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 740.0, "epoch": 0.7924415116976604, "grad_norm": 2.5933518409729004, "learning_rate": 1.02869648233596e-06, "logp/chosen": -330.0, "logp/rejected": -340.0, "loss": 0.29248046875, "reward/accuracy": 0.875, "reward/chosen": -1.1171875, "reward/margin": 2.078125, "reward/rejected": -3.1875, "step": 2642 }, { "approx. KL/chosen": 318.0, "approx. KL/rejected": 748.0, "epoch": 0.792741451709658, "grad_norm": 2.819336175918579, "learning_rate": 1.0258356787113305e-06, "logp/chosen": -490.0, "logp/rejected": -428.0, "loss": 0.24774169921875, "reward/accuracy": 0.875, "reward/chosen": -1.5703125, "reward/margin": 1.7734375, "reward/rejected": -3.34375, "step": 2643 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 716.0, "epoch": 0.7930413917216557, "grad_norm": 3.403033971786499, "learning_rate": 1.02297840378349e-06, "logp/chosen": -308.0, "logp/rejected": -282.0, "loss": 0.46466064453125, "reward/accuracy": 0.75, "reward/chosen": -1.328125, "reward/margin": 1.6796875, "reward/rejected": -3.015625, "step": 2644 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 876.0, "epoch": 0.7933413317336533, "grad_norm": 4.254542827606201, "learning_rate": 1.0201246600894387e-06, "logp/chosen": -264.0, "logp/rejected": -282.0, "loss": 0.359588623046875, "reward/accuracy": 0.8125, "reward/chosen": -1.6796875, "reward/margin": 1.8984375, "reward/rejected": -3.578125, "step": 2645 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 512.0, "epoch": 0.7936412717456509, "grad_norm": 4.003802299499512, "learning_rate": 1.0172744501630437e-06, "logp/chosen": -362.0, "logp/rejected": -247.0, "loss": 0.509521484375, "reward/accuracy": 0.75, "reward/chosen": -1.5703125, "reward/margin": 1.0390625, "reward/rejected": -2.609375, "step": 2646 }, { "approx. KL/chosen": 101.0, "approx. KL/rejected": 976.0, "epoch": 0.7939412117576484, "grad_norm": 3.0738415718078613, "learning_rate": 1.0144277765350302e-06, "logp/chosen": -370.0, "logp/rejected": -338.0, "loss": 0.333770751953125, "reward/accuracy": 0.8125, "reward/chosen": -0.98046875, "reward/margin": 2.46875, "reward/rejected": -3.453125, "step": 2647 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 608.0, "epoch": 0.794241151769646, "grad_norm": 5.429752349853516, "learning_rate": 1.01158464173299e-06, "logp/chosen": -284.0, "logp/rejected": -376.0, "loss": 0.59197998046875, "reward/accuracy": 0.6875, "reward/chosen": -1.953125, "reward/margin": 1.1171875, "reward/rejected": -3.078125, "step": 2648 }, { "approx. KL/chosen": 392.0, "approx. KL/rejected": 748.0, "epoch": 0.7945410917816437, "grad_norm": 6.004645347595215, "learning_rate": 1.0087450482813627e-06, "logp/chosen": -266.0, "logp/rejected": -324.0, "loss": 0.7317047119140625, "reward/accuracy": 0.8125, "reward/chosen": -2.125, "reward/margin": 1.3046875, "reward/rejected": -3.421875, "step": 2649 }, { "approx. KL/chosen": 113.0, "approx. KL/rejected": 592.0, "epoch": 0.7948410317936413, "grad_norm": 3.097008466720581, "learning_rate": 1.0059089987014536e-06, "logp/chosen": -258.0, "logp/rejected": -274.0, "loss": 0.32354736328125, "reward/accuracy": 0.8125, "reward/chosen": -1.1171875, "reward/margin": 1.75, "reward/rejected": -2.875, "step": 2650 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 652.0, "epoch": 0.7951409718056389, "grad_norm": 4.72925329208374, "learning_rate": 1.0030764955114137e-06, "logp/chosen": -356.0, "logp/rejected": -428.0, "loss": 0.66387939453125, "reward/accuracy": 0.75, "reward/chosen": -1.640625, "reward/margin": 1.5078125, "reward/rejected": -3.15625, "step": 2651 }, { "approx. KL/chosen": 115.0, "approx. KL/rejected": 600.0, "epoch": 0.7954409118176364, "grad_norm": 2.96506929397583, "learning_rate": 1.0002475412262513e-06, "logp/chosen": -298.0, "logp/rejected": -288.0, "loss": 0.392486572265625, "reward/accuracy": 0.8125, "reward/chosen": -1.140625, "reward/margin": 1.953125, "reward/rejected": -3.09375, "step": 2652 }, { "approx. KL/chosen": 392.0, "approx. KL/rejected": 944.0, "epoch": 0.795740851829634, "grad_norm": 5.753606796264648, "learning_rate": 9.974221383578198e-07, "logp/chosen": -404.0, "logp/rejected": -336.0, "loss": 0.5047760009765625, "reward/accuracy": 0.75, "reward/chosen": -2.171875, "reward/margin": 1.640625, "reward/rejected": -3.8125, "step": 2653 }, { "approx. KL/chosen": 52.5, "approx. KL/rejected": 432.0, "epoch": 0.7960407918416317, "grad_norm": 3.159065008163452, "learning_rate": 9.946002894148198e-07, "logp/chosen": -270.0, "logp/rejected": -260.0, "loss": 0.3701171875, "reward/accuracy": 0.9375, "reward/chosen": -0.8046875, "reward/margin": 1.515625, "reward/rejected": -2.328125, "step": 2654 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 712.0, "epoch": 0.7963407318536293, "grad_norm": 4.9184956550598145, "learning_rate": 9.917819969027964e-07, "logp/chosen": -410.0, "logp/rejected": -402.0, "loss": 0.658233642578125, "reward/accuracy": 0.6875, "reward/chosen": -1.9140625, "reward/margin": 1.2890625, "reward/rejected": -3.203125, "step": 2655 }, { "approx. KL/chosen": 217.0, "approx. KL/rejected": 672.0, "epoch": 0.7966406718656269, "grad_norm": 3.3737952709198, "learning_rate": 9.889672633241387e-07, "logp/chosen": -274.0, "logp/rejected": -340.0, "loss": 0.37261962890625, "reward/accuracy": 0.8125, "reward/chosen": -1.7109375, "reward/margin": 1.734375, "reward/rejected": -3.453125, "step": 2656 }, { "approx. KL/chosen": 396.0, "approx. KL/rejected": 498.0, "epoch": 0.7969406118776244, "grad_norm": 5.517680644989014, "learning_rate": 9.861560911780716e-07, "logp/chosen": -324.0, "logp/rejected": -290.0, "loss": 0.680419921875, "reward/accuracy": 0.75, "reward/chosen": -1.9140625, "reward/margin": 0.8359375, "reward/rejected": -2.75, "step": 2657 }, { "approx. KL/chosen": 219.0, "approx. KL/rejected": 492.0, "epoch": 0.7972405518896221, "grad_norm": 4.507782936096191, "learning_rate": 9.833484829606655e-07, "logp/chosen": -352.0, "logp/rejected": -356.0, "loss": 0.457733154296875, "reward/accuracy": 0.625, "reward/chosen": -1.5625, "reward/margin": 1.3046875, "reward/rejected": -2.859375, "step": 2658 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 520.0, "epoch": 0.7975404919016197, "grad_norm": 2.7681312561035156, "learning_rate": 9.805444411648158e-07, "logp/chosen": -348.0, "logp/rejected": -262.0, "loss": 0.324005126953125, "reward/accuracy": 0.9375, "reward/chosen": -1.1640625, "reward/margin": 1.7421875, "reward/rejected": -2.90625, "step": 2659 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 416.0, "epoch": 0.7978404319136173, "grad_norm": 5.595311641693115, "learning_rate": 9.777439682802603e-07, "logp/chosen": -284.0, "logp/rejected": -249.0, "loss": 0.87939453125, "reward/accuracy": 0.625, "reward/chosen": -1.28125, "reward/margin": 0.75390625, "reward/rejected": -2.03125, "step": 2660 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 704.0, "epoch": 0.7981403719256148, "grad_norm": 2.6861472129821777, "learning_rate": 9.749470667935623e-07, "logp/chosen": -326.0, "logp/rejected": -314.0, "loss": 0.304229736328125, "reward/accuracy": 0.75, "reward/chosen": -1.3671875, "reward/margin": 1.9609375, "reward/rejected": -3.328125, "step": 2661 }, { "approx. KL/chosen": 102.0, "approx. KL/rejected": 536.0, "epoch": 0.7984403119376124, "grad_norm": 3.7349629402160645, "learning_rate": 9.721537391881197e-07, "logp/chosen": -316.0, "logp/rejected": -330.0, "loss": 0.58001708984375, "reward/accuracy": 0.6875, "reward/chosen": -1.25, "reward/margin": 1.2890625, "reward/rejected": -2.53125, "step": 2662 }, { "approx. KL/chosen": 234.0, "approx. KL/rejected": 820.0, "epoch": 0.7987402519496101, "grad_norm": 3.823045253753662, "learning_rate": 9.693639879441486e-07, "logp/chosen": -340.0, "logp/rejected": -332.0, "loss": 0.484130859375, "reward/accuracy": 0.75, "reward/chosen": -1.5, "reward/margin": 1.6953125, "reward/rejected": -3.203125, "step": 2663 }, { "approx. KL/chosen": 101.5, "approx. KL/rejected": 508.0, "epoch": 0.7990401919616077, "grad_norm": 3.2038497924804688, "learning_rate": 9.665778155386984e-07, "logp/chosen": -350.0, "logp/rejected": -360.0, "loss": 0.407989501953125, "reward/accuracy": 0.75, "reward/chosen": -1.0625, "reward/margin": 1.5390625, "reward/rejected": -2.59375, "step": 2664 }, { "approx. KL/chosen": 221.0, "approx. KL/rejected": 856.0, "epoch": 0.7993401319736053, "grad_norm": 5.465497970581055, "learning_rate": 9.637952244456344e-07, "logp/chosen": -342.0, "logp/rejected": -400.0, "loss": 0.58856201171875, "reward/accuracy": 0.6875, "reward/chosen": -1.5859375, "reward/margin": 1.78125, "reward/rejected": -3.375, "step": 2665 }, { "approx. KL/chosen": 80.5, "approx. KL/rejected": 728.0, "epoch": 0.7996400719856028, "grad_norm": 2.0181455612182617, "learning_rate": 9.610162171356475e-07, "logp/chosen": -247.0, "logp/rejected": -336.0, "loss": 0.28743743896484375, "reward/accuracy": 0.875, "reward/chosen": -1.1328125, "reward/margin": 2.234375, "reward/rejected": -3.359375, "step": 2666 }, { "approx. KL/chosen": 362.0, "approx. KL/rejected": 1064.0, "epoch": 0.7999400119976005, "grad_norm": 5.218417644500732, "learning_rate": 9.582407960762418e-07, "logp/chosen": -264.0, "logp/rejected": -249.0, "loss": 0.6041259765625, "reward/accuracy": 0.8125, "reward/chosen": -2.0, "reward/margin": 1.6171875, "reward/rejected": -3.625, "step": 2667 }, { "approx. KL/chosen": 222.0, "approx. KL/rejected": 1240.0, "epoch": 0.8002399520095981, "grad_norm": 3.419177532196045, "learning_rate": 9.554689637317399e-07, "logp/chosen": -454.0, "logp/rejected": -380.0, "loss": 0.3370857238769531, "reward/accuracy": 0.8125, "reward/chosen": -1.4921875, "reward/margin": 2.828125, "reward/rejected": -4.3125, "step": 2668 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 1040.0, "epoch": 0.8005398920215957, "grad_norm": 5.3631062507629395, "learning_rate": 9.527007225632751e-07, "logp/chosen": -286.0, "logp/rejected": -241.0, "loss": 0.4813995361328125, "reward/accuracy": 0.75, "reward/chosen": -1.734375, "reward/margin": 2.09375, "reward/rejected": -3.828125, "step": 2669 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 568.0, "epoch": 0.8008398320335933, "grad_norm": 5.644811630249023, "learning_rate": 9.499360750287967e-07, "logp/chosen": -276.0, "logp/rejected": -420.0, "loss": 0.7347412109375, "reward/accuracy": 0.625, "reward/chosen": -1.6640625, "reward/margin": 0.84375, "reward/rejected": -2.515625, "step": 2670 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 724.0, "epoch": 0.8011397720455908, "grad_norm": 5.800131320953369, "learning_rate": 9.471750235830579e-07, "logp/chosen": -358.0, "logp/rejected": -342.0, "loss": 0.49664306640625, "reward/accuracy": 0.8125, "reward/chosen": -1.7890625, "reward/margin": 1.4140625, "reward/rejected": -3.203125, "step": 2671 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 920.0, "epoch": 0.8014397120575885, "grad_norm": 3.7807235717773438, "learning_rate": 9.444175706776259e-07, "logp/chosen": -362.0, "logp/rejected": -340.0, "loss": 0.5234832763671875, "reward/accuracy": 0.75, "reward/chosen": -1.5234375, "reward/margin": 2.09375, "reward/rejected": -3.609375, "step": 2672 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 912.0, "epoch": 0.8017396520695861, "grad_norm": 3.625788450241089, "learning_rate": 9.416637187608624e-07, "logp/chosen": -322.0, "logp/rejected": -346.0, "loss": 0.37744140625, "reward/accuracy": 0.8125, "reward/chosen": -1.8828125, "reward/margin": 1.9296875, "reward/rejected": -3.8125, "step": 2673 }, { "approx. KL/chosen": 34.5, "approx. KL/rejected": 512.0, "epoch": 0.8020395920815837, "grad_norm": 2.4401590824127197, "learning_rate": 9.389134702779412e-07, "logp/chosen": -270.0, "logp/rejected": -290.0, "loss": 0.319122314453125, "reward/accuracy": 0.8125, "reward/chosen": -0.49609375, "reward/margin": 2.09375, "reward/rejected": -2.59375, "step": 2674 }, { "approx. KL/chosen": 246.0, "approx. KL/rejected": 976.0, "epoch": 0.8023395320935813, "grad_norm": 5.3416428565979, "learning_rate": 9.361668276708319e-07, "logp/chosen": -442.0, "logp/rejected": -448.0, "loss": 0.348388671875, "reward/accuracy": 0.8125, "reward/chosen": -1.453125, "reward/margin": 2.4375, "reward/rejected": -3.90625, "step": 2675 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 604.0, "epoch": 0.8026394721055788, "grad_norm": 3.8522400856018066, "learning_rate": 9.334237933783047e-07, "logp/chosen": -434.0, "logp/rejected": -486.0, "loss": 0.430419921875, "reward/accuracy": 0.8125, "reward/chosen": -1.65625, "reward/margin": 1.40625, "reward/rejected": -3.0625, "step": 2676 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 532.0, "epoch": 0.8029394121175765, "grad_norm": 4.128145694732666, "learning_rate": 9.306843698359241e-07, "logp/chosen": -324.0, "logp/rejected": -232.0, "loss": 0.3463134765625, "reward/accuracy": 0.75, "reward/chosen": -0.92578125, "reward/margin": 1.890625, "reward/rejected": -2.8125, "step": 2677 }, { "approx. KL/chosen": 124.5, "approx. KL/rejected": 620.0, "epoch": 0.8032393521295741, "grad_norm": 2.6802947521209717, "learning_rate": 9.279485594760496e-07, "logp/chosen": -346.0, "logp/rejected": -482.0, "loss": 0.2479705810546875, "reward/accuracy": 1.0, "reward/chosen": -1.0703125, "reward/margin": 1.9921875, "reward/rejected": -3.0625, "step": 2678 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 502.0, "epoch": 0.8035392921415717, "grad_norm": 3.508357286453247, "learning_rate": 9.252163647278307e-07, "logp/chosen": -268.0, "logp/rejected": -243.0, "loss": 0.44635009765625, "reward/accuracy": 0.8125, "reward/chosen": -1.4296875, "reward/margin": 1.4453125, "reward/rejected": -2.875, "step": 2679 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 664.0, "epoch": 0.8038392321535693, "grad_norm": 8.097400665283203, "learning_rate": 9.2248778801721e-07, "logp/chosen": -249.0, "logp/rejected": -328.0, "loss": 0.5433349609375, "reward/accuracy": 0.625, "reward/chosen": -1.65625, "reward/margin": 1.578125, "reward/rejected": -3.234375, "step": 2680 }, { "approx. KL/chosen": 68.5, "approx. KL/rejected": 748.0, "epoch": 0.8041391721655669, "grad_norm": 2.939154624938965, "learning_rate": 9.197628317669143e-07, "logp/chosen": -286.0, "logp/rejected": -310.0, "loss": 0.3191986083984375, "reward/accuracy": 0.875, "reward/chosen": -0.83984375, "reward/margin": 2.40625, "reward/rejected": -3.25, "step": 2681 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 296.0, "epoch": 0.8044391121775645, "grad_norm": 3.8759877681732178, "learning_rate": 9.170414983964599e-07, "logp/chosen": -312.0, "logp/rejected": -336.0, "loss": 0.5146484375, "reward/accuracy": 0.6875, "reward/chosen": -1.4375, "reward/margin": 0.7734375, "reward/rejected": -2.203125, "step": 2682 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 1016.0, "epoch": 0.8047390521895621, "grad_norm": 2.2950377464294434, "learning_rate": 9.143237903221386e-07, "logp/chosen": -324.0, "logp/rejected": -316.0, "loss": 0.283538818359375, "reward/accuracy": 0.875, "reward/chosen": -1.3203125, "reward/margin": 2.515625, "reward/rejected": -3.84375, "step": 2683 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 812.0, "epoch": 0.8050389922015597, "grad_norm": 3.0904033184051514, "learning_rate": 9.116097099570326e-07, "logp/chosen": -316.0, "logp/rejected": -250.0, "loss": 0.308807373046875, "reward/accuracy": 0.875, "reward/chosen": -1.3984375, "reward/margin": 2.140625, "reward/rejected": -3.546875, "step": 2684 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 828.0, "epoch": 0.8053389322135572, "grad_norm": 2.2440359592437744, "learning_rate": 9.088992597109958e-07, "logp/chosen": -278.0, "logp/rejected": -320.0, "loss": 0.298004150390625, "reward/accuracy": 0.8125, "reward/chosen": -1.484375, "reward/margin": 2.078125, "reward/rejected": -3.5625, "step": 2685 }, { "approx. KL/chosen": 78.5, "approx. KL/rejected": 956.0, "epoch": 0.8056388722255549, "grad_norm": 2.8250389099121094, "learning_rate": 9.061924419906659e-07, "logp/chosen": -394.0, "logp/rejected": -316.0, "loss": 0.207977294921875, "reward/accuracy": 0.875, "reward/chosen": -0.79296875, "reward/margin": 2.90625, "reward/rejected": -3.703125, "step": 2686 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 720.0, "epoch": 0.8059388122375525, "grad_norm": 2.9092719554901123, "learning_rate": 9.034892591994448e-07, "logp/chosen": -454.0, "logp/rejected": -386.0, "loss": 0.20440673828125, "reward/accuracy": 0.9375, "reward/chosen": -1.2109375, "reward/margin": 2.328125, "reward/rejected": -3.53125, "step": 2687 }, { "approx. KL/chosen": 516.0, "approx. KL/rejected": 932.0, "epoch": 0.8062387522495501, "grad_norm": 7.766035079956055, "learning_rate": 9.007897137375182e-07, "logp/chosen": -376.0, "logp/rejected": -394.0, "loss": 0.6512451171875, "reward/accuracy": 0.6875, "reward/chosen": -2.390625, "reward/margin": 0.98046875, "reward/rejected": -3.359375, "step": 2688 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 540.0, "epoch": 0.8065386922615477, "grad_norm": 6.345881462097168, "learning_rate": 8.980938080018343e-07, "logp/chosen": -322.0, "logp/rejected": -332.0, "loss": 0.510894775390625, "reward/accuracy": 0.6875, "reward/chosen": -1.4375, "reward/margin": 1.4921875, "reward/rejected": -2.921875, "step": 2689 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 564.0, "epoch": 0.8068386322735452, "grad_norm": 4.151406288146973, "learning_rate": 8.954015443861152e-07, "logp/chosen": -340.0, "logp/rejected": -324.0, "loss": 0.5552520751953125, "reward/accuracy": 0.625, "reward/chosen": -1.5859375, "reward/margin": 1.2734375, "reward/rejected": -2.859375, "step": 2690 }, { "approx. KL/chosen": 89.5, "approx. KL/rejected": 352.0, "epoch": 0.8071385722855429, "grad_norm": 2.4537553787231445, "learning_rate": 8.927129252808442e-07, "logp/chosen": -424.0, "logp/rejected": -364.0, "loss": 0.32501220703125, "reward/accuracy": 0.9375, "reward/chosen": -0.87890625, "reward/margin": 1.328125, "reward/rejected": -2.203125, "step": 2691 }, { "approx. KL/chosen": 129.0, "approx. KL/rejected": 576.0, "epoch": 0.8074385122975405, "grad_norm": 3.832455635070801, "learning_rate": 8.900279530732713e-07, "logp/chosen": -486.0, "logp/rejected": -424.0, "loss": 0.419403076171875, "reward/accuracy": 0.8125, "reward/chosen": -1.1015625, "reward/margin": 1.875, "reward/rejected": -2.96875, "step": 2692 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 506.0, "epoch": 0.8077384523095381, "grad_norm": 5.4260149002075195, "learning_rate": 8.87346630147407e-07, "logp/chosen": -348.0, "logp/rejected": -284.0, "loss": 0.53961181640625, "reward/accuracy": 0.75, "reward/chosen": -1.3125, "reward/margin": 1.2890625, "reward/rejected": -2.59375, "step": 2693 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 520.0, "epoch": 0.8080383923215357, "grad_norm": 4.465621471405029, "learning_rate": 8.846689588840246e-07, "logp/chosen": -270.0, "logp/rejected": -294.0, "loss": 0.636474609375, "reward/accuracy": 0.625, "reward/chosen": -1.640625, "reward/margin": 1.15625, "reward/rejected": -2.796875, "step": 2694 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 884.0, "epoch": 0.8083383323335332, "grad_norm": 2.769569158554077, "learning_rate": 8.8199494166065e-07, "logp/chosen": -432.0, "logp/rejected": -496.0, "loss": 0.254608154296875, "reward/accuracy": 0.875, "reward/chosen": -1.421875, "reward/margin": 2.421875, "reward/rejected": -3.84375, "step": 2695 }, { "approx. KL/chosen": 116.0, "approx. KL/rejected": 364.0, "epoch": 0.8086382723455309, "grad_norm": 3.6402673721313477, "learning_rate": 8.793245808515716e-07, "logp/chosen": -306.0, "logp/rejected": -270.0, "loss": 0.4234619140625, "reward/accuracy": 0.75, "reward/chosen": -1.171875, "reward/margin": 1.234375, "reward/rejected": -2.40625, "step": 2696 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 792.0, "epoch": 0.8089382123575285, "grad_norm": 3.278120517730713, "learning_rate": 8.766578788278218e-07, "logp/chosen": -356.0, "logp/rejected": -368.0, "loss": 0.420196533203125, "reward/accuracy": 0.75, "reward/chosen": -1.875, "reward/margin": 1.59375, "reward/rejected": -3.46875, "step": 2697 }, { "approx. KL/chosen": 95.5, "approx. KL/rejected": 592.0, "epoch": 0.8092381523695261, "grad_norm": 2.8709259033203125, "learning_rate": 8.739948379571938e-07, "logp/chosen": -312.0, "logp/rejected": -362.0, "loss": 0.3143310546875, "reward/accuracy": 0.8125, "reward/chosen": -1.0859375, "reward/margin": 1.9609375, "reward/rejected": -3.046875, "step": 2698 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 1120.0, "epoch": 0.8095380923815237, "grad_norm": 3.0831828117370605, "learning_rate": 8.713354606042234e-07, "logp/chosen": -444.0, "logp/rejected": -458.0, "loss": 0.251617431640625, "reward/accuracy": 0.875, "reward/chosen": -1.609375, "reward/margin": 2.625, "reward/rejected": -4.25, "step": 2699 }, { "approx. KL/chosen": 247.0, "approx. KL/rejected": 540.0, "epoch": 0.8098380323935213, "grad_norm": 6.326877593994141, "learning_rate": 8.686797491301991e-07, "logp/chosen": -304.0, "logp/rejected": -320.0, "loss": 0.6059112548828125, "reward/accuracy": 0.75, "reward/chosen": -1.390625, "reward/margin": 1.375, "reward/rejected": -2.765625, "step": 2700 }, { "approx. KL/chosen": 118.0, "approx. KL/rejected": 524.0, "epoch": 0.8101379724055189, "grad_norm": 5.12708854675293, "learning_rate": 8.660277058931488e-07, "logp/chosen": -290.0, "logp/rejected": -238.0, "loss": 0.52777099609375, "reward/accuracy": 0.75, "reward/chosen": -1.1640625, "reward/margin": 1.421875, "reward/rejected": -2.578125, "step": 2701 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 536.0, "epoch": 0.8104379124175165, "grad_norm": 4.8375396728515625, "learning_rate": 8.633793332478469e-07, "logp/chosen": -410.0, "logp/rejected": -330.0, "loss": 0.567626953125, "reward/accuracy": 0.6875, "reward/chosen": -1.75, "reward/margin": 1.0234375, "reward/rejected": -2.78125, "step": 2702 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 824.0, "epoch": 0.8107378524295141, "grad_norm": 4.613837718963623, "learning_rate": 8.607346335458055e-07, "logp/chosen": -388.0, "logp/rejected": -376.0, "loss": 0.6375732421875, "reward/accuracy": 0.75, "reward/chosen": -2.203125, "reward/margin": 1.3203125, "reward/rejected": -3.53125, "step": 2703 }, { "approx. KL/chosen": 113.0, "approx. KL/rejected": 712.0, "epoch": 0.8110377924415118, "grad_norm": 2.410644769668579, "learning_rate": 8.580936091352815e-07, "logp/chosen": -318.0, "logp/rejected": -312.0, "loss": 0.25640869140625, "reward/accuracy": 0.875, "reward/chosen": -1.15625, "reward/margin": 2.234375, "reward/rejected": -3.375, "step": 2704 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 920.0, "epoch": 0.8113377324535093, "grad_norm": 2.7899937629699707, "learning_rate": 8.554562623612606e-07, "logp/chosen": -244.0, "logp/rejected": -298.0, "loss": 0.2572021484375, "reward/accuracy": 0.8125, "reward/chosen": -1.5546875, "reward/margin": 2.265625, "reward/rejected": -3.828125, "step": 2705 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 1000.0, "epoch": 0.8116376724655069, "grad_norm": 5.336358547210693, "learning_rate": 8.528225955654717e-07, "logp/chosen": -390.0, "logp/rejected": -382.0, "loss": 0.27069091796875, "reward/accuracy": 0.8125, "reward/chosen": -1.0859375, "reward/margin": 2.84375, "reward/rejected": -3.921875, "step": 2706 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 688.0, "epoch": 0.8119376124775045, "grad_norm": 3.66195011138916, "learning_rate": 8.501926110863668e-07, "logp/chosen": -394.0, "logp/rejected": -378.0, "loss": 0.40985107421875, "reward/accuracy": 0.625, "reward/chosen": -1.6015625, "reward/margin": 1.6953125, "reward/rejected": -3.3125, "step": 2707 }, { "approx. KL/chosen": 213.0, "approx. KL/rejected": 1152.0, "epoch": 0.8122375524895021, "grad_norm": 3.700594186782837, "learning_rate": 8.475663112591364e-07, "logp/chosen": -276.0, "logp/rejected": -322.0, "loss": 0.38690185546875, "reward/accuracy": 0.8125, "reward/chosen": -1.4375, "reward/margin": 2.5625, "reward/rejected": -4.0, "step": 2708 }, { "approx. KL/chosen": 142.0, "approx. KL/rejected": 672.0, "epoch": 0.8125374925014996, "grad_norm": 4.695955276489258, "learning_rate": 8.449436984156933e-07, "logp/chosen": -344.0, "logp/rejected": -318.0, "loss": 0.59814453125, "reward/accuracy": 0.6875, "reward/chosen": -1.1015625, "reward/margin": 1.3984375, "reward/rejected": -2.5, "step": 2709 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 648.0, "epoch": 0.8128374325134973, "grad_norm": 2.810227632522583, "learning_rate": 8.423247748846841e-07, "logp/chosen": -382.0, "logp/rejected": -320.0, "loss": 0.28424072265625, "reward/accuracy": 0.9375, "reward/chosen": -1.296875, "reward/margin": 1.765625, "reward/rejected": -3.0625, "step": 2710 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 968.0, "epoch": 0.8131373725254949, "grad_norm": 3.128810167312622, "learning_rate": 8.397095429914687e-07, "logp/chosen": -354.0, "logp/rejected": -448.0, "loss": 0.2669677734375, "reward/accuracy": 0.9375, "reward/chosen": -1.7734375, "reward/margin": 2.0, "reward/rejected": -3.78125, "step": 2711 }, { "approx. KL/chosen": 370.0, "approx. KL/rejected": 880.0, "epoch": 0.8134373125374925, "grad_norm": 4.948862075805664, "learning_rate": 8.370980050581406e-07, "logp/chosen": -332.0, "logp/rejected": -300.0, "loss": 0.3856201171875, "reward/accuracy": 0.8125, "reward/chosen": -2.15625, "reward/margin": 1.5625, "reward/rejected": -3.71875, "step": 2712 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 592.0, "epoch": 0.8137372525494901, "grad_norm": 3.573915481567383, "learning_rate": 8.344901634035041e-07, "logp/chosen": -272.0, "logp/rejected": -346.0, "loss": 0.490478515625, "reward/accuracy": 0.75, "reward/chosen": -1.4375, "reward/margin": 1.21875, "reward/rejected": -2.65625, "step": 2713 }, { "approx. KL/chosen": 165.0, "approx. KL/rejected": 548.0, "epoch": 0.8140371925614877, "grad_norm": 4.064361572265625, "learning_rate": 8.318860203430896e-07, "logp/chosen": -446.0, "logp/rejected": -408.0, "loss": 0.4776611328125, "reward/accuracy": 0.6875, "reward/chosen": -1.3359375, "reward/margin": 1.203125, "reward/rejected": -2.546875, "step": 2714 }, { "approx. KL/chosen": 86.5, "approx. KL/rejected": 708.0, "epoch": 0.8143371325734853, "grad_norm": 4.177372455596924, "learning_rate": 8.292855781891368e-07, "logp/chosen": -248.0, "logp/rejected": -286.0, "loss": 0.31976318359375, "reward/accuracy": 0.875, "reward/chosen": -0.7734375, "reward/margin": 2.296875, "reward/rejected": -3.0625, "step": 2715 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 1120.0, "epoch": 0.8146370725854829, "grad_norm": 1.745375156402588, "learning_rate": 8.266888392506034e-07, "logp/chosen": -356.0, "logp/rejected": -378.0, "loss": 0.1591033935546875, "reward/accuracy": 0.9375, "reward/chosen": -1.515625, "reward/margin": 2.65625, "reward/rejected": -4.1875, "step": 2716 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 928.0, "epoch": 0.8149370125974805, "grad_norm": 2.521054983139038, "learning_rate": 8.240958058331555e-07, "logp/chosen": -151.0, "logp/rejected": -258.0, "loss": 0.2204132080078125, "reward/accuracy": 1.0, "reward/chosen": -1.40625, "reward/margin": 2.078125, "reward/rejected": -3.484375, "step": 2717 }, { "approx. KL/chosen": 171.0, "approx. KL/rejected": 584.0, "epoch": 0.8152369526094781, "grad_norm": 2.613856315612793, "learning_rate": 8.215064802391737e-07, "logp/chosen": -288.0, "logp/rejected": -282.0, "loss": 0.31439208984375, "reward/accuracy": 0.875, "reward/chosen": -1.3828125, "reward/margin": 1.546875, "reward/rejected": -2.9375, "step": 2718 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 828.0, "epoch": 0.8155368926214757, "grad_norm": 3.051325798034668, "learning_rate": 8.189208647677415e-07, "logp/chosen": -432.0, "logp/rejected": -326.0, "loss": 0.282958984375, "reward/accuracy": 0.875, "reward/chosen": -1.0234375, "reward/margin": 2.421875, "reward/rejected": -3.453125, "step": 2719 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 712.0, "epoch": 0.8158368326334733, "grad_norm": 3.517512798309326, "learning_rate": 8.163389617146544e-07, "logp/chosen": -376.0, "logp/rejected": -308.0, "loss": 0.313385009765625, "reward/accuracy": 0.8125, "reward/chosen": -1.3515625, "reward/margin": 1.9609375, "reward/rejected": -3.3125, "step": 2720 }, { "approx. KL/chosen": 205.0, "approx. KL/rejected": 1184.0, "epoch": 0.8161367726454709, "grad_norm": 3.4380788803100586, "learning_rate": 8.137607733724029e-07, "logp/chosen": -282.0, "logp/rejected": -328.0, "loss": 0.3670654296875, "reward/accuracy": 0.8125, "reward/chosen": -1.7421875, "reward/margin": 2.34375, "reward/rejected": -4.09375, "step": 2721 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 656.0, "epoch": 0.8164367126574685, "grad_norm": 3.7131872177124023, "learning_rate": 8.111863020301869e-07, "logp/chosen": -312.0, "logp/rejected": -382.0, "loss": 0.37298583984375, "reward/accuracy": 0.75, "reward/chosen": -1.8671875, "reward/margin": 1.5078125, "reward/rejected": -3.375, "step": 2722 }, { "approx. KL/chosen": 412.0, "approx. KL/rejected": 632.0, "epoch": 0.8167366526694662, "grad_norm": 8.167724609375, "learning_rate": 8.086155499739013e-07, "logp/chosen": -422.0, "logp/rejected": -400.0, "loss": 0.838623046875, "reward/accuracy": 0.5625, "reward/chosen": -2.234375, "reward/margin": 0.72265625, "reward/rejected": -2.96875, "step": 2723 }, { "approx. KL/chosen": 88.5, "approx. KL/rejected": 876.0, "epoch": 0.8170365926814637, "grad_norm": 2.9679934978485107, "learning_rate": 8.060485194861423e-07, "logp/chosen": -358.0, "logp/rejected": -498.0, "loss": 0.3283424377441406, "reward/accuracy": 0.8125, "reward/chosen": -0.984375, "reward/margin": 2.484375, "reward/rejected": -3.46875, "step": 2724 }, { "approx. KL/chosen": 330.0, "approx. KL/rejected": 1016.0, "epoch": 0.8173365326934613, "grad_norm": 6.779970645904541, "learning_rate": 8.03485212846199e-07, "logp/chosen": -372.0, "logp/rejected": -340.0, "loss": 0.43829345703125, "reward/accuracy": 0.6875, "reward/chosen": -1.6484375, "reward/margin": 2.234375, "reward/rejected": -3.875, "step": 2725 }, { "approx. KL/chosen": 362.0, "approx. KL/rejected": 948.0, "epoch": 0.8176364727054589, "grad_norm": 6.035209655761719, "learning_rate": 8.009256323300541e-07, "logp/chosen": -207.0, "logp/rejected": -230.0, "loss": 0.5721397399902344, "reward/accuracy": 0.75, "reward/chosen": -1.953125, "reward/margin": 1.9296875, "reward/rejected": -3.875, "step": 2726 }, { "approx. KL/chosen": 253.0, "approx. KL/rejected": 1248.0, "epoch": 0.8179364127174565, "grad_norm": 4.49755334854126, "learning_rate": 7.983697802103813e-07, "logp/chosen": -239.0, "logp/rejected": -352.0, "loss": 0.5878334045410156, "reward/accuracy": 0.8125, "reward/chosen": -1.578125, "reward/margin": 2.765625, "reward/rejected": -4.34375, "step": 2727 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 498.0, "epoch": 0.8182363527294542, "grad_norm": 4.085971355438232, "learning_rate": 7.958176587565486e-07, "logp/chosen": -402.0, "logp/rejected": -414.0, "loss": 0.509033203125, "reward/accuracy": 0.625, "reward/chosen": -1.3046875, "reward/margin": 1.0859375, "reward/rejected": -2.390625, "step": 2728 }, { "approx. KL/chosen": 221.0, "approx. KL/rejected": 732.0, "epoch": 0.8185362927414517, "grad_norm": 3.6395437717437744, "learning_rate": 7.93269270234604e-07, "logp/chosen": -256.0, "logp/rejected": -312.0, "loss": 0.3414306640625, "reward/accuracy": 0.875, "reward/chosen": -1.5234375, "reward/margin": 1.65625, "reward/rejected": -3.1875, "step": 2729 }, { "approx. KL/chosen": 175.0, "approx. KL/rejected": 1168.0, "epoch": 0.8188362327534493, "grad_norm": 2.9157257080078125, "learning_rate": 7.907246169072896e-07, "logp/chosen": -162.0, "logp/rejected": -205.0, "loss": 0.283935546875, "reward/accuracy": 0.8125, "reward/chosen": -1.4140625, "reward/margin": 2.828125, "reward/rejected": -4.25, "step": 2730 }, { "approx. KL/chosen": 418.0, "approx. KL/rejected": 888.0, "epoch": 0.8191361727654469, "grad_norm": 7.128449440002441, "learning_rate": 7.881837010340215e-07, "logp/chosen": -390.0, "logp/rejected": -318.0, "loss": 0.41522216796875, "reward/accuracy": 0.75, "reward/chosen": -2.0, "reward/margin": 1.9765625, "reward/rejected": -3.984375, "step": 2731 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 748.0, "epoch": 0.8194361127774445, "grad_norm": 4.817836284637451, "learning_rate": 7.85646524870905e-07, "logp/chosen": -328.0, "logp/rejected": -326.0, "loss": 0.5820770263671875, "reward/accuracy": 0.75, "reward/chosen": -1.3828125, "reward/margin": 1.6640625, "reward/rejected": -3.046875, "step": 2732 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 1392.0, "epoch": 0.8197360527894421, "grad_norm": 2.5720913410186768, "learning_rate": 7.831130906707202e-07, "logp/chosen": -464.0, "logp/rejected": -588.0, "loss": 0.160919189453125, "reward/accuracy": 1.0, "reward/chosen": -1.53125, "reward/margin": 3.015625, "reward/rejected": -4.53125, "step": 2733 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 592.0, "epoch": 0.8200359928014397, "grad_norm": 3.787332773208618, "learning_rate": 7.8058340068293e-07, "logp/chosen": -294.0, "logp/rejected": -374.0, "loss": 0.457763671875, "reward/accuracy": 0.75, "reward/chosen": -1.5, "reward/margin": 1.3984375, "reward/rejected": -2.90625, "step": 2734 }, { "approx. KL/chosen": 209.0, "approx. KL/rejected": 1208.0, "epoch": 0.8203359328134373, "grad_norm": 3.894249439239502, "learning_rate": 7.780574571536642e-07, "logp/chosen": -380.0, "logp/rejected": -394.0, "loss": 0.2458953857421875, "reward/accuracy": 0.8125, "reward/chosen": -1.59375, "reward/margin": 2.875, "reward/rejected": -4.46875, "step": 2735 }, { "approx. KL/chosen": 292.0, "approx. KL/rejected": 804.0, "epoch": 0.8206358728254349, "grad_norm": 4.718132019042969, "learning_rate": 7.755352623257345e-07, "logp/chosen": -288.0, "logp/rejected": -298.0, "loss": 0.7461090087890625, "reward/accuracy": 0.5625, "reward/chosen": -2.078125, "reward/margin": 1.2890625, "reward/rejected": -3.375, "step": 2736 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 636.0, "epoch": 0.8209358128374326, "grad_norm": 4.974899768829346, "learning_rate": 7.730168184386188e-07, "logp/chosen": -338.0, "logp/rejected": -298.0, "loss": 0.51116943359375, "reward/accuracy": 0.75, "reward/chosen": -1.65625, "reward/margin": 1.3515625, "reward/rejected": -3.0, "step": 2737 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 864.0, "epoch": 0.8212357528494301, "grad_norm": 4.361364364624023, "learning_rate": 7.705021277284685e-07, "logp/chosen": -380.0, "logp/rejected": -308.0, "loss": 0.399261474609375, "reward/accuracy": 0.875, "reward/chosen": -1.4375, "reward/margin": 2.140625, "reward/rejected": -3.5625, "step": 2738 }, { "approx. KL/chosen": 510.0, "approx. KL/rejected": 592.0, "epoch": 0.8215356928614277, "grad_norm": 9.715679168701172, "learning_rate": 7.679911924280997e-07, "logp/chosen": -440.0, "logp/rejected": -356.0, "loss": 0.9046630859375, "reward/accuracy": 0.5625, "reward/chosen": -1.8046875, "reward/margin": 0.92578125, "reward/rejected": -2.734375, "step": 2739 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 1048.0, "epoch": 0.8218356328734253, "grad_norm": 3.754396915435791, "learning_rate": 7.654840147669945e-07, "logp/chosen": -408.0, "logp/rejected": -330.0, "loss": 0.35945892333984375, "reward/accuracy": 0.8125, "reward/chosen": -1.6953125, "reward/margin": 2.25, "reward/rejected": -3.953125, "step": 2740 }, { "approx. KL/chosen": 136.0, "approx. KL/rejected": 540.0, "epoch": 0.8221355728854229, "grad_norm": 3.4593493938446045, "learning_rate": 7.629805969712978e-07, "logp/chosen": -308.0, "logp/rejected": -298.0, "loss": 0.4542999267578125, "reward/accuracy": 0.75, "reward/chosen": -1.40625, "reward/margin": 1.3359375, "reward/rejected": -2.75, "step": 2741 }, { "approx. KL/chosen": 246.0, "approx. KL/rejected": 844.0, "epoch": 0.8224355128974206, "grad_norm": 3.8095757961273193, "learning_rate": 7.604809412638186e-07, "logp/chosen": -426.0, "logp/rejected": -358.0, "loss": 0.31146240234375, "reward/accuracy": 0.8125, "reward/chosen": -1.71875, "reward/margin": 1.953125, "reward/rejected": -3.671875, "step": 2742 }, { "approx. KL/chosen": 127.0, "approx. KL/rejected": 556.0, "epoch": 0.8227354529094181, "grad_norm": 3.7779805660247803, "learning_rate": 7.579850498640229e-07, "logp/chosen": -454.0, "logp/rejected": -474.0, "loss": 0.36199951171875, "reward/accuracy": 0.75, "reward/chosen": -1.0234375, "reward/margin": 1.75, "reward/rejected": -2.78125, "step": 2743 }, { "approx. KL/chosen": 196.0, "approx. KL/rejected": 1020.0, "epoch": 0.8230353929214157, "grad_norm": 3.716928005218506, "learning_rate": 7.554929249880383e-07, "logp/chosen": -253.0, "logp/rejected": -320.0, "loss": 0.384490966796875, "reward/accuracy": 0.875, "reward/chosen": -1.546875, "reward/margin": 2.03125, "reward/rejected": -3.578125, "step": 2744 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 688.0, "epoch": 0.8233353329334133, "grad_norm": 5.238277912139893, "learning_rate": 7.530045688486409e-07, "logp/chosen": -314.0, "logp/rejected": -404.0, "loss": 0.749420166015625, "reward/accuracy": 0.5625, "reward/chosen": -1.7734375, "reward/margin": 1.265625, "reward/rejected": -3.03125, "step": 2745 }, { "approx. KL/chosen": 167.0, "approx. KL/rejected": 688.0, "epoch": 0.823635272945411, "grad_norm": 5.455094337463379, "learning_rate": 7.505199836552684e-07, "logp/chosen": -312.0, "logp/rejected": -350.0, "loss": 0.5850887298583984, "reward/accuracy": 0.6875, "reward/chosen": -1.3359375, "reward/margin": 1.6875, "reward/rejected": -3.03125, "step": 2746 }, { "approx. KL/chosen": 94.0, "approx. KL/rejected": 884.0, "epoch": 0.8239352129574086, "grad_norm": 1.9511628150939941, "learning_rate": 7.480391716140034e-07, "logp/chosen": -428.0, "logp/rejected": -272.0, "loss": 0.223968505859375, "reward/accuracy": 0.875, "reward/chosen": -0.828125, "reward/margin": 2.796875, "reward/rejected": -3.625, "step": 2747 }, { "approx. KL/chosen": 354.0, "approx. KL/rejected": 752.0, "epoch": 0.8242351529694061, "grad_norm": 4.842219352722168, "learning_rate": 7.455621349275854e-07, "logp/chosen": -314.0, "logp/rejected": -286.0, "loss": 0.35711669921875, "reward/accuracy": 0.8125, "reward/chosen": -2.09375, "reward/margin": 1.5234375, "reward/rejected": -3.609375, "step": 2748 }, { "approx. KL/chosen": 234.0, "approx. KL/rejected": 784.0, "epoch": 0.8245350929814037, "grad_norm": 3.8908839225769043, "learning_rate": 7.430888757953952e-07, "logp/chosen": -199.0, "logp/rejected": -310.0, "loss": 0.414154052734375, "reward/accuracy": 0.75, "reward/chosen": -1.734375, "reward/margin": 1.6640625, "reward/rejected": -3.40625, "step": 2749 }, { "approx. KL/chosen": 251.0, "approx. KL/rejected": 712.0, "epoch": 0.8248350329934013, "grad_norm": 5.870274543762207, "learning_rate": 7.406193964134634e-07, "logp/chosen": -350.0, "logp/rejected": -250.0, "loss": 0.58642578125, "reward/accuracy": 0.625, "reward/chosen": -1.8828125, "reward/margin": 1.3046875, "reward/rejected": -3.1875, "step": 2750 }, { "approx. KL/chosen": 65.0, "approx. KL/rejected": 860.0, "epoch": 0.825134973005399, "grad_norm": 2.0447492599487305, "learning_rate": 7.381536989744615e-07, "logp/chosen": -318.0, "logp/rejected": -306.0, "loss": 0.1639404296875, "reward/accuracy": 0.9375, "reward/chosen": -0.59375, "reward/margin": 3.046875, "reward/rejected": -3.640625, "step": 2751 }, { "approx. KL/chosen": 252.0, "approx. KL/rejected": 1048.0, "epoch": 0.8254349130173965, "grad_norm": 3.39516019821167, "learning_rate": 7.35691785667707e-07, "logp/chosen": -390.0, "logp/rejected": -524.0, "loss": 0.30120849609375, "reward/accuracy": 0.8125, "reward/chosen": -1.703125, "reward/margin": 1.984375, "reward/rejected": -3.6875, "step": 2752 }, { "approx. KL/chosen": 96.5, "approx. KL/rejected": 900.0, "epoch": 0.8257348530293941, "grad_norm": 1.7765575647354126, "learning_rate": 7.33233658679155e-07, "logp/chosen": -392.0, "logp/rejected": -436.0, "loss": 0.152740478515625, "reward/accuracy": 1.0, "reward/chosen": -0.921875, "reward/margin": 2.671875, "reward/rejected": -3.59375, "step": 2753 }, { "approx. KL/chosen": 450.0, "approx. KL/rejected": 648.0, "epoch": 0.8260347930413917, "grad_norm": 7.361942768096924, "learning_rate": 7.307793201913977e-07, "logp/chosen": -424.0, "logp/rejected": -240.0, "loss": 0.6512451171875, "reward/accuracy": 0.6875, "reward/chosen": -1.9921875, "reward/margin": 1.0078125, "reward/rejected": -3.0, "step": 2754 }, { "approx. KL/chosen": 246.0, "approx. KL/rejected": 864.0, "epoch": 0.8263347330533893, "grad_norm": 3.9444618225097656, "learning_rate": 7.283287723836635e-07, "logp/chosen": -404.0, "logp/rejected": -432.0, "loss": 0.5190582275390625, "reward/accuracy": 0.8125, "reward/chosen": -1.59375, "reward/margin": 1.9765625, "reward/rejected": -3.5625, "step": 2755 }, { "approx. KL/chosen": 350.0, "approx. KL/rejected": 1112.0, "epoch": 0.826634673065387, "grad_norm": 4.576372146606445, "learning_rate": 7.258820174318198e-07, "logp/chosen": -358.0, "logp/rejected": -424.0, "loss": 0.5431976318359375, "reward/accuracy": 0.75, "reward/chosen": -2.125, "reward/margin": 2.125, "reward/rejected": -4.25, "step": 2756 }, { "approx. KL/chosen": 316.0, "approx. KL/rejected": 468.0, "epoch": 0.8269346130773845, "grad_norm": 6.614381790161133, "learning_rate": 7.2343905750836e-07, "logp/chosen": -236.0, "logp/rejected": -198.0, "loss": 0.68206787109375, "reward/accuracy": 0.6875, "reward/chosen": -2.046875, "reward/margin": 0.703125, "reward/rejected": -2.75, "step": 2757 }, { "approx. KL/chosen": 416.0, "approx. KL/rejected": 1360.0, "epoch": 0.8272345530893821, "grad_norm": 4.2450385093688965, "learning_rate": 7.209998947824115e-07, "logp/chosen": -288.0, "logp/rejected": -528.0, "loss": 0.3794374465942383, "reward/accuracy": 0.875, "reward/chosen": -2.15625, "reward/margin": 2.21875, "reward/rejected": -4.375, "step": 2758 }, { "approx. KL/chosen": 244.0, "approx. KL/rejected": 552.0, "epoch": 0.8275344931013797, "grad_norm": 4.585495471954346, "learning_rate": 7.185645314197276e-07, "logp/chosen": -306.0, "logp/rejected": -316.0, "loss": 0.664794921875, "reward/accuracy": 0.5625, "reward/chosen": -1.765625, "reward/margin": 0.68359375, "reward/rejected": -2.453125, "step": 2759 }, { "approx. KL/chosen": 314.0, "approx. KL/rejected": 892.0, "epoch": 0.8278344331133773, "grad_norm": 2.8764545917510986, "learning_rate": 7.161329695826925e-07, "logp/chosen": -334.0, "logp/rejected": -366.0, "loss": 0.31170654296875, "reward/accuracy": 0.9375, "reward/chosen": -2.09375, "reward/margin": 1.7734375, "reward/rejected": -3.859375, "step": 2760 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 908.0, "epoch": 0.828134373125375, "grad_norm": 3.6905508041381836, "learning_rate": 7.137052114303089e-07, "logp/chosen": -394.0, "logp/rejected": -386.0, "loss": 0.3488311767578125, "reward/accuracy": 0.8125, "reward/chosen": -1.5390625, "reward/margin": 2.234375, "reward/rejected": -3.765625, "step": 2761 }, { "approx. KL/chosen": 212.0, "approx. KL/rejected": 828.0, "epoch": 0.8284343131373725, "grad_norm": 4.713716983795166, "learning_rate": 7.112812591182084e-07, "logp/chosen": -270.0, "logp/rejected": -370.0, "loss": 0.550140380859375, "reward/accuracy": 0.6875, "reward/chosen": -1.578125, "reward/margin": 1.9453125, "reward/rejected": -3.515625, "step": 2762 }, { "approx. KL/chosen": 540.0, "approx. KL/rejected": 1320.0, "epoch": 0.8287342531493701, "grad_norm": 4.608727931976318, "learning_rate": 7.088611147986391e-07, "logp/chosen": -346.0, "logp/rejected": -344.0, "loss": 0.478729248046875, "reward/accuracy": 0.75, "reward/chosen": -2.234375, "reward/margin": 1.96875, "reward/rejected": -4.21875, "step": 2763 }, { "approx. KL/chosen": 95.5, "approx. KL/rejected": 1448.0, "epoch": 0.8290341931613677, "grad_norm": 1.7191089391708374, "learning_rate": 7.064447806204694e-07, "logp/chosen": -264.0, "logp/rejected": -362.0, "loss": 0.11706256866455078, "reward/accuracy": 0.9375, "reward/chosen": -1.0078125, "reward/margin": 3.625, "reward/rejected": -4.625, "step": 2764 }, { "approx. KL/chosen": 564.0, "approx. KL/rejected": 772.0, "epoch": 0.8293341331733654, "grad_norm": 7.449337005615234, "learning_rate": 7.040322587291826e-07, "logp/chosen": -304.0, "logp/rejected": -288.0, "loss": 1.045654296875, "reward/accuracy": 0.5625, "reward/chosen": -2.109375, "reward/margin": 1.015625, "reward/rejected": -3.125, "step": 2765 }, { "approx. KL/chosen": 154.0, "approx. KL/rejected": 704.0, "epoch": 0.829634073185363, "grad_norm": 3.250033140182495, "learning_rate": 7.016235512668823e-07, "logp/chosen": -322.0, "logp/rejected": -348.0, "loss": 0.365509033203125, "reward/accuracy": 0.8125, "reward/chosen": -1.171875, "reward/margin": 1.8359375, "reward/rejected": -3.0, "step": 2766 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 852.0, "epoch": 0.8299340131973605, "grad_norm": 2.626451015472412, "learning_rate": 6.9921866037228e-07, "logp/chosen": -336.0, "logp/rejected": -402.0, "loss": 0.2554931640625, "reward/accuracy": 0.875, "reward/chosen": -1.4921875, "reward/margin": 2.265625, "reward/rejected": -3.75, "step": 2767 }, { "approx. KL/chosen": 115.5, "approx. KL/rejected": 784.0, "epoch": 0.8302339532093581, "grad_norm": 3.186842679977417, "learning_rate": 6.968175881807004e-07, "logp/chosen": -268.0, "logp/rejected": -296.0, "loss": 0.27761077880859375, "reward/accuracy": 0.875, "reward/chosen": -0.953125, "reward/margin": 2.625, "reward/rejected": -3.578125, "step": 2768 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 952.0, "epoch": 0.8305338932213557, "grad_norm": 4.664588451385498, "learning_rate": 6.94420336824077e-07, "logp/chosen": -360.0, "logp/rejected": -356.0, "loss": 0.5877685546875, "reward/accuracy": 0.75, "reward/chosen": -1.7421875, "reward/margin": 1.7265625, "reward/rejected": -3.46875, "step": 2769 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 920.0, "epoch": 0.8308338332333534, "grad_norm": 3.1915056705474854, "learning_rate": 6.920269084309522e-07, "logp/chosen": -324.0, "logp/rejected": -342.0, "loss": 0.3764190673828125, "reward/accuracy": 0.75, "reward/chosen": -1.296875, "reward/margin": 2.25, "reward/rejected": -3.546875, "step": 2770 }, { "approx. KL/chosen": 149.0, "approx. KL/rejected": 708.0, "epoch": 0.831133773245351, "grad_norm": 3.129819393157959, "learning_rate": 6.896373051264721e-07, "logp/chosen": -434.0, "logp/rejected": -374.0, "loss": 0.313385009765625, "reward/accuracy": 0.8125, "reward/chosen": -1.265625, "reward/margin": 1.7578125, "reward/rejected": -3.015625, "step": 2771 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 876.0, "epoch": 0.8314337132573485, "grad_norm": 2.2422282695770264, "learning_rate": 6.872515290323873e-07, "logp/chosen": -294.0, "logp/rejected": -278.0, "loss": 0.2167816162109375, "reward/accuracy": 0.8125, "reward/chosen": -1.5078125, "reward/margin": 2.4375, "reward/rejected": -3.953125, "step": 2772 }, { "approx. KL/chosen": 149.0, "approx. KL/rejected": 948.0, "epoch": 0.8317336532693461, "grad_norm": 2.3994743824005127, "learning_rate": 6.848695822670509e-07, "logp/chosen": -306.0, "logp/rejected": -420.0, "loss": 0.274078369140625, "reward/accuracy": 0.9375, "reward/chosen": -1.21875, "reward/margin": 2.671875, "reward/rejected": -3.890625, "step": 2773 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 996.0, "epoch": 0.8320335932813437, "grad_norm": 5.1154680252075195, "learning_rate": 6.824914669454153e-07, "logp/chosen": -346.0, "logp/rejected": -338.0, "loss": 0.515350341796875, "reward/accuracy": 0.625, "reward/chosen": -2.03125, "reward/margin": 1.6875, "reward/rejected": -3.734375, "step": 2774 }, { "approx. KL/chosen": 360.0, "approx. KL/rejected": 968.0, "epoch": 0.8323335332933414, "grad_norm": 4.561246871948242, "learning_rate": 6.801171851790295e-07, "logp/chosen": -316.0, "logp/rejected": -242.0, "loss": 0.57818603515625, "reward/accuracy": 0.625, "reward/chosen": -2.359375, "reward/margin": 1.3125, "reward/rejected": -3.671875, "step": 2775 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 1232.0, "epoch": 0.8326334733053389, "grad_norm": 3.418412685394287, "learning_rate": 6.777467390760401e-07, "logp/chosen": -288.0, "logp/rejected": -398.0, "loss": 0.48764801025390625, "reward/accuracy": 0.8125, "reward/chosen": -1.640625, "reward/margin": 2.390625, "reward/rejected": -4.03125, "step": 2776 }, { "approx. KL/chosen": 330.0, "approx. KL/rejected": 1112.0, "epoch": 0.8329334133173365, "grad_norm": 5.889405727386475, "learning_rate": 6.753801307411895e-07, "logp/chosen": -350.0, "logp/rejected": -384.0, "loss": 0.6293792724609375, "reward/accuracy": 0.6875, "reward/chosen": -2.078125, "reward/margin": 1.8984375, "reward/rejected": -3.984375, "step": 2777 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 772.0, "epoch": 0.8332333533293341, "grad_norm": 2.681143045425415, "learning_rate": 6.730173622758096e-07, "logp/chosen": -384.0, "logp/rejected": -370.0, "loss": 0.31365203857421875, "reward/accuracy": 0.9375, "reward/chosen": -1.328125, "reward/margin": 2.203125, "reward/rejected": -3.53125, "step": 2778 }, { "approx. KL/chosen": 172.0, "approx. KL/rejected": 856.0, "epoch": 0.8335332933413317, "grad_norm": 3.166808843612671, "learning_rate": 6.706584357778235e-07, "logp/chosen": -494.0, "logp/rejected": -428.0, "loss": 0.29791259765625, "reward/accuracy": 0.875, "reward/chosen": -1.5234375, "reward/margin": 2.171875, "reward/rejected": -3.6875, "step": 2779 }, { "approx. KL/chosen": 348.0, "approx. KL/rejected": 1040.0, "epoch": 0.8338332333533294, "grad_norm": 5.903453826904297, "learning_rate": 6.683033533417454e-07, "logp/chosen": -246.0, "logp/rejected": -346.0, "loss": 0.4688262939453125, "reward/accuracy": 0.6875, "reward/chosen": -2.21875, "reward/margin": 1.7578125, "reward/rejected": -3.96875, "step": 2780 }, { "approx. KL/chosen": 448.0, "approx. KL/rejected": 1004.0, "epoch": 0.8341331733653269, "grad_norm": 4.536708831787109, "learning_rate": 6.659521170586736e-07, "logp/chosen": -262.0, "logp/rejected": -388.0, "loss": 0.5216560363769531, "reward/accuracy": 0.6875, "reward/chosen": -2.296875, "reward/margin": 1.5625, "reward/rejected": -3.859375, "step": 2781 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 772.0, "epoch": 0.8344331133773245, "grad_norm": 4.210517883300781, "learning_rate": 6.636047290162917e-07, "logp/chosen": -266.0, "logp/rejected": -292.0, "loss": 0.5850830078125, "reward/accuracy": 0.6875, "reward/chosen": -1.6875, "reward/margin": 1.5234375, "reward/rejected": -3.203125, "step": 2782 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 672.0, "epoch": 0.8347330533893221, "grad_norm": 5.4288835525512695, "learning_rate": 6.612611912988664e-07, "logp/chosen": -314.0, "logp/rejected": -294.0, "loss": 0.5890350341796875, "reward/accuracy": 0.8125, "reward/chosen": -1.5859375, "reward/margin": 1.390625, "reward/rejected": -2.96875, "step": 2783 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 688.0, "epoch": 0.8350329934013198, "grad_norm": 5.266287326812744, "learning_rate": 6.589215059872489e-07, "logp/chosen": -226.0, "logp/rejected": -243.0, "loss": 0.5554351806640625, "reward/accuracy": 0.8125, "reward/chosen": -1.5, "reward/margin": 1.578125, "reward/rejected": -3.078125, "step": 2784 }, { "approx. KL/chosen": 408.0, "approx. KL/rejected": 580.0, "epoch": 0.8353329334133174, "grad_norm": 6.082633972167969, "learning_rate": 6.565856751588651e-07, "logp/chosen": -342.0, "logp/rejected": -258.0, "loss": 0.897125244140625, "reward/accuracy": 0.625, "reward/chosen": -2.140625, "reward/margin": 0.79296875, "reward/rejected": -2.9375, "step": 2785 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 656.0, "epoch": 0.8356328734253149, "grad_norm": 3.110790252685547, "learning_rate": 6.542537008877198e-07, "logp/chosen": -282.0, "logp/rejected": -326.0, "loss": 0.4285888671875, "reward/accuracy": 0.8125, "reward/chosen": -1.6875, "reward/margin": 1.4609375, "reward/rejected": -3.140625, "step": 2786 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 450.0, "epoch": 0.8359328134373125, "grad_norm": 4.695925712585449, "learning_rate": 6.519255852443968e-07, "logp/chosen": -374.0, "logp/rejected": -332.0, "loss": 0.5626220703125, "reward/accuracy": 0.6875, "reward/chosen": -1.875, "reward/margin": 0.875, "reward/rejected": -2.75, "step": 2787 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 1064.0, "epoch": 0.8362327534493101, "grad_norm": 4.183576583862305, "learning_rate": 6.496013302960491e-07, "logp/chosen": -266.0, "logp/rejected": -452.0, "loss": 0.3910675048828125, "reward/accuracy": 0.8125, "reward/chosen": -1.578125, "reward/margin": 2.4375, "reward/rejected": -4.03125, "step": 2788 }, { "approx. KL/chosen": 348.0, "approx. KL/rejected": 656.0, "epoch": 0.8365326934613078, "grad_norm": 5.446656227111816, "learning_rate": 6.472809381064055e-07, "logp/chosen": -440.0, "logp/rejected": -340.0, "loss": 0.46063232421875, "reward/accuracy": 0.9375, "reward/chosen": -1.734375, "reward/margin": 1.5390625, "reward/rejected": -3.265625, "step": 2789 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 636.0, "epoch": 0.8368326334733054, "grad_norm": 3.215730905532837, "learning_rate": 6.449644107357606e-07, "logp/chosen": -318.0, "logp/rejected": -268.0, "loss": 0.362274169921875, "reward/accuracy": 0.875, "reward/chosen": -1.6953125, "reward/margin": 1.546875, "reward/rejected": -3.25, "step": 2790 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 446.0, "epoch": 0.8371325734853029, "grad_norm": 5.799509048461914, "learning_rate": 6.42651750240984e-07, "logp/chosen": -218.0, "logp/rejected": -229.0, "loss": 0.607421875, "reward/accuracy": 0.6875, "reward/chosen": -1.90625, "reward/margin": 0.74609375, "reward/rejected": -2.65625, "step": 2791 }, { "approx. KL/chosen": 290.0, "approx. KL/rejected": 1072.0, "epoch": 0.8374325134973005, "grad_norm": 3.5673766136169434, "learning_rate": 6.403429586755055e-07, "logp/chosen": -314.0, "logp/rejected": -340.0, "loss": 0.227081298828125, "reward/accuracy": 0.875, "reward/chosen": -1.8515625, "reward/margin": 2.4375, "reward/rejected": -4.28125, "step": 2792 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 800.0, "epoch": 0.8377324535092981, "grad_norm": 3.7100601196289062, "learning_rate": 6.380380380893231e-07, "logp/chosen": -264.0, "logp/rejected": -274.0, "loss": 0.3763275146484375, "reward/accuracy": 0.75, "reward/chosen": -1.40625, "reward/margin": 2.0625, "reward/rejected": -3.46875, "step": 2793 }, { "approx. KL/chosen": 450.0, "approx. KL/rejected": 490.0, "epoch": 0.8380323935212958, "grad_norm": 6.364771842956543, "learning_rate": 6.357369905289962e-07, "logp/chosen": -251.0, "logp/rejected": -342.0, "loss": 0.947265625, "reward/accuracy": 0.4375, "reward/chosen": -2.578125, "reward/margin": 0.042236328125, "reward/rejected": -2.625, "step": 2794 }, { "approx. KL/chosen": 205.0, "approx. KL/rejected": 716.0, "epoch": 0.8383323335332934, "grad_norm": 2.9860291481018066, "learning_rate": 6.334398180376467e-07, "logp/chosen": -264.0, "logp/rejected": -262.0, "loss": 0.39090728759765625, "reward/accuracy": 0.8125, "reward/chosen": -1.6875, "reward/margin": 1.5, "reward/rejected": -3.1875, "step": 2795 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 628.0, "epoch": 0.8386322735452909, "grad_norm": 5.462041854858398, "learning_rate": 6.31146522654954e-07, "logp/chosen": -320.0, "logp/rejected": -255.0, "loss": 0.58001708984375, "reward/accuracy": 0.625, "reward/chosen": -1.8515625, "reward/margin": 1.265625, "reward/rejected": -3.125, "step": 2796 }, { "approx. KL/chosen": 154.0, "approx. KL/rejected": 628.0, "epoch": 0.8389322135572885, "grad_norm": 3.7167954444885254, "learning_rate": 6.288571064171578e-07, "logp/chosen": -442.0, "logp/rejected": -408.0, "loss": 0.367645263671875, "reward/accuracy": 0.75, "reward/chosen": -1.3671875, "reward/margin": 1.7265625, "reward/rejected": -3.09375, "step": 2797 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 928.0, "epoch": 0.8392321535692862, "grad_norm": 3.0479896068573, "learning_rate": 6.265715713570503e-07, "logp/chosen": -364.0, "logp/rejected": -402.0, "loss": 0.27020263671875, "reward/accuracy": 0.75, "reward/chosen": -1.328125, "reward/margin": 2.609375, "reward/rejected": -3.9375, "step": 2798 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 656.0, "epoch": 0.8395320935812838, "grad_norm": 2.9114036560058594, "learning_rate": 6.242899195039798e-07, "logp/chosen": -436.0, "logp/rejected": -352.0, "loss": 0.3138427734375, "reward/accuracy": 0.8125, "reward/chosen": -1.1875, "reward/margin": 1.8515625, "reward/rejected": -3.03125, "step": 2799 }, { "approx. KL/chosen": 175.0, "approx. KL/rejected": 936.0, "epoch": 0.8398320335932813, "grad_norm": 2.4966561794281006, "learning_rate": 6.220121528838435e-07, "logp/chosen": -402.0, "logp/rejected": -346.0, "loss": 0.32525634765625, "reward/accuracy": 0.8125, "reward/chosen": -1.390625, "reward/margin": 2.0625, "reward/rejected": -3.453125, "step": 2800 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 800.0, "epoch": 0.8401319736052789, "grad_norm": 3.370004892349243, "learning_rate": 6.197382735190943e-07, "logp/chosen": -330.0, "logp/rejected": -302.0, "loss": 0.46954345703125, "reward/accuracy": 0.75, "reward/chosen": -2.0625, "reward/margin": 1.53125, "reward/rejected": -3.59375, "step": 2801 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 796.0, "epoch": 0.8404319136172765, "grad_norm": 3.8649137020111084, "learning_rate": 6.174682834287294e-07, "logp/chosen": -340.0, "logp/rejected": -356.0, "loss": 0.3314971923828125, "reward/accuracy": 0.8125, "reward/chosen": -1.640625, "reward/margin": 2.0, "reward/rejected": -3.640625, "step": 2802 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 868.0, "epoch": 0.8407318536292742, "grad_norm": 3.6385416984558105, "learning_rate": 6.152021846282935e-07, "logp/chosen": -374.0, "logp/rejected": -372.0, "loss": 0.326263427734375, "reward/accuracy": 0.75, "reward/chosen": -1.828125, "reward/margin": 1.8359375, "reward/rejected": -3.65625, "step": 2803 }, { "approx. KL/chosen": 80.5, "approx. KL/rejected": 740.0, "epoch": 0.8410317936412718, "grad_norm": 3.542945623397827, "learning_rate": 6.129399791298751e-07, "logp/chosen": -344.0, "logp/rejected": -386.0, "loss": 0.3028564453125, "reward/accuracy": 0.75, "reward/chosen": -0.953125, "reward/margin": 2.171875, "reward/rejected": -3.125, "step": 2804 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 752.0, "epoch": 0.8413317336532693, "grad_norm": 2.7692203521728516, "learning_rate": 6.106816689421091e-07, "logp/chosen": -342.0, "logp/rejected": -286.0, "loss": 0.3067474365234375, "reward/accuracy": 0.9375, "reward/chosen": -1.265625, "reward/margin": 2.1875, "reward/rejected": -3.453125, "step": 2805 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 732.0, "epoch": 0.8416316736652669, "grad_norm": 2.909884452819824, "learning_rate": 6.084272560701693e-07, "logp/chosen": -324.0, "logp/rejected": -302.0, "loss": 0.3018798828125, "reward/accuracy": 0.8125, "reward/chosen": -1.53125, "reward/margin": 1.8125, "reward/rejected": -3.34375, "step": 2806 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 412.0, "epoch": 0.8419316136772645, "grad_norm": 4.32975435256958, "learning_rate": 6.061767425157694e-07, "logp/chosen": -308.0, "logp/rejected": -264.0, "loss": 0.722137451171875, "reward/accuracy": 0.6875, "reward/chosen": -1.5859375, "reward/margin": 0.83203125, "reward/rejected": -2.421875, "step": 2807 }, { "approx. KL/chosen": 324.0, "approx. KL/rejected": 370.0, "epoch": 0.8422315536892622, "grad_norm": 7.092596530914307, "learning_rate": 6.039301302771594e-07, "logp/chosen": -376.0, "logp/rejected": -314.0, "loss": 0.941650390625, "reward/accuracy": 0.6875, "reward/chosen": -1.828125, "reward/margin": 0.1337890625, "reward/rejected": -1.9609375, "step": 2808 }, { "approx. KL/chosen": 104.0, "approx. KL/rejected": 332.0, "epoch": 0.8425314937012598, "grad_norm": 3.5950286388397217, "learning_rate": 6.016874213491292e-07, "logp/chosen": -408.0, "logp/rejected": -388.0, "loss": 0.4535675048828125, "reward/accuracy": 0.8125, "reward/chosen": -1.09375, "reward/margin": 1.1796875, "reward/rejected": -2.28125, "step": 2809 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 624.0, "epoch": 0.8428314337132573, "grad_norm": 5.38472318649292, "learning_rate": 5.994486177229985e-07, "logp/chosen": -442.0, "logp/rejected": -366.0, "loss": 0.5232620239257812, "reward/accuracy": 0.6875, "reward/chosen": -1.6796875, "reward/margin": 1.2265625, "reward/rejected": -2.90625, "step": 2810 }, { "approx. KL/chosen": 356.0, "approx. KL/rejected": 1008.0, "epoch": 0.8431313737252549, "grad_norm": 5.818995952606201, "learning_rate": 5.972137213866241e-07, "logp/chosen": -354.0, "logp/rejected": -316.0, "loss": 0.5302734375, "reward/accuracy": 0.75, "reward/chosen": -1.9296875, "reward/margin": 1.8515625, "reward/rejected": -3.78125, "step": 2811 }, { "approx. KL/chosen": 212.0, "approx. KL/rejected": 684.0, "epoch": 0.8434313137372526, "grad_norm": 5.670734882354736, "learning_rate": 5.949827343243881e-07, "logp/chosen": -372.0, "logp/rejected": -372.0, "loss": 0.50714111328125, "reward/accuracy": 0.6875, "reward/chosen": -1.6484375, "reward/margin": 1.40625, "reward/rejected": -3.0625, "step": 2812 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 936.0, "epoch": 0.8437312537492502, "grad_norm": 2.554914712905884, "learning_rate": 5.927556585172057e-07, "logp/chosen": -260.0, "logp/rejected": -318.0, "loss": 0.3199462890625, "reward/accuracy": 0.8125, "reward/chosen": -1.3046875, "reward/margin": 2.546875, "reward/rejected": -3.84375, "step": 2813 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 948.0, "epoch": 0.8440311937612478, "grad_norm": 5.507119655609131, "learning_rate": 5.905324959425163e-07, "logp/chosen": -406.0, "logp/rejected": -233.0, "loss": 0.6682281494140625, "reward/accuracy": 0.75, "reward/chosen": -1.578125, "reward/margin": 2.0625, "reward/rejected": -3.640625, "step": 2814 }, { "approx. KL/chosen": 398.0, "approx. KL/rejected": 1168.0, "epoch": 0.8443311337732453, "grad_norm": 4.893433094024658, "learning_rate": 5.883132485742893e-07, "logp/chosen": -306.0, "logp/rejected": -406.0, "loss": 0.4638671875, "reward/accuracy": 0.8125, "reward/chosen": -2.40625, "reward/margin": 2.03125, "reward/rejected": -4.4375, "step": 2815 }, { "approx. KL/chosen": 245.0, "approx. KL/rejected": 744.0, "epoch": 0.8446310737852429, "grad_norm": 4.936428070068359, "learning_rate": 5.860979183830124e-07, "logp/chosen": -362.0, "logp/rejected": -436.0, "loss": 0.369140625, "reward/accuracy": 0.75, "reward/chosen": -1.6796875, "reward/margin": 1.765625, "reward/rejected": -3.4375, "step": 2816 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 728.0, "epoch": 0.8449310137972406, "grad_norm": 3.2892796993255615, "learning_rate": 5.838865073356981e-07, "logp/chosen": -286.0, "logp/rejected": -346.0, "loss": 0.361602783203125, "reward/accuracy": 0.8125, "reward/chosen": -1.4296875, "reward/margin": 1.9140625, "reward/rejected": -3.34375, "step": 2817 }, { "approx. KL/chosen": 322.0, "approx. KL/rejected": 692.0, "epoch": 0.8452309538092382, "grad_norm": 3.9054954051971436, "learning_rate": 5.816790173958776e-07, "logp/chosen": -328.0, "logp/rejected": -318.0, "loss": 0.436248779296875, "reward/accuracy": 0.75, "reward/chosen": -1.9765625, "reward/margin": 1.3046875, "reward/rejected": -3.28125, "step": 2818 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 1408.0, "epoch": 0.8455308938212358, "grad_norm": 2.628596067428589, "learning_rate": 5.794754505236027e-07, "logp/chosen": -308.0, "logp/rejected": -356.0, "loss": 0.2421283721923828, "reward/accuracy": 0.875, "reward/chosen": -1.2421875, "reward/margin": 3.453125, "reward/rejected": -4.6875, "step": 2819 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 868.0, "epoch": 0.8458308338332333, "grad_norm": 5.210336208343506, "learning_rate": 5.772758086754393e-07, "logp/chosen": -324.0, "logp/rejected": -388.0, "loss": 0.756378173828125, "reward/accuracy": 0.625, "reward/chosen": -2.015625, "reward/margin": 1.4296875, "reward/rejected": -3.453125, "step": 2820 }, { "approx. KL/chosen": 222.0, "approx. KL/rejected": 948.0, "epoch": 0.8461307738452309, "grad_norm": 2.992858409881592, "learning_rate": 5.750800938044721e-07, "logp/chosen": -196.0, "logp/rejected": -249.0, "loss": 0.37244129180908203, "reward/accuracy": 0.8125, "reward/chosen": -1.6484375, "reward/margin": 2.03125, "reward/rejected": -3.671875, "step": 2821 }, { "approx. KL/chosen": 197.0, "approx. KL/rejected": 724.0, "epoch": 0.8464307138572286, "grad_norm": 3.503166913986206, "learning_rate": 5.728883078602926e-07, "logp/chosen": -221.0, "logp/rejected": -338.0, "loss": 0.3257904052734375, "reward/accuracy": 0.875, "reward/chosen": -1.46875, "reward/margin": 2.0625, "reward/rejected": -3.53125, "step": 2822 }, { "approx. KL/chosen": 212.0, "approx. KL/rejected": 1064.0, "epoch": 0.8467306538692262, "grad_norm": 2.8769049644470215, "learning_rate": 5.707004527890103e-07, "logp/chosen": -276.0, "logp/rejected": -338.0, "loss": 0.33177995681762695, "reward/accuracy": 0.8125, "reward/chosen": -1.7109375, "reward/margin": 2.34375, "reward/rejected": -4.0625, "step": 2823 }, { "approx. KL/chosen": 396.0, "approx. KL/rejected": 880.0, "epoch": 0.8470305938812237, "grad_norm": 4.311319828033447, "learning_rate": 5.685165305332396e-07, "logp/chosen": -286.0, "logp/rejected": -308.0, "loss": 0.632476806640625, "reward/accuracy": 0.8125, "reward/chosen": -2.234375, "reward/margin": 1.4296875, "reward/rejected": -3.671875, "step": 2824 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 816.0, "epoch": 0.8473305338932213, "grad_norm": 5.029665470123291, "learning_rate": 5.663365430321089e-07, "logp/chosen": -368.0, "logp/rejected": -492.0, "loss": 0.5757999420166016, "reward/accuracy": 0.75, "reward/chosen": -1.609375, "reward/margin": 1.8984375, "reward/rejected": -3.5, "step": 2825 }, { "approx. KL/chosen": 108.5, "approx. KL/rejected": 460.0, "epoch": 0.847630473905219, "grad_norm": 4.192134380340576, "learning_rate": 5.641604922212435e-07, "logp/chosen": -302.0, "logp/rejected": -268.0, "loss": 0.572265625, "reward/accuracy": 0.625, "reward/chosen": -1.21875, "reward/margin": 1.15625, "reward/rejected": -2.375, "step": 2826 }, { "approx. KL/chosen": 422.0, "approx. KL/rejected": 1032.0, "epoch": 0.8479304139172166, "grad_norm": 5.393936634063721, "learning_rate": 5.61988380032783e-07, "logp/chosen": -428.0, "logp/rejected": -320.0, "loss": 0.47052001953125, "reward/accuracy": 0.8125, "reward/chosen": -2.140625, "reward/margin": 1.9296875, "reward/rejected": -4.0625, "step": 2827 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 880.0, "epoch": 0.8482303539292142, "grad_norm": 3.3177778720855713, "learning_rate": 5.59820208395363e-07, "logp/chosen": -378.0, "logp/rejected": -502.0, "loss": 0.38446044921875, "reward/accuracy": 0.875, "reward/chosen": -1.9296875, "reward/margin": 1.515625, "reward/rejected": -3.4375, "step": 2828 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 708.0, "epoch": 0.8485302939412117, "grad_norm": 4.784970760345459, "learning_rate": 5.576559792341257e-07, "logp/chosen": -296.0, "logp/rejected": -306.0, "loss": 0.533935546875, "reward/accuracy": 0.6875, "reward/chosen": -1.8828125, "reward/margin": 1.1875, "reward/rejected": -3.0625, "step": 2829 }, { "approx. KL/chosen": 225.0, "approx. KL/rejected": 540.0, "epoch": 0.8488302339532093, "grad_norm": 5.17057466506958, "learning_rate": 5.554956944707079e-07, "logp/chosen": -492.0, "logp/rejected": -636.0, "loss": 0.611328125, "reward/accuracy": 0.625, "reward/chosen": -1.6171875, "reward/margin": 0.8828125, "reward/rejected": -2.5, "step": 2830 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 788.0, "epoch": 0.849130173965207, "grad_norm": 2.974118709564209, "learning_rate": 5.533393560232469e-07, "logp/chosen": -328.0, "logp/rejected": -346.0, "loss": 0.33831787109375, "reward/accuracy": 0.9375, "reward/chosen": -1.6171875, "reward/margin": 1.953125, "reward/rejected": -3.5625, "step": 2831 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 748.0, "epoch": 0.8494301139772046, "grad_norm": 3.232757806777954, "learning_rate": 5.511869658063739e-07, "logp/chosen": -388.0, "logp/rejected": -494.0, "loss": 0.30560302734375, "reward/accuracy": 0.75, "reward/chosen": -1.6953125, "reward/margin": 1.890625, "reward/rejected": -3.578125, "step": 2832 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 928.0, "epoch": 0.8497300539892022, "grad_norm": 4.308130264282227, "learning_rate": 5.490385257312181e-07, "logp/chosen": -338.0, "logp/rejected": -450.0, "loss": 0.42919921875, "reward/accuracy": 0.75, "reward/chosen": -2.015625, "reward/margin": 1.59375, "reward/rejected": -3.609375, "step": 2833 }, { "approx. KL/chosen": 310.0, "approx. KL/rejected": 928.0, "epoch": 0.8500299940011997, "grad_norm": 5.362013339996338, "learning_rate": 5.468940377053972e-07, "logp/chosen": -348.0, "logp/rejected": -362.0, "loss": 0.5354080200195312, "reward/accuracy": 0.5625, "reward/chosen": -1.875, "reward/margin": 1.5625, "reward/rejected": -3.4375, "step": 2834 }, { "approx. KL/chosen": 127.5, "approx. KL/rejected": 548.0, "epoch": 0.8503299340131973, "grad_norm": 3.878284454345703, "learning_rate": 5.447535036330237e-07, "logp/chosen": -406.0, "logp/rejected": -374.0, "loss": 0.45184326171875, "reward/accuracy": 0.75, "reward/chosen": -0.96875, "reward/margin": 1.6875, "reward/rejected": -2.65625, "step": 2835 }, { "approx. KL/chosen": 304.0, "approx. KL/rejected": 1288.0, "epoch": 0.850629874025195, "grad_norm": 3.354250431060791, "learning_rate": 5.426169254146946e-07, "logp/chosen": -344.0, "logp/rejected": -344.0, "loss": 0.3123779296875, "reward/accuracy": 0.8125, "reward/chosen": -1.8125, "reward/margin": 2.5, "reward/rejected": -4.3125, "step": 2836 }, { "approx. KL/chosen": 188.0, "approx. KL/rejected": 756.0, "epoch": 0.8509298140371926, "grad_norm": 2.8898816108703613, "learning_rate": 5.404843049475e-07, "logp/chosen": -372.0, "logp/rejected": -344.0, "loss": 0.21448516845703125, "reward/accuracy": 0.9375, "reward/chosen": -1.3671875, "reward/margin": 2.09375, "reward/rejected": -3.453125, "step": 2837 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 844.0, "epoch": 0.8512297540491902, "grad_norm": 3.0442261695861816, "learning_rate": 5.383556441250103e-07, "logp/chosen": -230.0, "logp/rejected": -244.0, "loss": 0.2783355712890625, "reward/accuracy": 0.8125, "reward/chosen": -1.59375, "reward/margin": 2.09375, "reward/rejected": -3.6875, "step": 2838 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 640.0, "epoch": 0.8515296940611877, "grad_norm": 3.6199791431427, "learning_rate": 5.362309448372855e-07, "logp/chosen": -382.0, "logp/rejected": -274.0, "loss": 0.544464111328125, "reward/accuracy": 0.6875, "reward/chosen": -1.7890625, "reward/margin": 1.1875, "reward/rejected": -2.96875, "step": 2839 }, { "approx. KL/chosen": 185.0, "approx. KL/rejected": 600.0, "epoch": 0.8518296340731853, "grad_norm": 2.466562509536743, "learning_rate": 5.341102089708644e-07, "logp/chosen": -406.0, "logp/rejected": -270.0, "loss": 0.29229736328125, "reward/accuracy": 0.8125, "reward/chosen": -1.328125, "reward/margin": 1.6015625, "reward/rejected": -2.9375, "step": 2840 }, { "approx. KL/chosen": 318.0, "approx. KL/rejected": 848.0, "epoch": 0.852129574085183, "grad_norm": 4.407772541046143, "learning_rate": 5.319934384087677e-07, "logp/chosen": -302.0, "logp/rejected": -264.0, "loss": 0.56689453125, "reward/accuracy": 0.875, "reward/chosen": -1.90625, "reward/margin": 1.5390625, "reward/rejected": -3.453125, "step": 2841 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 668.0, "epoch": 0.8524295140971806, "grad_norm": 3.79687762260437, "learning_rate": 5.298806350304941e-07, "logp/chosen": -170.0, "logp/rejected": -251.0, "loss": 0.518310546875, "reward/accuracy": 0.75, "reward/chosen": -1.4765625, "reward/margin": 1.4921875, "reward/rejected": -2.96875, "step": 2842 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 1256.0, "epoch": 0.8527294541091782, "grad_norm": 2.2859811782836914, "learning_rate": 5.277718007120236e-07, "logp/chosen": -334.0, "logp/rejected": -388.0, "loss": 0.220977783203125, "reward/accuracy": 0.9375, "reward/chosen": -1.4765625, "reward/margin": 2.8125, "reward/rejected": -4.28125, "step": 2843 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 548.0, "epoch": 0.8530293941211757, "grad_norm": 4.27061653137207, "learning_rate": 5.256669373258067e-07, "logp/chosen": -239.0, "logp/rejected": -332.0, "loss": 0.52972412109375, "reward/accuracy": 0.625, "reward/chosen": -1.84375, "reward/margin": 1.03125, "reward/rejected": -2.875, "step": 2844 }, { "approx. KL/chosen": 235.0, "approx. KL/rejected": 476.0, "epoch": 0.8533293341331734, "grad_norm": 4.720940113067627, "learning_rate": 5.235660467407744e-07, "logp/chosen": -312.0, "logp/rejected": -264.0, "loss": 0.66925048828125, "reward/accuracy": 0.4375, "reward/chosen": -1.78125, "reward/margin": 0.83203125, "reward/rejected": -2.609375, "step": 2845 }, { "approx. KL/chosen": 119.5, "approx. KL/rejected": 672.0, "epoch": 0.853629274145171, "grad_norm": 3.103593587875366, "learning_rate": 5.214691308223224e-07, "logp/chosen": -290.0, "logp/rejected": -292.0, "loss": 0.371429443359375, "reward/accuracy": 0.6875, "reward/chosen": -1.078125, "reward/margin": 2.03125, "reward/rejected": -3.09375, "step": 2846 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 1328.0, "epoch": 0.8539292141571686, "grad_norm": 4.884860038757324, "learning_rate": 5.193761914323248e-07, "logp/chosen": -454.0, "logp/rejected": -332.0, "loss": 0.31964111328125, "reward/accuracy": 0.8125, "reward/chosen": -1.8046875, "reward/margin": 2.875, "reward/rejected": -4.6875, "step": 2847 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 516.0, "epoch": 0.8542291541691661, "grad_norm": 4.259115695953369, "learning_rate": 5.172872304291188e-07, "logp/chosen": -452.0, "logp/rejected": -358.0, "loss": 0.478240966796875, "reward/accuracy": 0.75, "reward/chosen": -1.984375, "reward/margin": 1.046875, "reward/rejected": -3.03125, "step": 2848 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 760.0, "epoch": 0.8545290941811637, "grad_norm": 2.9181840419769287, "learning_rate": 5.152022496675157e-07, "logp/chosen": -304.0, "logp/rejected": -251.0, "loss": 0.292236328125, "reward/accuracy": 0.9375, "reward/chosen": -1.84375, "reward/margin": 1.640625, "reward/rejected": -3.484375, "step": 2849 }, { "approx. KL/chosen": 324.0, "approx. KL/rejected": 816.0, "epoch": 0.8548290341931614, "grad_norm": 4.6002984046936035, "learning_rate": 5.131212509987848e-07, "logp/chosen": -386.0, "logp/rejected": -360.0, "loss": 0.46136474609375, "reward/accuracy": 0.875, "reward/chosen": -1.796875, "reward/margin": 1.7890625, "reward/rejected": -3.578125, "step": 2850 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 1384.0, "epoch": 0.855128974205159, "grad_norm": 4.352561950683594, "learning_rate": 5.110442362706669e-07, "logp/chosen": -446.0, "logp/rejected": -420.0, "loss": 0.353851318359375, "reward/accuracy": 0.75, "reward/chosen": -1.8984375, "reward/margin": 2.578125, "reward/rejected": -4.46875, "step": 2851 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 904.0, "epoch": 0.8554289142171566, "grad_norm": 3.6033272743225098, "learning_rate": 5.089712073273601e-07, "logp/chosen": -386.0, "logp/rejected": -346.0, "loss": 0.4045867919921875, "reward/accuracy": 0.75, "reward/chosen": -2.0, "reward/margin": 1.71875, "reward/rejected": -3.71875, "step": 2852 }, { "approx. KL/chosen": 206.0, "approx. KL/rejected": 576.0, "epoch": 0.8557288542291541, "grad_norm": 4.5239644050598145, "learning_rate": 5.069021660095274e-07, "logp/chosen": -306.0, "logp/rejected": -302.0, "loss": 0.5358123779296875, "reward/accuracy": 0.6875, "reward/chosen": -1.734375, "reward/margin": 1.21875, "reward/rejected": -2.953125, "step": 2853 }, { "approx. KL/chosen": 248.0, "approx. KL/rejected": 980.0, "epoch": 0.8560287942411517, "grad_norm": 5.235565185546875, "learning_rate": 5.04837114154289e-07, "logp/chosen": -264.0, "logp/rejected": -366.0, "loss": 0.78839111328125, "reward/accuracy": 0.4375, "reward/chosen": -1.90625, "reward/margin": 1.4140625, "reward/rejected": -3.3125, "step": 2854 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 1168.0, "epoch": 0.8563287342531494, "grad_norm": 2.6471364498138428, "learning_rate": 5.027760535952236e-07, "logp/chosen": -380.0, "logp/rejected": -382.0, "loss": 0.3270072937011719, "reward/accuracy": 0.9375, "reward/chosen": -1.5859375, "reward/margin": 2.625, "reward/rejected": -4.21875, "step": 2855 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 936.0, "epoch": 0.856628674265147, "grad_norm": 3.5520217418670654, "learning_rate": 5.00718986162364e-07, "logp/chosen": -332.0, "logp/rejected": -424.0, "loss": 0.26629638671875, "reward/accuracy": 0.875, "reward/chosen": -1.796875, "reward/margin": 2.09375, "reward/rejected": -3.890625, "step": 2856 }, { "approx. KL/chosen": 404.0, "approx. KL/rejected": 796.0, "epoch": 0.8569286142771446, "grad_norm": 4.916659832000732, "learning_rate": 4.986659136822014e-07, "logp/chosen": -388.0, "logp/rejected": -438.0, "loss": 0.50665283203125, "reward/accuracy": 0.75, "reward/chosen": -2.5625, "reward/margin": 1.125, "reward/rejected": -3.6875, "step": 2857 }, { "approx. KL/chosen": 187.0, "approx. KL/rejected": 660.0, "epoch": 0.8572285542891421, "grad_norm": 3.24202561378479, "learning_rate": 4.966168379776759e-07, "logp/chosen": -564.0, "logp/rejected": -466.0, "loss": 0.36505126953125, "reward/accuracy": 0.8125, "reward/chosen": -1.5625, "reward/margin": 1.765625, "reward/rejected": -3.328125, "step": 2858 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 624.0, "epoch": 0.8575284943011398, "grad_norm": 4.765718460083008, "learning_rate": 4.945717608681833e-07, "logp/chosen": -372.0, "logp/rejected": -296.0, "loss": 0.42974853515625, "reward/accuracy": 0.875, "reward/chosen": -1.7265625, "reward/margin": 1.375, "reward/rejected": -3.09375, "step": 2859 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 636.0, "epoch": 0.8578284343131374, "grad_norm": 4.985974311828613, "learning_rate": 4.925306841695621e-07, "logp/chosen": -356.0, "logp/rejected": -434.0, "loss": 0.612060546875, "reward/accuracy": 0.625, "reward/chosen": -1.9609375, "reward/margin": 1.0703125, "reward/rejected": -3.03125, "step": 2860 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 748.0, "epoch": 0.858128374325135, "grad_norm": 3.790099620819092, "learning_rate": 4.90493609694106e-07, "logp/chosen": -230.0, "logp/rejected": -280.0, "loss": 0.443206787109375, "reward/accuracy": 0.75, "reward/chosen": -1.3203125, "reward/margin": 1.875, "reward/rejected": -3.1875, "step": 2861 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 498.0, "epoch": 0.8584283143371326, "grad_norm": 5.364766597747803, "learning_rate": 4.884605392505498e-07, "logp/chosen": -368.0, "logp/rejected": -448.0, "loss": 0.60455322265625, "reward/accuracy": 0.5625, "reward/chosen": -1.8125, "reward/margin": 0.9921875, "reward/rejected": -2.8125, "step": 2862 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 576.0, "epoch": 0.8587282543491301, "grad_norm": 5.4586944580078125, "learning_rate": 4.864314746440779e-07, "logp/chosen": -378.0, "logp/rejected": -302.0, "loss": 0.5771636962890625, "reward/accuracy": 0.8125, "reward/chosen": -1.75, "reward/margin": 1.2421875, "reward/rejected": -2.984375, "step": 2863 }, { "approx. KL/chosen": 96.0, "approx. KL/rejected": 400.0, "epoch": 0.8590281943611278, "grad_norm": 3.456618070602417, "learning_rate": 4.844064176763142e-07, "logp/chosen": -398.0, "logp/rejected": -418.0, "loss": 0.4560546875, "reward/accuracy": 0.9375, "reward/chosen": -1.125, "reward/margin": 1.25, "reward/rejected": -2.375, "step": 2864 }, { "approx. KL/chosen": 110.5, "approx. KL/rejected": 430.0, "epoch": 0.8593281343731254, "grad_norm": 3.390285015106201, "learning_rate": 4.823853701453246e-07, "logp/chosen": -400.0, "logp/rejected": -300.0, "loss": 0.358306884765625, "reward/accuracy": 0.8125, "reward/chosen": -0.875, "reward/margin": 1.6875, "reward/rejected": -2.5625, "step": 2865 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 860.0, "epoch": 0.859628074385123, "grad_norm": 4.263490676879883, "learning_rate": 4.803683338456155e-07, "logp/chosen": -338.0, "logp/rejected": -229.0, "loss": 0.3258056640625, "reward/accuracy": 0.875, "reward/chosen": -2.125, "reward/margin": 1.7734375, "reward/rejected": -3.90625, "step": 2866 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 808.0, "epoch": 0.8599280143971205, "grad_norm": 2.821298360824585, "learning_rate": 4.783553105681332e-07, "logp/chosen": -378.0, "logp/rejected": -310.0, "loss": 0.29717254638671875, "reward/accuracy": 0.875, "reward/chosen": -1.5234375, "reward/margin": 2.015625, "reward/rejected": -3.546875, "step": 2867 }, { "approx. KL/chosen": 352.0, "approx. KL/rejected": 580.0, "epoch": 0.8602279544091181, "grad_norm": 5.216100692749023, "learning_rate": 4.7634630210025845e-07, "logp/chosen": -490.0, "logp/rejected": -364.0, "loss": 0.5130615234375, "reward/accuracy": 0.75, "reward/chosen": -1.890625, "reward/margin": 1.171875, "reward/rejected": -3.0625, "step": 2868 }, { "approx. KL/chosen": 213.0, "approx. KL/rejected": 532.0, "epoch": 0.8605278944211158, "grad_norm": 3.7153477668762207, "learning_rate": 4.7434131022580875e-07, "logp/chosen": -342.0, "logp/rejected": -304.0, "loss": 0.548583984375, "reward/accuracy": 0.8125, "reward/chosen": -1.375, "reward/margin": 1.4140625, "reward/rejected": -2.796875, "step": 2869 }, { "approx. KL/chosen": 202.0, "approx. KL/rejected": 628.0, "epoch": 0.8608278344331134, "grad_norm": 3.573979377746582, "learning_rate": 4.72340336725034e-07, "logp/chosen": -314.0, "logp/rejected": -362.0, "loss": 0.5389404296875, "reward/accuracy": 0.8125, "reward/chosen": -1.4609375, "reward/margin": 1.53125, "reward/rejected": -3.0, "step": 2870 }, { "approx. KL/chosen": 352.0, "approx. KL/rejected": 572.0, "epoch": 0.861127774445111, "grad_norm": 4.115719318389893, "learning_rate": 4.7034338337461737e-07, "logp/chosen": -280.0, "logp/rejected": -256.0, "loss": 0.5224609375, "reward/accuracy": 0.625, "reward/chosen": -2.0625, "reward/margin": 0.8046875, "reward/rejected": -2.875, "step": 2871 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 740.0, "epoch": 0.8614277144571085, "grad_norm": 3.7533140182495117, "learning_rate": 4.6835045194767113e-07, "logp/chosen": -354.0, "logp/rejected": -284.0, "loss": 0.39261627197265625, "reward/accuracy": 0.8125, "reward/chosen": -1.6875, "reward/margin": 1.53125, "reward/rejected": -3.21875, "step": 2872 }, { "approx. KL/chosen": 184.0, "approx. KL/rejected": 700.0, "epoch": 0.8617276544691062, "grad_norm": 3.6274139881134033, "learning_rate": 4.6636154421374034e-07, "logp/chosen": -213.0, "logp/rejected": -274.0, "loss": 0.3662567138671875, "reward/accuracy": 0.8125, "reward/chosen": -1.4921875, "reward/margin": 1.8984375, "reward/rejected": -3.390625, "step": 2873 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 876.0, "epoch": 0.8620275944811038, "grad_norm": 4.256228446960449, "learning_rate": 4.643766619387896e-07, "logp/chosen": -282.0, "logp/rejected": -214.0, "loss": 0.49810791015625, "reward/accuracy": 0.625, "reward/chosen": -1.6796875, "reward/margin": 1.8359375, "reward/rejected": -3.515625, "step": 2874 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 820.0, "epoch": 0.8623275344931014, "grad_norm": 3.4117610454559326, "learning_rate": 4.6239580688521745e-07, "logp/chosen": -344.0, "logp/rejected": -406.0, "loss": 0.34503173828125, "reward/accuracy": 0.8125, "reward/chosen": -1.59375, "reward/margin": 1.7890625, "reward/rejected": -3.390625, "step": 2875 }, { "approx. KL/chosen": 260.0, "approx. KL/rejected": 740.0, "epoch": 0.862627474505099, "grad_norm": 9.341109275817871, "learning_rate": 4.604189808118414e-07, "logp/chosen": -442.0, "logp/rejected": -524.0, "loss": 0.68505859375, "reward/accuracy": 0.625, "reward/chosen": -1.9609375, "reward/margin": 1.265625, "reward/rejected": -3.21875, "step": 2876 }, { "approx. KL/chosen": 272.0, "approx. KL/rejected": 1112.0, "epoch": 0.8629274145170965, "grad_norm": 3.0977649688720703, "learning_rate": 4.584461854739053e-07, "logp/chosen": -270.0, "logp/rejected": -314.0, "loss": 0.2629890441894531, "reward/accuracy": 0.9375, "reward/chosen": -2.015625, "reward/margin": 2.28125, "reward/rejected": -4.28125, "step": 2877 }, { "approx. KL/chosen": 354.0, "approx. KL/rejected": 712.0, "epoch": 0.8632273545290942, "grad_norm": 4.323110580444336, "learning_rate": 4.564774226230706e-07, "logp/chosen": -362.0, "logp/rejected": -298.0, "loss": 0.58331298828125, "reward/accuracy": 0.75, "reward/chosen": -2.046875, "reward/margin": 1.2109375, "reward/rejected": -3.25, "step": 2878 }, { "approx. KL/chosen": 620.0, "approx. KL/rejected": 588.0, "epoch": 0.8635272945410918, "grad_norm": 7.240518569946289, "learning_rate": 4.5451269400742016e-07, "logp/chosen": -316.0, "logp/rejected": -372.0, "loss": 0.846435546875, "reward/accuracy": 0.5625, "reward/chosen": -2.8125, "reward/margin": 0.1953125, "reward/rejected": -3.0, "step": 2879 }, { "approx. KL/chosen": 215.0, "approx. KL/rejected": 948.0, "epoch": 0.8638272345530894, "grad_norm": 4.9943647384643555, "learning_rate": 4.5255200137145404e-07, "logp/chosen": -255.0, "logp/rejected": -296.0, "loss": 0.35433197021484375, "reward/accuracy": 0.8125, "reward/chosen": -1.5078125, "reward/margin": 2.1875, "reward/rejected": -3.703125, "step": 2880 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 624.0, "epoch": 0.864127174565087, "grad_norm": 3.9053890705108643, "learning_rate": 4.505953464560897e-07, "logp/chosen": -360.0, "logp/rejected": -352.0, "loss": 0.4306640625, "reward/accuracy": 0.75, "reward/chosen": -1.5078125, "reward/margin": 1.3671875, "reward/rejected": -2.875, "step": 2881 }, { "approx. KL/chosen": 229.0, "approx. KL/rejected": 1208.0, "epoch": 0.8644271145770845, "grad_norm": 2.7226104736328125, "learning_rate": 4.486427309986585e-07, "logp/chosen": -294.0, "logp/rejected": -354.0, "loss": 0.2921142578125, "reward/accuracy": 0.875, "reward/chosen": -1.2421875, "reward/margin": 2.96875, "reward/rejected": -4.21875, "step": 2882 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 612.0, "epoch": 0.8647270545890822, "grad_norm": 3.214592218399048, "learning_rate": 4.466941567329075e-07, "logp/chosen": -372.0, "logp/rejected": -286.0, "loss": 0.409912109375, "reward/accuracy": 0.8125, "reward/chosen": -1.4765625, "reward/margin": 1.484375, "reward/rejected": -2.953125, "step": 2883 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 462.0, "epoch": 0.8650269946010798, "grad_norm": 6.903348922729492, "learning_rate": 4.447496253889899e-07, "logp/chosen": -372.0, "logp/rejected": -498.0, "loss": 0.760498046875, "reward/accuracy": 0.625, "reward/chosen": -1.734375, "reward/margin": 0.65234375, "reward/rejected": -2.390625, "step": 2884 }, { "approx. KL/chosen": 57.25, "approx. KL/rejected": 652.0, "epoch": 0.8653269346130774, "grad_norm": 3.4505951404571533, "learning_rate": 4.428091386934763e-07, "logp/chosen": -340.0, "logp/rejected": -334.0, "loss": 0.342041015625, "reward/accuracy": 0.8125, "reward/chosen": -0.78125, "reward/margin": 2.0, "reward/rejected": -2.78125, "step": 2885 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 852.0, "epoch": 0.865626874625075, "grad_norm": 3.9771671295166016, "learning_rate": 4.4087269836933955e-07, "logp/chosen": -462.0, "logp/rejected": -390.0, "loss": 0.327117919921875, "reward/accuracy": 0.8125, "reward/chosen": -1.671875, "reward/margin": 2.15625, "reward/rejected": -3.828125, "step": 2886 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 596.0, "epoch": 0.8659268146370726, "grad_norm": 4.963085651397705, "learning_rate": 4.389403061359654e-07, "logp/chosen": -348.0, "logp/rejected": -338.0, "loss": 0.57769775390625, "reward/accuracy": 0.625, "reward/chosen": -1.6328125, "reward/margin": 1.4765625, "reward/rejected": -3.109375, "step": 2887 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 848.0, "epoch": 0.8662267546490702, "grad_norm": 2.905548334121704, "learning_rate": 4.370119637091413e-07, "logp/chosen": -304.0, "logp/rejected": -255.0, "loss": 0.302825927734375, "reward/accuracy": 0.9375, "reward/chosen": -1.5078125, "reward/margin": 1.875, "reward/rejected": -3.375, "step": 2888 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 864.0, "epoch": 0.8665266946610678, "grad_norm": 2.8846709728240967, "learning_rate": 4.3508767280106e-07, "logp/chosen": -342.0, "logp/rejected": -350.0, "loss": 0.320465087890625, "reward/accuracy": 0.8125, "reward/chosen": -1.953125, "reward/margin": 1.6953125, "reward/rejected": -3.65625, "step": 2889 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 1128.0, "epoch": 0.8668266346730654, "grad_norm": 4.827239036560059, "learning_rate": 4.3316743512031687e-07, "logp/chosen": -384.0, "logp/rejected": -428.0, "loss": 0.41766357421875, "reward/accuracy": 0.8125, "reward/chosen": -2.15625, "reward/margin": 2.140625, "reward/rejected": -4.28125, "step": 2890 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 772.0, "epoch": 0.8671265746850629, "grad_norm": 4.150543689727783, "learning_rate": 4.3125125237190923e-07, "logp/chosen": -294.0, "logp/rejected": -366.0, "loss": 0.388519287109375, "reward/accuracy": 0.875, "reward/chosen": -1.4140625, "reward/margin": 1.8359375, "reward/rejected": -3.25, "step": 2891 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 548.0, "epoch": 0.8674265146970606, "grad_norm": 4.674097061157227, "learning_rate": 4.2933912625723275e-07, "logp/chosen": -328.0, "logp/rejected": -358.0, "loss": 0.624969482421875, "reward/accuracy": 0.6875, "reward/chosen": -1.3828125, "reward/margin": 1.1484375, "reward/rejected": -2.53125, "step": 2892 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 952.0, "epoch": 0.8677264547090582, "grad_norm": 2.279299259185791, "learning_rate": 4.2743105847408205e-07, "logp/chosen": -380.0, "logp/rejected": -280.0, "loss": 0.205780029296875, "reward/accuracy": 0.9375, "reward/chosen": -1.34375, "reward/margin": 2.421875, "reward/rejected": -3.765625, "step": 2893 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 944.0, "epoch": 0.8680263947210558, "grad_norm": 2.535492181777954, "learning_rate": 4.255270507166465e-07, "logp/chosen": -328.0, "logp/rejected": -318.0, "loss": 0.215728759765625, "reward/accuracy": 1.0, "reward/chosen": -1.640625, "reward/margin": 2.25, "reward/rejected": -3.890625, "step": 2894 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 752.0, "epoch": 0.8683263347330534, "grad_norm": 5.178339004516602, "learning_rate": 4.2362710467551414e-07, "logp/chosen": -231.0, "logp/rejected": -322.0, "loss": 0.58984375, "reward/accuracy": 0.75, "reward/chosen": -1.9765625, "reward/margin": 1.375, "reward/rejected": -3.34375, "step": 2895 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 640.0, "epoch": 0.8686262747450509, "grad_norm": 5.526851177215576, "learning_rate": 4.2173122203766236e-07, "logp/chosen": -490.0, "logp/rejected": -296.0, "loss": 0.52777099609375, "reward/accuracy": 0.75, "reward/chosen": -1.53125, "reward/margin": 1.390625, "reward/rejected": -2.921875, "step": 2896 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 1064.0, "epoch": 0.8689262147570486, "grad_norm": 2.219268321990967, "learning_rate": 4.198394044864651e-07, "logp/chosen": -226.0, "logp/rejected": -326.0, "loss": 0.199462890625, "reward/accuracy": 1.0, "reward/chosen": -1.5625, "reward/margin": 2.359375, "reward/rejected": -3.921875, "step": 2897 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 604.0, "epoch": 0.8692261547690462, "grad_norm": 14.281905174255371, "learning_rate": 4.179516537016814e-07, "logp/chosen": -346.0, "logp/rejected": -332.0, "loss": 0.6336593627929688, "reward/accuracy": 0.625, "reward/chosen": -1.4375, "reward/margin": 1.53125, "reward/rejected": -2.96875, "step": 2898 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 572.0, "epoch": 0.8695260947810438, "grad_norm": 4.677883625030518, "learning_rate": 4.160679713594651e-07, "logp/chosen": -494.0, "logp/rejected": -412.0, "loss": 0.53363037109375, "reward/accuracy": 0.625, "reward/chosen": -1.7734375, "reward/margin": 1.234375, "reward/rejected": -3.015625, "step": 2899 }, { "approx. KL/chosen": 314.0, "approx. KL/rejected": 724.0, "epoch": 0.8698260347930414, "grad_norm": 4.691015243530273, "learning_rate": 4.1418835913235256e-07, "logp/chosen": -368.0, "logp/rejected": -490.0, "loss": 0.5174560546875, "reward/accuracy": 0.75, "reward/chosen": -1.7421875, "reward/margin": 1.5234375, "reward/rejected": -3.265625, "step": 2900 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 524.0, "epoch": 0.870125974805039, "grad_norm": 3.794253349304199, "learning_rate": 4.1231281868927107e-07, "logp/chosen": -249.0, "logp/rejected": -238.0, "loss": 0.4866943359375, "reward/accuracy": 0.6875, "reward/chosen": -1.265625, "reward/margin": 1.2109375, "reward/rejected": -2.46875, "step": 2901 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 640.0, "epoch": 0.8704259148170366, "grad_norm": 5.371550559997559, "learning_rate": 4.1044135169552913e-07, "logp/chosen": -288.0, "logp/rejected": -314.0, "loss": 0.6356201171875, "reward/accuracy": 0.6875, "reward/chosen": -1.7109375, "reward/margin": 1.296875, "reward/rejected": -3.015625, "step": 2902 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 948.0, "epoch": 0.8707258548290342, "grad_norm": 2.458890914916992, "learning_rate": 4.085739598128191e-07, "logp/chosen": -304.0, "logp/rejected": -344.0, "loss": 0.30410003662109375, "reward/accuracy": 0.8125, "reward/chosen": -1.4921875, "reward/margin": 2.359375, "reward/rejected": -3.859375, "step": 2903 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 1288.0, "epoch": 0.8710257948410318, "grad_norm": 4.284444808959961, "learning_rate": 4.067106446992142e-07, "logp/chosen": -223.0, "logp/rejected": -306.0, "loss": 0.606689453125, "reward/accuracy": 0.6875, "reward/chosen": -1.7734375, "reward/margin": 2.28125, "reward/rejected": -4.0625, "step": 2904 }, { "approx. KL/chosen": 122.5, "approx. KL/rejected": 540.0, "epoch": 0.8713257348530294, "grad_norm": 4.285706996917725, "learning_rate": 4.0485140800917146e-07, "logp/chosen": -424.0, "logp/rejected": -300.0, "loss": 0.43603515625, "reward/accuracy": 0.75, "reward/chosen": -1.0625, "reward/margin": 1.6484375, "reward/rejected": -2.71875, "step": 2905 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 864.0, "epoch": 0.871625674865027, "grad_norm": 4.510179042816162, "learning_rate": 4.029962513935215e-07, "logp/chosen": -272.0, "logp/rejected": -318.0, "loss": 0.431304931640625, "reward/accuracy": 0.75, "reward/chosen": -1.7578125, "reward/margin": 1.90625, "reward/rejected": -3.65625, "step": 2906 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 540.0, "epoch": 0.8719256148770246, "grad_norm": 3.824429750442505, "learning_rate": 4.011451764994784e-07, "logp/chosen": -304.0, "logp/rejected": -330.0, "loss": 0.42340087890625, "reward/accuracy": 0.75, "reward/chosen": -1.3125, "reward/margin": 1.546875, "reward/rejected": -2.859375, "step": 2907 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 528.0, "epoch": 0.8722255548890222, "grad_norm": 2.9740865230560303, "learning_rate": 3.99298184970624e-07, "logp/chosen": -306.0, "logp/rejected": -288.0, "loss": 0.329864501953125, "reward/accuracy": 0.9375, "reward/chosen": -1.421875, "reward/margin": 1.484375, "reward/rejected": -2.90625, "step": 2908 }, { "approx. KL/chosen": 245.0, "approx. KL/rejected": 732.0, "epoch": 0.8725254949010198, "grad_norm": 4.119333267211914, "learning_rate": 3.974552784469221e-07, "logp/chosen": -308.0, "logp/rejected": -288.0, "loss": 0.503662109375, "reward/accuracy": 0.75, "reward/chosen": -1.6875, "reward/margin": 1.5, "reward/rejected": -3.1875, "step": 2909 }, { "approx. KL/chosen": 152.0, "approx. KL/rejected": 792.0, "epoch": 0.8728254349130175, "grad_norm": 3.7259621620178223, "learning_rate": 3.9561645856470433e-07, "logp/chosen": -388.0, "logp/rejected": -306.0, "loss": 0.33270263671875, "reward/accuracy": 0.75, "reward/chosen": -1.4140625, "reward/margin": 2.09375, "reward/rejected": -3.5, "step": 2910 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 656.0, "epoch": 0.873125374925015, "grad_norm": 4.8792243003845215, "learning_rate": 3.937817269566774e-07, "logp/chosen": -524.0, "logp/rejected": -360.0, "loss": 0.498260498046875, "reward/accuracy": 0.75, "reward/chosen": -1.5, "reward/margin": 1.5703125, "reward/rejected": -3.0625, "step": 2911 }, { "approx. KL/chosen": 376.0, "approx. KL/rejected": 880.0, "epoch": 0.8734253149370126, "grad_norm": 5.002411365509033, "learning_rate": 3.919510852519154e-07, "logp/chosen": -498.0, "logp/rejected": -516.0, "loss": 0.52032470703125, "reward/accuracy": 0.75, "reward/chosen": -1.7890625, "reward/margin": 1.8828125, "reward/rejected": -3.671875, "step": 2912 }, { "approx. KL/chosen": 218.0, "approx. KL/rejected": 1004.0, "epoch": 0.8737252549490102, "grad_norm": 2.6351797580718994, "learning_rate": 3.90124535075862e-07, "logp/chosen": -278.0, "logp/rejected": -249.0, "loss": 0.3188934326171875, "reward/accuracy": 0.875, "reward/chosen": -1.6640625, "reward/margin": 2.265625, "reward/rejected": -3.9375, "step": 2913 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 836.0, "epoch": 0.8740251949610078, "grad_norm": 3.9406821727752686, "learning_rate": 3.883020780503266e-07, "logp/chosen": -438.0, "logp/rejected": -398.0, "loss": 0.50115966796875, "reward/accuracy": 0.6875, "reward/chosen": -1.3359375, "reward/margin": 1.9453125, "reward/rejected": -3.28125, "step": 2914 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 752.0, "epoch": 0.8743251349730053, "grad_norm": 4.856829643249512, "learning_rate": 3.8648371579348753e-07, "logp/chosen": -310.0, "logp/rejected": -346.0, "loss": 0.5919189453125, "reward/accuracy": 0.8125, "reward/chosen": -1.9453125, "reward/margin": 1.4296875, "reward/rejected": -3.375, "step": 2915 }, { "approx. KL/chosen": 298.0, "approx. KL/rejected": 816.0, "epoch": 0.874625074985003, "grad_norm": 3.3257129192352295, "learning_rate": 3.8466944991988384e-07, "logp/chosen": -370.0, "logp/rejected": -372.0, "loss": 0.32061767578125, "reward/accuracy": 0.8125, "reward/chosen": -1.8828125, "reward/margin": 1.8515625, "reward/rejected": -3.734375, "step": 2916 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 904.0, "epoch": 0.8749250149970006, "grad_norm": 3.064378261566162, "learning_rate": 3.828592820404192e-07, "logp/chosen": -390.0, "logp/rejected": -426.0, "loss": 0.345947265625, "reward/accuracy": 0.75, "reward/chosen": -1.8984375, "reward/margin": 1.8828125, "reward/rejected": -3.78125, "step": 2917 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 1032.0, "epoch": 0.8752249550089982, "grad_norm": 2.897329807281494, "learning_rate": 3.810532137623574e-07, "logp/chosen": -288.0, "logp/rejected": -328.0, "loss": 0.3990936279296875, "reward/accuracy": 0.875, "reward/chosen": -1.5859375, "reward/margin": 2.21875, "reward/rejected": -3.8125, "step": 2918 }, { "approx. KL/chosen": 114.5, "approx. KL/rejected": 912.0, "epoch": 0.8755248950209958, "grad_norm": 3.247375249862671, "learning_rate": 3.7925124668932444e-07, "logp/chosen": -306.0, "logp/rejected": -400.0, "loss": 0.4777679443359375, "reward/accuracy": 0.75, "reward/chosen": -1.15625, "reward/margin": 2.078125, "reward/rejected": -3.234375, "step": 2919 }, { "approx. KL/chosen": 316.0, "approx. KL/rejected": 1160.0, "epoch": 0.8758248350329934, "grad_norm": 2.929776430130005, "learning_rate": 3.774533824213017e-07, "logp/chosen": -314.0, "logp/rejected": -322.0, "loss": 0.27094078063964844, "reward/accuracy": 0.875, "reward/chosen": -1.9453125, "reward/margin": 2.546875, "reward/rejected": -4.5, "step": 2920 }, { "approx. KL/chosen": 255.0, "approx. KL/rejected": 672.0, "epoch": 0.876124775044991, "grad_norm": 4.19946813583374, "learning_rate": 3.7565962255463164e-07, "logp/chosen": -442.0, "logp/rejected": -412.0, "loss": 0.61370849609375, "reward/accuracy": 0.625, "reward/chosen": -1.984375, "reward/margin": 0.98046875, "reward/rejected": -2.96875, "step": 2921 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 362.0, "epoch": 0.8764247150569886, "grad_norm": 3.6971566677093506, "learning_rate": 3.738699686820074e-07, "logp/chosen": -211.0, "logp/rejected": -207.0, "loss": 0.52691650390625, "reward/accuracy": 0.625, "reward/chosen": -1.3125, "reward/margin": 0.9765625, "reward/rejected": -2.28125, "step": 2922 }, { "approx. KL/chosen": 292.0, "approx. KL/rejected": 1240.0, "epoch": 0.8767246550689862, "grad_norm": 3.9154255390167236, "learning_rate": 3.720844223924802e-07, "logp/chosen": -360.0, "logp/rejected": -652.0, "loss": 0.29538726806640625, "reward/accuracy": 0.875, "reward/chosen": -1.71875, "reward/margin": 2.734375, "reward/rejected": -4.4375, "step": 2923 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 504.0, "epoch": 0.8770245950809838, "grad_norm": 5.572096347808838, "learning_rate": 3.7030298527145236e-07, "logp/chosen": -282.0, "logp/rejected": -368.0, "loss": 0.5660247802734375, "reward/accuracy": 0.5, "reward/chosen": -1.5703125, "reward/margin": 1.109375, "reward/rejected": -2.6875, "step": 2924 }, { "approx. KL/chosen": 350.0, "approx. KL/rejected": 772.0, "epoch": 0.8773245350929814, "grad_norm": 4.352794647216797, "learning_rate": 3.6852565890067915e-07, "logp/chosen": -370.0, "logp/rejected": -368.0, "loss": 0.41656494140625, "reward/accuracy": 0.75, "reward/chosen": -1.875, "reward/margin": 1.65625, "reward/rejected": -3.53125, "step": 2925 }, { "approx. KL/chosen": 410.0, "approx. KL/rejected": 1264.0, "epoch": 0.877624475104979, "grad_norm": 3.7591190338134766, "learning_rate": 3.6675244485826487e-07, "logp/chosen": -260.0, "logp/rejected": -356.0, "loss": 0.41815185546875, "reward/accuracy": 0.75, "reward/chosen": -2.359375, "reward/margin": 1.8671875, "reward/rejected": -4.21875, "step": 2926 }, { "approx. KL/chosen": 221.0, "approx. KL/rejected": 364.0, "epoch": 0.8779244151169766, "grad_norm": 5.067882537841797, "learning_rate": 3.649833447186618e-07, "logp/chosen": -258.0, "logp/rejected": -258.0, "loss": 0.63323974609375, "reward/accuracy": 0.5625, "reward/chosen": -1.5625, "reward/margin": 0.74609375, "reward/rejected": -2.3125, "step": 2927 }, { "approx. KL/chosen": 154.0, "approx. KL/rejected": 1112.0, "epoch": 0.8782243551289742, "grad_norm": 2.3574225902557373, "learning_rate": 3.6321836005266896e-07, "logp/chosen": -332.0, "logp/rejected": -384.0, "loss": 0.2674980163574219, "reward/accuracy": 0.875, "reward/chosen": -1.4921875, "reward/margin": 2.609375, "reward/rejected": -4.09375, "step": 2928 }, { "approx. KL/chosen": 237.0, "approx. KL/rejected": 512.0, "epoch": 0.8785242951409719, "grad_norm": 5.530925750732422, "learning_rate": 3.61457492427435e-07, "logp/chosen": -370.0, "logp/rejected": -378.0, "loss": 0.61785888671875, "reward/accuracy": 0.75, "reward/chosen": -1.4375, "reward/margin": 1.0703125, "reward/rejected": -2.515625, "step": 2929 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 888.0, "epoch": 0.8788242351529694, "grad_norm": 4.401627063751221, "learning_rate": 3.597007434064487e-07, "logp/chosen": -322.0, "logp/rejected": -312.0, "loss": 0.506591796875, "reward/accuracy": 0.625, "reward/chosen": -1.5546875, "reward/margin": 1.5859375, "reward/rejected": -3.140625, "step": 2930 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 632.0, "epoch": 0.879124175164967, "grad_norm": 3.4141924381256104, "learning_rate": 3.579481145495439e-07, "logp/chosen": -288.0, "logp/rejected": -290.0, "loss": 0.333251953125, "reward/accuracy": 0.75, "reward/chosen": -1.203125, "reward/margin": 1.90625, "reward/rejected": -3.109375, "step": 2931 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 984.0, "epoch": 0.8794241151769646, "grad_norm": 3.846097946166992, "learning_rate": 3.561996074128948e-07, "logp/chosen": -338.0, "logp/rejected": -356.0, "loss": 0.45562744140625, "reward/accuracy": 0.75, "reward/chosen": -1.8359375, "reward/margin": 1.90625, "reward/rejected": -3.75, "step": 2932 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 1280.0, "epoch": 0.8797240551889622, "grad_norm": 1.135872721672058, "learning_rate": 3.544552235490184e-07, "logp/chosen": -248.0, "logp/rejected": -376.0, "loss": 0.09079742431640625, "reward/accuracy": 1.0, "reward/chosen": -1.390625, "reward/margin": 3.1875, "reward/rejected": -4.59375, "step": 2933 }, { "approx. KL/chosen": 354.0, "approx. KL/rejected": 908.0, "epoch": 0.8800239952009599, "grad_norm": 3.7841975688934326, "learning_rate": 3.527149645067679e-07, "logp/chosen": -251.0, "logp/rejected": -406.0, "loss": 0.52362060546875, "reward/accuracy": 0.8125, "reward/chosen": -2.0, "reward/margin": 1.34375, "reward/rejected": -3.34375, "step": 2934 }, { "approx. KL/chosen": 238.0, "approx. KL/rejected": 848.0, "epoch": 0.8803239352129574, "grad_norm": 3.2100470066070557, "learning_rate": 3.509788318313367e-07, "logp/chosen": -280.0, "logp/rejected": -302.0, "loss": 0.3564453125, "reward/accuracy": 0.75, "reward/chosen": -1.7578125, "reward/margin": 1.9453125, "reward/rejected": -3.703125, "step": 2935 }, { "approx. KL/chosen": 225.0, "approx. KL/rejected": 480.0, "epoch": 0.880623875224955, "grad_norm": 4.397668838500977, "learning_rate": 3.4924682706425185e-07, "logp/chosen": -302.0, "logp/rejected": -245.0, "loss": 0.5557861328125, "reward/accuracy": 0.6875, "reward/chosen": -1.8828125, "reward/margin": 0.671875, "reward/rejected": -2.546875, "step": 2936 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 564.0, "epoch": 0.8809238152369526, "grad_norm": 4.51069450378418, "learning_rate": 3.475189517433758e-07, "logp/chosen": -270.0, "logp/rejected": -286.0, "loss": 0.45404052734375, "reward/accuracy": 0.75, "reward/chosen": -1.40625, "reward/margin": 1.578125, "reward/rejected": -2.984375, "step": 2937 }, { "approx. KL/chosen": 684.0, "approx. KL/rejected": 792.0, "epoch": 0.8812237552489502, "grad_norm": 6.974018096923828, "learning_rate": 3.4579520740290474e-07, "logp/chosen": -268.0, "logp/rejected": -249.0, "loss": 0.87939453125, "reward/accuracy": 0.625, "reward/chosen": -3.015625, "reward/margin": 0.58984375, "reward/rejected": -3.609375, "step": 2938 }, { "approx. KL/chosen": 94.0, "approx. KL/rejected": 444.0, "epoch": 0.8815236952609478, "grad_norm": 3.941584587097168, "learning_rate": 3.4407559557336825e-07, "logp/chosen": -298.0, "logp/rejected": -336.0, "loss": 0.4530029296875, "reward/accuracy": 0.6875, "reward/chosen": -1.140625, "reward/margin": 1.1953125, "reward/rejected": -2.328125, "step": 2939 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 616.0, "epoch": 0.8818236352729454, "grad_norm": 5.328302383422852, "learning_rate": 3.423601177816249e-07, "logp/chosen": -448.0, "logp/rejected": -394.0, "loss": 0.67303466796875, "reward/accuracy": 0.625, "reward/chosen": -1.359375, "reward/margin": 1.2890625, "reward/rejected": -2.640625, "step": 2940 }, { "approx. KL/chosen": 502.0, "approx. KL/rejected": 696.0, "epoch": 0.882123575284943, "grad_norm": 5.382233619689941, "learning_rate": 3.406487755508625e-07, "logp/chosen": -406.0, "logp/rejected": -440.0, "loss": 0.607666015625, "reward/accuracy": 0.625, "reward/chosen": -2.421875, "reward/margin": 0.8203125, "reward/rejected": -3.25, "step": 2941 }, { "approx. KL/chosen": 410.0, "approx. KL/rejected": 888.0, "epoch": 0.8824235152969406, "grad_norm": 6.253288745880127, "learning_rate": 3.3894157040059716e-07, "logp/chosen": -326.0, "logp/rejected": -398.0, "loss": 0.645263671875, "reward/accuracy": 0.6875, "reward/chosen": -2.140625, "reward/margin": 1.6328125, "reward/rejected": -3.765625, "step": 2942 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 880.0, "epoch": 0.8827234553089383, "grad_norm": 2.915219783782959, "learning_rate": 3.37238503846673e-07, "logp/chosen": -314.0, "logp/rejected": -328.0, "loss": 0.35626220703125, "reward/accuracy": 0.6875, "reward/chosen": -1.390625, "reward/margin": 2.0, "reward/rejected": -3.390625, "step": 2943 }, { "approx. KL/chosen": 67.5, "approx. KL/rejected": 644.0, "epoch": 0.8830233953209358, "grad_norm": 2.6980831623077393, "learning_rate": 3.355395774012582e-07, "logp/chosen": -340.0, "logp/rejected": -408.0, "loss": 0.183441162109375, "reward/accuracy": 1.0, "reward/chosen": -0.828125, "reward/margin": 2.5, "reward/rejected": -3.328125, "step": 2944 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 624.0, "epoch": 0.8833233353329334, "grad_norm": 4.28921365737915, "learning_rate": 3.33844792572845e-07, "logp/chosen": -310.0, "logp/rejected": -340.0, "loss": 0.3911285400390625, "reward/accuracy": 0.9375, "reward/chosen": -1.3203125, "reward/margin": 1.8125, "reward/rejected": -3.125, "step": 2945 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 756.0, "epoch": 0.883623275344931, "grad_norm": 3.4430932998657227, "learning_rate": 3.321541508662468e-07, "logp/chosen": -342.0, "logp/rejected": -354.0, "loss": 0.465087890625, "reward/accuracy": 0.75, "reward/chosen": -1.3984375, "reward/margin": 1.6015625, "reward/rejected": -3.0, "step": 2946 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 836.0, "epoch": 0.8839232153569286, "grad_norm": 4.660454273223877, "learning_rate": 3.3046765378260236e-07, "logp/chosen": -314.0, "logp/rejected": -302.0, "loss": 0.6624755859375, "reward/accuracy": 0.5625, "reward/chosen": -2.21875, "reward/margin": 1.3125, "reward/rejected": -3.53125, "step": 2947 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 498.0, "epoch": 0.8842231553689263, "grad_norm": 4.79371976852417, "learning_rate": 3.287853028193666e-07, "logp/chosen": -382.0, "logp/rejected": -366.0, "loss": 0.54364013671875, "reward/accuracy": 0.75, "reward/chosen": -1.8203125, "reward/margin": 0.98828125, "reward/rejected": -2.8125, "step": 2948 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 652.0, "epoch": 0.8845230953809238, "grad_norm": 3.531710147857666, "learning_rate": 3.2710709947031296e-07, "logp/chosen": -316.0, "logp/rejected": -328.0, "loss": 0.31585693359375, "reward/accuracy": 0.75, "reward/chosen": -1.4609375, "reward/margin": 1.8515625, "reward/rejected": -3.3125, "step": 2949 }, { "approx. KL/chosen": 142.0, "approx. KL/rejected": 712.0, "epoch": 0.8848230353929214, "grad_norm": 4.800008773803711, "learning_rate": 3.254330452255361e-07, "logp/chosen": -644.0, "logp/rejected": -360.0, "loss": 0.427490234375, "reward/accuracy": 0.8125, "reward/chosen": -1.34375, "reward/margin": 1.75, "reward/rejected": -3.09375, "step": 2950 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 884.0, "epoch": 0.885122975404919, "grad_norm": 3.158586263656616, "learning_rate": 3.237631415714426e-07, "logp/chosen": -430.0, "logp/rejected": -352.0, "loss": 0.2860107421875, "reward/accuracy": 0.8125, "reward/chosen": -1.734375, "reward/margin": 2.046875, "reward/rejected": -3.78125, "step": 2951 }, { "approx. KL/chosen": 146.0, "approx. KL/rejected": 868.0, "epoch": 0.8854229154169166, "grad_norm": 3.0774776935577393, "learning_rate": 3.220973899907548e-07, "logp/chosen": -159.0, "logp/rejected": -310.0, "loss": 0.409393310546875, "reward/accuracy": 0.75, "reward/chosen": -1.4921875, "reward/margin": 1.8828125, "reward/rejected": -3.375, "step": 2952 }, { "approx. KL/chosen": 180.0, "approx. KL/rejected": 440.0, "epoch": 0.8857228554289143, "grad_norm": 5.413686275482178, "learning_rate": 3.2043579196250954e-07, "logp/chosen": -408.0, "logp/rejected": -444.0, "loss": 0.602294921875, "reward/accuracy": 0.8125, "reward/chosen": -1.4375, "reward/margin": 1.0234375, "reward/rejected": -2.453125, "step": 2953 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 304.0, "epoch": 0.8860227954409118, "grad_norm": 4.713141918182373, "learning_rate": 3.1877834896205493e-07, "logp/chosen": -364.0, "logp/rejected": -352.0, "loss": 0.673095703125, "reward/accuracy": 0.625, "reward/chosen": -1.5234375, "reward/margin": 0.451171875, "reward/rejected": -1.9765625, "step": 2954 }, { "approx. KL/chosen": 244.0, "approx. KL/rejected": 812.0, "epoch": 0.8863227354529094, "grad_norm": 3.9864678382873535, "learning_rate": 3.1712506246105e-07, "logp/chosen": -364.0, "logp/rejected": -346.0, "loss": 0.42877197265625, "reward/accuracy": 0.75, "reward/chosen": -1.6953125, "reward/margin": 1.6953125, "reward/rejected": -3.390625, "step": 2955 }, { "approx. KL/chosen": 424.0, "approx. KL/rejected": 884.0, "epoch": 0.886622675464907, "grad_norm": 3.5594677925109863, "learning_rate": 3.15475933927461e-07, "logp/chosen": -224.0, "logp/rejected": -272.0, "loss": 0.441375732421875, "reward/accuracy": 0.8125, "reward/chosen": -2.265625, "reward/margin": 1.25, "reward/rejected": -3.515625, "step": 2956 }, { "approx. KL/chosen": 225.0, "approx. KL/rejected": 832.0, "epoch": 0.8869226154769047, "grad_norm": 3.2141664028167725, "learning_rate": 3.1383096482556665e-07, "logp/chosen": -280.0, "logp/rejected": -494.0, "loss": 0.49622344970703125, "reward/accuracy": 0.75, "reward/chosen": -1.9609375, "reward/margin": 1.578125, "reward/rejected": -3.53125, "step": 2957 }, { "approx. KL/chosen": 206.0, "approx. KL/rejected": 660.0, "epoch": 0.8872225554889022, "grad_norm": 3.2622921466827393, "learning_rate": 3.121901566159491e-07, "logp/chosen": -264.0, "logp/rejected": -260.0, "loss": 0.409271240234375, "reward/accuracy": 0.8125, "reward/chosen": -1.640625, "reward/margin": 1.3203125, "reward/rejected": -2.953125, "step": 2958 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 928.0, "epoch": 0.8875224955008998, "grad_norm": 2.031869888305664, "learning_rate": 3.1055351075549633e-07, "logp/chosen": -198.0, "logp/rejected": -239.0, "loss": 0.2849273681640625, "reward/accuracy": 0.8125, "reward/chosen": -1.5625, "reward/margin": 2.109375, "reward/rejected": -3.671875, "step": 2959 }, { "approx. KL/chosen": 98.0, "approx. KL/rejected": 836.0, "epoch": 0.8878224355128974, "grad_norm": 2.401324510574341, "learning_rate": 3.0892102869740147e-07, "logp/chosen": -370.0, "logp/rejected": -324.0, "loss": 0.19012832641601562, "reward/accuracy": 0.875, "reward/chosen": -0.7578125, "reward/margin": 2.796875, "reward/rejected": -3.5625, "step": 2960 }, { "approx. KL/chosen": 252.0, "approx. KL/rejected": 724.0, "epoch": 0.888122375524895, "grad_norm": 7.530544281005859, "learning_rate": 3.072927118911606e-07, "logp/chosen": -294.0, "logp/rejected": -360.0, "loss": 0.5030517578125, "reward/accuracy": 0.75, "reward/chosen": -1.65625, "reward/margin": 1.7265625, "reward/rejected": -3.390625, "step": 2961 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 528.0, "epoch": 0.8884223155368927, "grad_norm": 2.959347724914551, "learning_rate": 3.0566856178257165e-07, "logp/chosen": -356.0, "logp/rejected": -298.0, "loss": 0.3165283203125, "reward/accuracy": 0.8125, "reward/chosen": -1.375, "reward/margin": 1.453125, "reward/rejected": -2.828125, "step": 2962 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 948.0, "epoch": 0.8887222555488902, "grad_norm": 2.763335943222046, "learning_rate": 3.040485798137305e-07, "logp/chosen": -362.0, "logp/rejected": -328.0, "loss": 0.2130126953125, "reward/accuracy": 0.9375, "reward/chosen": -1.4609375, "reward/margin": 2.46875, "reward/rejected": -3.9375, "step": 2963 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 820.0, "epoch": 0.8890221955608878, "grad_norm": 5.036373138427734, "learning_rate": 3.024327674230354e-07, "logp/chosen": -157.0, "logp/rejected": -249.0, "loss": 0.5321044921875, "reward/accuracy": 0.8125, "reward/chosen": -1.3828125, "reward/margin": 1.953125, "reward/rejected": -3.34375, "step": 2964 }, { "approx. KL/chosen": 470.0, "approx. KL/rejected": 410.0, "epoch": 0.8893221355728854, "grad_norm": 6.231085777282715, "learning_rate": 3.00821126045181e-07, "logp/chosen": -342.0, "logp/rejected": -306.0, "loss": 1.0936279296875, "reward/accuracy": 0.4375, "reward/chosen": -2.21875, "reward/margin": -0.04150390625, "reward/rejected": -2.171875, "step": 2965 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 510.0, "epoch": 0.889622075584883, "grad_norm": 3.8288543224334717, "learning_rate": 2.99213657111157e-07, "logp/chosen": -362.0, "logp/rejected": -324.0, "loss": 0.408935546875, "reward/accuracy": 0.75, "reward/chosen": -1.4921875, "reward/margin": 1.4296875, "reward/rejected": -2.921875, "step": 2966 }, { "approx. KL/chosen": 145.0, "approx. KL/rejected": 608.0, "epoch": 0.8899220155968807, "grad_norm": 3.224597930908203, "learning_rate": 2.976103620482518e-07, "logp/chosen": -326.0, "logp/rejected": -314.0, "loss": 0.38079833984375, "reward/accuracy": 0.6875, "reward/chosen": -1.2421875, "reward/margin": 1.6171875, "reward/rejected": -2.859375, "step": 2967 }, { "approx. KL/chosen": 118.0, "approx. KL/rejected": 808.0, "epoch": 0.8902219556088782, "grad_norm": 2.8955600261688232, "learning_rate": 2.960112422800443e-07, "logp/chosen": -278.0, "logp/rejected": -300.0, "loss": 0.23377227783203125, "reward/accuracy": 0.9375, "reward/chosen": -1.0078125, "reward/margin": 2.59375, "reward/rejected": -3.59375, "step": 2968 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 528.0, "epoch": 0.8905218956208758, "grad_norm": 5.209580421447754, "learning_rate": 2.944162992264077e-07, "logp/chosen": -444.0, "logp/rejected": -374.0, "loss": 0.52252197265625, "reward/accuracy": 0.75, "reward/chosen": -1.71875, "reward/margin": 1.0625, "reward/rejected": -2.78125, "step": 2969 }, { "approx. KL/chosen": 322.0, "approx. KL/rejected": 948.0, "epoch": 0.8908218356328734, "grad_norm": 3.3939898014068604, "learning_rate": 2.9282553430350637e-07, "logp/chosen": -247.0, "logp/rejected": -356.0, "loss": 0.4261474609375, "reward/accuracy": 0.8125, "reward/chosen": -2.078125, "reward/margin": 1.6640625, "reward/rejected": -3.75, "step": 2970 }, { "approx. KL/chosen": 205.0, "approx. KL/rejected": 776.0, "epoch": 0.891121775644871, "grad_norm": 4.138793468475342, "learning_rate": 2.9123894892379547e-07, "logp/chosen": -472.0, "logp/rejected": -424.0, "loss": 0.513916015625, "reward/accuracy": 0.75, "reward/chosen": -1.3359375, "reward/margin": 1.7734375, "reward/rejected": -3.109375, "step": 2971 }, { "approx. KL/chosen": 568.0, "approx. KL/rejected": 844.0, "epoch": 0.8914217156568687, "grad_norm": 7.223564624786377, "learning_rate": 2.8965654449601865e-07, "logp/chosen": -374.0, "logp/rejected": -354.0, "loss": 1.04766845703125, "reward/accuracy": 0.5625, "reward/chosen": -2.625, "reward/margin": 0.86328125, "reward/rejected": -3.484375, "step": 2972 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 628.0, "epoch": 0.8917216556688662, "grad_norm": 2.3380181789398193, "learning_rate": 2.880783224252065e-07, "logp/chosen": -288.0, "logp/rejected": -270.0, "loss": 0.38916015625, "reward/accuracy": 0.8125, "reward/chosen": -1.34375, "reward/margin": 1.4375, "reward/rejected": -2.78125, "step": 2973 }, { "approx. KL/chosen": 98.5, "approx. KL/rejected": 832.0, "epoch": 0.8920215956808638, "grad_norm": 3.308854818344116, "learning_rate": 2.8650428411267805e-07, "logp/chosen": -254.0, "logp/rejected": -262.0, "loss": 0.379608154296875, "reward/accuracy": 0.6875, "reward/chosen": -1.1015625, "reward/margin": 2.125, "reward/rejected": -3.21875, "step": 2974 }, { "approx. KL/chosen": 206.0, "approx. KL/rejected": 792.0, "epoch": 0.8923215356928614, "grad_norm": 4.718365669250488, "learning_rate": 2.849344309560359e-07, "logp/chosen": -250.0, "logp/rejected": -282.0, "loss": 0.503448486328125, "reward/accuracy": 0.6875, "reward/chosen": -1.1875, "reward/margin": 2.34375, "reward/rejected": -3.53125, "step": 2975 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 422.0, "epoch": 0.8926214757048591, "grad_norm": 7.1621904373168945, "learning_rate": 2.833687643491667e-07, "logp/chosen": -356.0, "logp/rejected": -328.0, "loss": 0.728759765625, "reward/accuracy": 0.625, "reward/chosen": -1.6328125, "reward/margin": 0.78515625, "reward/rejected": -2.421875, "step": 2976 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 660.0, "epoch": 0.8929214157168567, "grad_norm": 3.091085910797119, "learning_rate": 2.8180728568223904e-07, "logp/chosen": -364.0, "logp/rejected": -332.0, "loss": 0.334686279296875, "reward/accuracy": 0.875, "reward/chosen": -1.5078125, "reward/margin": 1.5625, "reward/rejected": -3.0625, "step": 2977 }, { "approx. KL/chosen": 452.0, "approx. KL/rejected": 450.0, "epoch": 0.8932213557288542, "grad_norm": 5.812022686004639, "learning_rate": 2.8024999634170603e-07, "logp/chosen": -304.0, "logp/rejected": -229.0, "loss": 0.926513671875, "reward/accuracy": 0.5625, "reward/chosen": -2.5, "reward/margin": 0.16796875, "reward/rejected": -2.671875, "step": 2978 }, { "approx. KL/chosen": 119.5, "approx. KL/rejected": 672.0, "epoch": 0.8935212957408518, "grad_norm": 3.258518695831299, "learning_rate": 2.7869689771029885e-07, "logp/chosen": -500.0, "logp/rejected": -338.0, "loss": 0.25799560546875, "reward/accuracy": 0.9375, "reward/chosen": -1.015625, "reward/margin": 2.234375, "reward/rejected": -3.25, "step": 2979 }, { "approx. KL/chosen": 253.0, "approx. KL/rejected": 556.0, "epoch": 0.8938212357528494, "grad_norm": 5.963681697845459, "learning_rate": 2.771479911670266e-07, "logp/chosen": -300.0, "logp/rejected": -270.0, "loss": 0.6246337890625, "reward/accuracy": 0.625, "reward/chosen": -1.953125, "reward/margin": 0.890625, "reward/rejected": -2.84375, "step": 2980 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 472.0, "epoch": 0.8941211757648471, "grad_norm": 4.185481071472168, "learning_rate": 2.7560327808717815e-07, "logp/chosen": -316.0, "logp/rejected": -247.0, "loss": 0.45355224609375, "reward/accuracy": 0.6875, "reward/chosen": -1.40625, "reward/margin": 1.375, "reward/rejected": -2.78125, "step": 2981 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 1032.0, "epoch": 0.8944211157768446, "grad_norm": 2.937018394470215, "learning_rate": 2.74062759842319e-07, "logp/chosen": -282.0, "logp/rejected": -354.0, "loss": 0.351348876953125, "reward/accuracy": 0.875, "reward/chosen": -1.7421875, "reward/margin": 2.15625, "reward/rejected": -3.890625, "step": 2982 }, { "approx. KL/chosen": 410.0, "approx. KL/rejected": 788.0, "epoch": 0.8947210557888422, "grad_norm": 5.994164943695068, "learning_rate": 2.7252643780028897e-07, "logp/chosen": -272.0, "logp/rejected": -374.0, "loss": 0.609222412109375, "reward/accuracy": 0.6875, "reward/chosen": -2.34375, "reward/margin": 1.1796875, "reward/rejected": -3.515625, "step": 2983 }, { "approx. KL/chosen": 166.0, "approx. KL/rejected": 524.0, "epoch": 0.8950209958008398, "grad_norm": 3.6589548587799072, "learning_rate": 2.709943133252013e-07, "logp/chosen": -390.0, "logp/rejected": -392.0, "loss": 0.48126220703125, "reward/accuracy": 0.6875, "reward/chosen": -1.4765625, "reward/margin": 1.2421875, "reward/rejected": -2.71875, "step": 2984 }, { "approx. KL/chosen": 304.0, "approx. KL/rejected": 1056.0, "epoch": 0.8953209358128374, "grad_norm": 4.6896257400512695, "learning_rate": 2.694663877774451e-07, "logp/chosen": -324.0, "logp/rejected": -324.0, "loss": 0.434539794921875, "reward/accuracy": 0.75, "reward/chosen": -1.984375, "reward/margin": 2.09375, "reward/rejected": -4.0625, "step": 2985 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 908.0, "epoch": 0.8956208758248351, "grad_norm": 3.7628707885742188, "learning_rate": 2.6794266251367887e-07, "logp/chosen": -536.0, "logp/rejected": -350.0, "loss": 0.3218994140625, "reward/accuracy": 0.875, "reward/chosen": -1.7734375, "reward/margin": 2.03125, "reward/rejected": -3.8125, "step": 2986 }, { "approx. KL/chosen": 241.0, "approx. KL/rejected": 696.0, "epoch": 0.8959208158368326, "grad_norm": 4.103707790374756, "learning_rate": 2.6642313888683156e-07, "logp/chosen": -276.0, "logp/rejected": -318.0, "loss": 0.33544921875, "reward/accuracy": 0.8125, "reward/chosen": -1.640625, "reward/margin": 1.8359375, "reward/rejected": -3.46875, "step": 2987 }, { "approx. KL/chosen": 251.0, "approx. KL/rejected": 1160.0, "epoch": 0.8962207558488302, "grad_norm": 3.758331537246704, "learning_rate": 2.649078182461029e-07, "logp/chosen": -322.0, "logp/rejected": -380.0, "loss": 0.40956878662109375, "reward/accuracy": 0.875, "reward/chosen": -1.765625, "reward/margin": 2.515625, "reward/rejected": -4.28125, "step": 2988 }, { "approx. KL/chosen": 170.0, "approx. KL/rejected": 600.0, "epoch": 0.8965206958608278, "grad_norm": 2.736565589904785, "learning_rate": 2.633967019369593e-07, "logp/chosen": -376.0, "logp/rejected": -302.0, "loss": 0.253173828125, "reward/accuracy": 1.0, "reward/chosen": -1.515625, "reward/margin": 1.6328125, "reward/rejected": -3.140625, "step": 2989 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 708.0, "epoch": 0.8968206358728255, "grad_norm": 4.261321067810059, "learning_rate": 2.6188979130113536e-07, "logp/chosen": -456.0, "logp/rejected": -416.0, "loss": 0.48138427734375, "reward/accuracy": 0.75, "reward/chosen": -1.421875, "reward/margin": 1.6640625, "reward/rejected": -3.078125, "step": 2990 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 800.0, "epoch": 0.8971205758848231, "grad_norm": 4.663149833679199, "learning_rate": 2.6038708767662954e-07, "logp/chosen": -274.0, "logp/rejected": -306.0, "loss": 0.6417236328125, "reward/accuracy": 0.625, "reward/chosen": -1.796875, "reward/margin": 1.640625, "reward/rejected": -3.4375, "step": 2991 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 572.0, "epoch": 0.8974205158968206, "grad_norm": 2.7884209156036377, "learning_rate": 2.5888859239770747e-07, "logp/chosen": -392.0, "logp/rejected": -350.0, "loss": 0.3887176513671875, "reward/accuracy": 0.8125, "reward/chosen": -1.28125, "reward/margin": 1.6171875, "reward/rejected": -2.90625, "step": 2992 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 944.0, "epoch": 0.8977204559088182, "grad_norm": 5.2261643409729, "learning_rate": 2.5739430679489643e-07, "logp/chosen": -432.0, "logp/rejected": -334.0, "loss": 0.33025360107421875, "reward/accuracy": 0.8125, "reward/chosen": -1.609375, "reward/margin": 2.4375, "reward/rejected": -4.0625, "step": 2993 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 450.0, "epoch": 0.8980203959208158, "grad_norm": 3.6283960342407227, "learning_rate": 2.5590423219498614e-07, "logp/chosen": -268.0, "logp/rejected": -255.0, "loss": 0.4688720703125, "reward/accuracy": 0.75, "reward/chosen": -1.484375, "reward/margin": 1.234375, "reward/rejected": -2.71875, "step": 2994 }, { "approx. KL/chosen": 532.0, "approx. KL/rejected": 868.0, "epoch": 0.8983203359328135, "grad_norm": 5.619036674499512, "learning_rate": 2.544183699210262e-07, "logp/chosen": -322.0, "logp/rejected": -308.0, "loss": 0.78692626953125, "reward/accuracy": 0.75, "reward/chosen": -2.546875, "reward/margin": 1.1171875, "reward/rejected": -3.671875, "step": 2995 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 1032.0, "epoch": 0.8986202759448111, "grad_norm": 5.167501449584961, "learning_rate": 2.529367212923295e-07, "logp/chosen": -211.0, "logp/rejected": -332.0, "loss": 0.561553955078125, "reward/accuracy": 0.6875, "reward/chosen": -2.109375, "reward/margin": 1.8046875, "reward/rejected": -3.90625, "step": 2996 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 776.0, "epoch": 0.8989202159568086, "grad_norm": 2.1650021076202393, "learning_rate": 2.5145928762446425e-07, "logp/chosen": -390.0, "logp/rejected": -344.0, "loss": 0.256195068359375, "reward/accuracy": 0.9375, "reward/chosen": -1.6484375, "reward/margin": 1.7890625, "reward/rejected": -3.4375, "step": 2997 }, { "approx. KL/chosen": 125.0, "approx. KL/rejected": 708.0, "epoch": 0.8992201559688062, "grad_norm": 4.602954387664795, "learning_rate": 2.499860702292578e-07, "logp/chosen": -444.0, "logp/rejected": -486.0, "loss": 0.341094970703125, "reward/accuracy": 0.875, "reward/chosen": -1.0703125, "reward/margin": 1.890625, "reward/rejected": -2.96875, "step": 2998 }, { "approx. KL/chosen": 167.0, "approx. KL/rejected": 708.0, "epoch": 0.8995200959808038, "grad_norm": 3.520519733428955, "learning_rate": 2.485170704147921e-07, "logp/chosen": -350.0, "logp/rejected": -456.0, "loss": 0.423583984375, "reward/accuracy": 0.75, "reward/chosen": -1.390625, "reward/margin": 1.640625, "reward/rejected": -3.03125, "step": 2999 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 844.0, "epoch": 0.8998200359928015, "grad_norm": 3.3419189453125, "learning_rate": 2.470522894854072e-07, "logp/chosen": -366.0, "logp/rejected": -354.0, "loss": 0.34018707275390625, "reward/accuracy": 0.75, "reward/chosen": -1.6015625, "reward/margin": 2.0625, "reward/rejected": -3.65625, "step": 3000 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 944.0, "epoch": 0.9001199760047991, "grad_norm": 3.4854605197906494, "learning_rate": 2.455917287416937e-07, "logp/chosen": -250.0, "logp/rejected": -226.0, "loss": 0.346771240234375, "reward/accuracy": 0.8125, "reward/chosen": -1.6953125, "reward/margin": 2.328125, "reward/rejected": -4.03125, "step": 3001 }, { "approx. KL/chosen": 245.0, "approx. KL/rejected": 616.0, "epoch": 0.9004199160167966, "grad_norm": 4.760001182556152, "learning_rate": 2.441353894804982e-07, "logp/chosen": -298.0, "logp/rejected": -350.0, "loss": 0.4295654296875, "reward/accuracy": 0.6875, "reward/chosen": -1.546875, "reward/margin": 1.4921875, "reward/rejected": -3.03125, "step": 3002 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 672.0, "epoch": 0.9007198560287942, "grad_norm": 3.38447904586792, "learning_rate": 2.4268327299491745e-07, "logp/chosen": -354.0, "logp/rejected": -328.0, "loss": 0.310394287109375, "reward/accuracy": 0.9375, "reward/chosen": -1.4296875, "reward/margin": 1.84375, "reward/rejected": -3.265625, "step": 3003 }, { "approx. KL/chosen": 328.0, "approx. KL/rejected": 848.0, "epoch": 0.9010197960407919, "grad_norm": 5.144416809082031, "learning_rate": 2.4123538057429843e-07, "logp/chosen": -338.0, "logp/rejected": -350.0, "loss": 0.353546142578125, "reward/accuracy": 0.875, "reward/chosen": -1.71875, "reward/margin": 1.9375, "reward/rejected": -3.65625, "step": 3004 }, { "approx. KL/chosen": 195.0, "approx. KL/rejected": 684.0, "epoch": 0.9013197360527895, "grad_norm": 2.972024917602539, "learning_rate": 2.397917135042377e-07, "logp/chosen": -316.0, "logp/rejected": -264.0, "loss": 0.367919921875, "reward/accuracy": 0.75, "reward/chosen": -1.6796875, "reward/margin": 1.640625, "reward/rejected": -3.3125, "step": 3005 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 580.0, "epoch": 0.901619676064787, "grad_norm": 3.000667095184326, "learning_rate": 2.383522730665816e-07, "logp/chosen": -256.0, "logp/rejected": -272.0, "loss": 0.421661376953125, "reward/accuracy": 0.8125, "reward/chosen": -1.3203125, "reward/margin": 1.6484375, "reward/rejected": -2.96875, "step": 3006 }, { "approx. KL/chosen": 233.0, "approx. KL/rejected": 1240.0, "epoch": 0.9019196160767846, "grad_norm": 4.103479862213135, "learning_rate": 2.369170605394222e-07, "logp/chosen": -262.0, "logp/rejected": -324.0, "loss": 0.37478160858154297, "reward/accuracy": 0.8125, "reward/chosen": -1.6640625, "reward/margin": 2.765625, "reward/rejected": -4.4375, "step": 3007 }, { "approx. KL/chosen": 330.0, "approx. KL/rejected": 952.0, "epoch": 0.9022195560887822, "grad_norm": 3.466740131378174, "learning_rate": 2.3548607719709682e-07, "logp/chosen": -294.0, "logp/rejected": -284.0, "loss": 0.4647979736328125, "reward/accuracy": 0.8125, "reward/chosen": -2.171875, "reward/margin": 1.78125, "reward/rejected": -3.9375, "step": 3008 }, { "approx. KL/chosen": 89.5, "approx. KL/rejected": 620.0, "epoch": 0.9025194961007799, "grad_norm": 2.7675819396972656, "learning_rate": 2.3405932431018908e-07, "logp/chosen": -344.0, "logp/rejected": -394.0, "loss": 0.2923736572265625, "reward/accuracy": 0.8125, "reward/chosen": -0.94140625, "reward/margin": 1.9921875, "reward/rejected": -2.9375, "step": 3009 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 736.0, "epoch": 0.9028194361127775, "grad_norm": 4.032504558563232, "learning_rate": 2.326368031455267e-07, "logp/chosen": -440.0, "logp/rejected": -342.0, "loss": 0.45867919921875, "reward/accuracy": 0.875, "reward/chosen": -1.6328125, "reward/margin": 1.84375, "reward/rejected": -3.484375, "step": 3010 }, { "approx. KL/chosen": 192.0, "approx. KL/rejected": 824.0, "epoch": 0.903119376124775, "grad_norm": 3.5041511058807373, "learning_rate": 2.312185149661772e-07, "logp/chosen": -368.0, "logp/rejected": -358.0, "loss": 0.40338134765625, "reward/accuracy": 0.75, "reward/chosen": -1.5546875, "reward/margin": 1.765625, "reward/rejected": -3.3125, "step": 3011 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 784.0, "epoch": 0.9034193161367726, "grad_norm": 3.017195463180542, "learning_rate": 2.2980446103145482e-07, "logp/chosen": -304.0, "logp/rejected": -264.0, "loss": 0.2388916015625, "reward/accuracy": 0.9375, "reward/chosen": -1.3203125, "reward/margin": 2.09375, "reward/rejected": -3.40625, "step": 3012 }, { "approx. KL/chosen": 75.5, "approx. KL/rejected": 382.0, "epoch": 0.9037192561487702, "grad_norm": 3.07625412940979, "learning_rate": 2.2839464259690804e-07, "logp/chosen": -320.0, "logp/rejected": -316.0, "loss": 0.384307861328125, "reward/accuracy": 0.8125, "reward/chosen": -0.83984375, "reward/margin": 1.4375, "reward/rejected": -2.28125, "step": 3013 }, { "approx. KL/chosen": 57.75, "approx. KL/rejected": 608.0, "epoch": 0.9040191961607679, "grad_norm": 1.8200383186340332, "learning_rate": 2.2698906091432882e-07, "logp/chosen": -314.0, "logp/rejected": -338.0, "loss": 0.213623046875, "reward/accuracy": 1.0, "reward/chosen": -0.73046875, "reward/margin": 2.375, "reward/rejected": -3.109375, "step": 3014 }, { "approx. KL/chosen": 236.0, "approx. KL/rejected": 760.0, "epoch": 0.9043191361727655, "grad_norm": 3.1977195739746094, "learning_rate": 2.2558771723174554e-07, "logp/chosen": -298.0, "logp/rejected": -272.0, "loss": 0.3905029296875, "reward/accuracy": 0.8125, "reward/chosen": -1.7734375, "reward/margin": 1.6328125, "reward/rejected": -3.40625, "step": 3015 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 532.0, "epoch": 0.904619076184763, "grad_norm": 7.033242702484131, "learning_rate": 2.2419061279342457e-07, "logp/chosen": -312.0, "logp/rejected": -266.0, "loss": 1.1085205078125, "reward/accuracy": 0.5625, "reward/chosen": -1.9140625, "reward/margin": 0.50390625, "reward/rejected": -2.421875, "step": 3016 }, { "approx. KL/chosen": 628.0, "approx. KL/rejected": 752.0, "epoch": 0.9049190161967606, "grad_norm": 8.287936210632324, "learning_rate": 2.227977488398675e-07, "logp/chosen": -356.0, "logp/rejected": -288.0, "loss": 0.892333984375, "reward/accuracy": 0.5625, "reward/chosen": -3.109375, "reward/margin": 0.416015625, "reward/rejected": -3.515625, "step": 3017 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 1112.0, "epoch": 0.9052189562087583, "grad_norm": 2.746079683303833, "learning_rate": 2.214091266078111e-07, "logp/chosen": -258.0, "logp/rejected": -324.0, "loss": 0.2740020751953125, "reward/accuracy": 0.875, "reward/chosen": -1.4609375, "reward/margin": 2.515625, "reward/rejected": -3.96875, "step": 3018 }, { "approx. KL/chosen": 254.0, "approx. KL/rejected": 736.0, "epoch": 0.9055188962207559, "grad_norm": 4.688965797424316, "learning_rate": 2.2002474733022415e-07, "logp/chosen": -394.0, "logp/rejected": -378.0, "loss": 0.51361083984375, "reward/accuracy": 0.625, "reward/chosen": -1.6875, "reward/margin": 1.3125, "reward/rejected": -3.0, "step": 3019 }, { "approx. KL/chosen": 222.0, "approx. KL/rejected": 584.0, "epoch": 0.9058188362327535, "grad_norm": 3.864603281021118, "learning_rate": 2.1864461223631227e-07, "logp/chosen": -326.0, "logp/rejected": -360.0, "loss": 0.459625244140625, "reward/accuracy": 0.8125, "reward/chosen": -1.53125, "reward/margin": 1.40625, "reward/rejected": -2.9375, "step": 3020 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 1104.0, "epoch": 0.906118776244751, "grad_norm": 1.8210759162902832, "learning_rate": 2.17268722551508e-07, "logp/chosen": -221.0, "logp/rejected": -312.0, "loss": 0.21722412109375, "reward/accuracy": 0.9375, "reward/chosen": -1.65625, "reward/margin": 2.3125, "reward/rejected": -3.96875, "step": 3021 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 704.0, "epoch": 0.9064187162567486, "grad_norm": 3.275087833404541, "learning_rate": 2.1589707949747795e-07, "logp/chosen": -306.0, "logp/rejected": -362.0, "loss": 0.3540763854980469, "reward/accuracy": 0.8125, "reward/chosen": -1.09375, "reward/margin": 2.0, "reward/rejected": -3.09375, "step": 3022 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 1192.0, "epoch": 0.9067186562687463, "grad_norm": 3.4356043338775635, "learning_rate": 2.1452968429211463e-07, "logp/chosen": -444.0, "logp/rejected": -370.0, "loss": 0.3517913818359375, "reward/accuracy": 0.75, "reward/chosen": -1.609375, "reward/margin": 2.359375, "reward/rejected": -3.96875, "step": 3023 }, { "approx. KL/chosen": 322.0, "approx. KL/rejected": 708.0, "epoch": 0.9070185962807439, "grad_norm": 7.283192157745361, "learning_rate": 2.1316653814954235e-07, "logp/chosen": -296.0, "logp/rejected": -296.0, "loss": 0.61016845703125, "reward/accuracy": 0.6875, "reward/chosen": -2.046875, "reward/margin": 1.1796875, "reward/rejected": -3.234375, "step": 3024 }, { "approx. KL/chosen": 205.0, "approx. KL/rejected": 604.0, "epoch": 0.9073185362927415, "grad_norm": 3.160700798034668, "learning_rate": 2.1180764228010962e-07, "logp/chosen": -356.0, "logp/rejected": -300.0, "loss": 0.3385009765625, "reward/accuracy": 0.875, "reward/chosen": -1.5546875, "reward/margin": 1.6796875, "reward/rejected": -3.234375, "step": 3025 }, { "approx. KL/chosen": 175.0, "approx. KL/rejected": 728.0, "epoch": 0.907618476304739, "grad_norm": 4.622803688049316, "learning_rate": 2.1045299789039407e-07, "logp/chosen": -384.0, "logp/rejected": -404.0, "loss": 0.35279083251953125, "reward/accuracy": 0.6875, "reward/chosen": -1.1328125, "reward/margin": 2.25, "reward/rejected": -3.375, "step": 3026 }, { "approx. KL/chosen": 436.0, "approx. KL/rejected": 1280.0, "epoch": 0.9079184163167366, "grad_norm": 4.023387908935547, "learning_rate": 2.0910260618319633e-07, "logp/chosen": -322.0, "logp/rejected": -392.0, "loss": 0.4887237548828125, "reward/accuracy": 0.6875, "reward/chosen": -2.28125, "reward/margin": 1.9609375, "reward/rejected": -4.25, "step": 3027 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 920.0, "epoch": 0.9082183563287343, "grad_norm": 3.2883105278015137, "learning_rate": 2.077564683575406e-07, "logp/chosen": -262.0, "logp/rejected": -372.0, "loss": 0.443267822265625, "reward/accuracy": 0.875, "reward/chosen": -1.7109375, "reward/margin": 2.078125, "reward/rejected": -3.796875, "step": 3028 }, { "approx. KL/chosen": 354.0, "approx. KL/rejected": 904.0, "epoch": 0.9085182963407319, "grad_norm": 6.087784767150879, "learning_rate": 2.0641458560867577e-07, "logp/chosen": -268.0, "logp/rejected": -466.0, "loss": 0.475494384765625, "reward/accuracy": 0.8125, "reward/chosen": -2.1875, "reward/margin": 1.7421875, "reward/rejected": -3.9375, "step": 3029 }, { "approx. KL/chosen": 374.0, "approx. KL/rejected": 1200.0, "epoch": 0.9088182363527294, "grad_norm": 3.770092248916626, "learning_rate": 2.0507695912807158e-07, "logp/chosen": -358.0, "logp/rejected": -436.0, "loss": 0.4321174621582031, "reward/accuracy": 0.75, "reward/chosen": -2.078125, "reward/margin": 2.1875, "reward/rejected": -4.25, "step": 3030 }, { "approx. KL/chosen": 96.0, "approx. KL/rejected": 716.0, "epoch": 0.909118176364727, "grad_norm": 2.8229434490203857, "learning_rate": 2.0374359010341848e-07, "logp/chosen": -234.0, "logp/rejected": -306.0, "loss": 0.3836212158203125, "reward/accuracy": 0.8125, "reward/chosen": -0.91796875, "reward/margin": 2.1875, "reward/rejected": -3.109375, "step": 3031 }, { "approx. KL/chosen": 133.0, "approx. KL/rejected": 556.0, "epoch": 0.9094181163767247, "grad_norm": 3.794178009033203, "learning_rate": 2.0241447971862783e-07, "logp/chosen": -354.0, "logp/rejected": -330.0, "loss": 0.552001953125, "reward/accuracy": 0.625, "reward/chosen": -1.328125, "reward/margin": 1.1640625, "reward/rejected": -2.484375, "step": 3032 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 840.0, "epoch": 0.9097180563887223, "grad_norm": 3.9768450260162354, "learning_rate": 2.010896291538278e-07, "logp/chosen": -302.0, "logp/rejected": -388.0, "loss": 0.5525321960449219, "reward/accuracy": 0.625, "reward/chosen": -1.375, "reward/margin": 1.71875, "reward/rejected": -3.09375, "step": 3033 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 800.0, "epoch": 0.9100179964007199, "grad_norm": 3.13258695602417, "learning_rate": 1.9976903958536631e-07, "logp/chosen": -286.0, "logp/rejected": -318.0, "loss": 0.347412109375, "reward/accuracy": 0.8125, "reward/chosen": -1.890625, "reward/margin": 1.5234375, "reward/rejected": -3.40625, "step": 3034 }, { "approx. KL/chosen": 155.0, "approx. KL/rejected": 796.0, "epoch": 0.9103179364127174, "grad_norm": 3.6390020847320557, "learning_rate": 1.9845271218580597e-07, "logp/chosen": -294.0, "logp/rejected": -376.0, "loss": 0.3521881103515625, "reward/accuracy": 0.75, "reward/chosen": -1.40625, "reward/margin": 2.015625, "reward/rejected": -3.421875, "step": 3035 }, { "approx. KL/chosen": 210.0, "approx. KL/rejected": 624.0, "epoch": 0.910617876424715, "grad_norm": 3.065708875656128, "learning_rate": 1.971406481239274e-07, "logp/chosen": -376.0, "logp/rejected": -418.0, "loss": 0.31793212890625, "reward/accuracy": 0.875, "reward/chosen": -1.625, "reward/margin": 1.53125, "reward/rejected": -3.15625, "step": 3036 }, { "approx. KL/chosen": 186.0, "approx. KL/rejected": 1064.0, "epoch": 0.9109178164367127, "grad_norm": 3.440749406814575, "learning_rate": 1.9583284856472262e-07, "logp/chosen": -312.0, "logp/rejected": -544.0, "loss": 0.375396728515625, "reward/accuracy": 0.875, "reward/chosen": -1.3828125, "reward/margin": 2.4375, "reward/rejected": -3.828125, "step": 3037 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 800.0, "epoch": 0.9112177564487103, "grad_norm": 6.4058027267456055, "learning_rate": 1.945293146693994e-07, "logp/chosen": -338.0, "logp/rejected": -352.0, "loss": 0.6525192260742188, "reward/accuracy": 0.6875, "reward/chosen": -1.78125, "reward/margin": 1.609375, "reward/rejected": -3.390625, "step": 3038 }, { "approx. KL/chosen": 248.0, "approx. KL/rejected": 940.0, "epoch": 0.9115176964607079, "grad_norm": 2.630373477935791, "learning_rate": 1.932300475953769e-07, "logp/chosen": -484.0, "logp/rejected": -326.0, "loss": 0.2524871826171875, "reward/accuracy": 0.875, "reward/chosen": -1.7890625, "reward/margin": 1.9765625, "reward/rejected": -3.765625, "step": 3039 }, { "approx. KL/chosen": 536.0, "approx. KL/rejected": 992.0, "epoch": 0.9118176364727054, "grad_norm": 6.264981269836426, "learning_rate": 1.919350484962873e-07, "logp/chosen": -310.0, "logp/rejected": -388.0, "loss": 0.69549560546875, "reward/accuracy": 0.4375, "reward/chosen": -2.921875, "reward/margin": 0.79296875, "reward/rejected": -3.71875, "step": 3040 }, { "approx. KL/chosen": 414.0, "approx. KL/rejected": 744.0, "epoch": 0.912117576484703, "grad_norm": 5.007439613342285, "learning_rate": 1.9064431852197197e-07, "logp/chosen": -338.0, "logp/rejected": -328.0, "loss": 0.6521453857421875, "reward/accuracy": 0.75, "reward/chosen": -2.46875, "reward/margin": 0.8984375, "reward/rejected": -3.375, "step": 3041 }, { "approx. KL/chosen": 82.5, "approx. KL/rejected": 1032.0, "epoch": 0.9124175164967007, "grad_norm": 2.879135847091675, "learning_rate": 1.8935785881848135e-07, "logp/chosen": -362.0, "logp/rejected": -360.0, "loss": 0.21311187744140625, "reward/accuracy": 0.9375, "reward/chosen": -0.90625, "reward/margin": 2.953125, "reward/rejected": -3.859375, "step": 3042 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 720.0, "epoch": 0.9127174565086983, "grad_norm": 3.5701019763946533, "learning_rate": 1.8807567052807397e-07, "logp/chosen": -388.0, "logp/rejected": -332.0, "loss": 0.37451171875, "reward/accuracy": 0.8125, "reward/chosen": -1.984375, "reward/margin": 1.5390625, "reward/rejected": -3.515625, "step": 3043 }, { "approx. KL/chosen": 290.0, "approx. KL/rejected": 1440.0, "epoch": 0.9130173965206959, "grad_norm": 2.29780650138855, "learning_rate": 1.86797754789218e-07, "logp/chosen": -396.0, "logp/rejected": -460.0, "loss": 0.1861419677734375, "reward/accuracy": 0.9375, "reward/chosen": -1.8203125, "reward/margin": 3.046875, "reward/rejected": -4.875, "step": 3044 }, { "approx. KL/chosen": 302.0, "approx. KL/rejected": 788.0, "epoch": 0.9133173365326934, "grad_norm": 6.138743877410889, "learning_rate": 1.8552411273658533e-07, "logp/chosen": -400.0, "logp/rejected": -310.0, "loss": 0.514556884765625, "reward/accuracy": 0.8125, "reward/chosen": -1.953125, "reward/margin": 1.5703125, "reward/rejected": -3.515625, "step": 3045 }, { "approx. KL/chosen": 248.0, "approx. KL/rejected": 1288.0, "epoch": 0.913617276544691, "grad_norm": 1.769639253616333, "learning_rate": 1.8425474550105626e-07, "logp/chosen": -386.0, "logp/rejected": -432.0, "loss": 0.14829254150390625, "reward/accuracy": 0.9375, "reward/chosen": -1.765625, "reward/margin": 2.78125, "reward/rejected": -4.53125, "step": 3046 }, { "approx. KL/chosen": 159.0, "approx. KL/rejected": 708.0, "epoch": 0.9139172165566887, "grad_norm": 4.689706325531006, "learning_rate": 1.8298965420971092e-07, "logp/chosen": -314.0, "logp/rejected": -280.0, "loss": 0.44158935546875, "reward/accuracy": 0.875, "reward/chosen": -1.359375, "reward/margin": 2.015625, "reward/rejected": -3.359375, "step": 3047 }, { "approx. KL/chosen": 137.0, "approx. KL/rejected": 608.0, "epoch": 0.9142171565686863, "grad_norm": 5.220993518829346, "learning_rate": 1.8172883998583745e-07, "logp/chosen": -148.0, "logp/rejected": -236.0, "loss": 0.692413330078125, "reward/accuracy": 0.6875, "reward/chosen": -1.1875, "reward/margin": 1.484375, "reward/rejected": -2.671875, "step": 3048 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 760.0, "epoch": 0.9145170965806838, "grad_norm": 3.7249033451080322, "learning_rate": 1.8047230394892256e-07, "logp/chosen": -344.0, "logp/rejected": -376.0, "loss": 0.410614013671875, "reward/accuracy": 0.6875, "reward/chosen": -1.8125, "reward/margin": 1.609375, "reward/rejected": -3.421875, "step": 3049 }, { "approx. KL/chosen": 502.0, "approx. KL/rejected": 848.0, "epoch": 0.9148170365926814, "grad_norm": 7.151951313018799, "learning_rate": 1.792200472146588e-07, "logp/chosen": -334.0, "logp/rejected": -358.0, "loss": 0.756103515625, "reward/accuracy": 0.625, "reward/chosen": -2.59375, "reward/margin": 1.0703125, "reward/rejected": -3.671875, "step": 3050 }, { "approx. KL/chosen": 304.0, "approx. KL/rejected": 716.0, "epoch": 0.915116976604679, "grad_norm": 3.9673802852630615, "learning_rate": 1.779720708949334e-07, "logp/chosen": -352.0, "logp/rejected": -302.0, "loss": 0.491912841796875, "reward/accuracy": 0.75, "reward/chosen": -1.9296875, "reward/margin": 1.375, "reward/rejected": -3.296875, "step": 3051 }, { "approx. KL/chosen": 209.0, "approx. KL/rejected": 532.0, "epoch": 0.9154169166166767, "grad_norm": 4.400283336639404, "learning_rate": 1.767283760978372e-07, "logp/chosen": -342.0, "logp/rejected": -314.0, "loss": 0.4296875, "reward/accuracy": 0.8125, "reward/chosen": -1.59375, "reward/margin": 1.265625, "reward/rejected": -2.859375, "step": 3052 }, { "approx. KL/chosen": 99.5, "approx. KL/rejected": 548.0, "epoch": 0.9157168566286743, "grad_norm": 3.9907066822052, "learning_rate": 1.75488963927658e-07, "logp/chosen": -470.0, "logp/rejected": -474.0, "loss": 0.576171875, "reward/accuracy": 0.8125, "reward/chosen": -1.234375, "reward/margin": 1.40625, "reward/rejected": -2.640625, "step": 3053 }, { "approx. KL/chosen": 308.0, "approx. KL/rejected": 828.0, "epoch": 0.9160167966406718, "grad_norm": 4.039622783660889, "learning_rate": 1.7425383548488216e-07, "logp/chosen": -378.0, "logp/rejected": -324.0, "loss": 0.384033203125, "reward/accuracy": 0.8125, "reward/chosen": -1.921875, "reward/margin": 1.765625, "reward/rejected": -3.6875, "step": 3054 }, { "approx. KL/chosen": 290.0, "approx. KL/rejected": 636.0, "epoch": 0.9163167366526694, "grad_norm": 4.045914649963379, "learning_rate": 1.730229918661902e-07, "logp/chosen": -205.0, "logp/rejected": -202.0, "loss": 0.4188232421875, "reward/accuracy": 0.8125, "reward/chosen": -1.8515625, "reward/margin": 1.234375, "reward/rejected": -3.09375, "step": 3055 }, { "approx. KL/chosen": 157.0, "approx. KL/rejected": 656.0, "epoch": 0.9166166766646671, "grad_norm": 3.253002882003784, "learning_rate": 1.7179643416446013e-07, "logp/chosen": -292.0, "logp/rejected": -354.0, "loss": 0.30120849609375, "reward/accuracy": 0.875, "reward/chosen": -1.4765625, "reward/margin": 1.6015625, "reward/rejected": -3.078125, "step": 3056 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 764.0, "epoch": 0.9169166166766647, "grad_norm": 4.678734302520752, "learning_rate": 1.70574163468763e-07, "logp/chosen": -404.0, "logp/rejected": -428.0, "loss": 0.4061737060546875, "reward/accuracy": 0.75, "reward/chosen": -1.390625, "reward/margin": 1.9921875, "reward/rejected": -3.375, "step": 3057 }, { "approx. KL/chosen": 247.0, "approx. KL/rejected": 616.0, "epoch": 0.9172165566886623, "grad_norm": 3.9302141666412354, "learning_rate": 1.6935618086436455e-07, "logp/chosen": -414.0, "logp/rejected": -390.0, "loss": 0.471405029296875, "reward/accuracy": 0.75, "reward/chosen": -1.6640625, "reward/margin": 1.3125, "reward/rejected": -2.984375, "step": 3058 }, { "approx. KL/chosen": 442.0, "approx. KL/rejected": 1120.0, "epoch": 0.9175164967006598, "grad_norm": 4.476776599884033, "learning_rate": 1.6814248743272255e-07, "logp/chosen": -420.0, "logp/rejected": -392.0, "loss": 0.41820526123046875, "reward/accuracy": 0.8125, "reward/chosen": -1.9140625, "reward/margin": 2.046875, "reward/rejected": -3.953125, "step": 3059 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 1024.0, "epoch": 0.9178164367126574, "grad_norm": 3.671314239501953, "learning_rate": 1.6693308425148825e-07, "logp/chosen": -250.0, "logp/rejected": -292.0, "loss": 0.533447265625, "reward/accuracy": 0.6875, "reward/chosen": -1.84375, "reward/margin": 1.5390625, "reward/rejected": -3.390625, "step": 3060 }, { "approx. KL/chosen": 182.0, "approx. KL/rejected": 660.0, "epoch": 0.9181163767246551, "grad_norm": 4.427555561065674, "learning_rate": 1.6572797239449823e-07, "logp/chosen": -332.0, "logp/rejected": -350.0, "loss": 0.3721923828125, "reward/accuracy": 0.875, "reward/chosen": -1.4140625, "reward/margin": 1.7421875, "reward/rejected": -3.15625, "step": 3061 }, { "approx. KL/chosen": 266.0, "approx. KL/rejected": 656.0, "epoch": 0.9184163167366527, "grad_norm": 4.070122241973877, "learning_rate": 1.6452715293178545e-07, "logp/chosen": -330.0, "logp/rejected": -284.0, "loss": 0.50830078125, "reward/accuracy": 0.75, "reward/chosen": -2.046875, "reward/margin": 1.1328125, "reward/rejected": -3.171875, "step": 3062 }, { "approx. KL/chosen": 221.0, "approx. KL/rejected": 744.0, "epoch": 0.9187162567486503, "grad_norm": 3.6513431072235107, "learning_rate": 1.6333062692956648e-07, "logp/chosen": -318.0, "logp/rejected": -266.0, "loss": 0.443145751953125, "reward/accuracy": 0.8125, "reward/chosen": -1.5703125, "reward/margin": 1.875, "reward/rejected": -3.453125, "step": 3063 }, { "approx. KL/chosen": 73.0, "approx. KL/rejected": 446.0, "epoch": 0.9190161967606478, "grad_norm": 2.9336583614349365, "learning_rate": 1.6213839545024867e-07, "logp/chosen": -340.0, "logp/rejected": -312.0, "loss": 0.3475494384765625, "reward/accuracy": 0.875, "reward/chosen": -0.87109375, "reward/margin": 1.609375, "reward/rejected": -2.484375, "step": 3064 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 520.0, "epoch": 0.9193161367726455, "grad_norm": 3.8318915367126465, "learning_rate": 1.6095045955242527e-07, "logp/chosen": -290.0, "logp/rejected": -320.0, "loss": 0.4832611083984375, "reward/accuracy": 0.8125, "reward/chosen": -1.8125, "reward/margin": 1.03125, "reward/rejected": -2.84375, "step": 3065 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 740.0, "epoch": 0.9196160767846431, "grad_norm": 3.8889386653900146, "learning_rate": 1.597668202908753e-07, "logp/chosen": -412.0, "logp/rejected": -440.0, "loss": 0.3729095458984375, "reward/accuracy": 0.875, "reward/chosen": -1.7734375, "reward/margin": 1.5859375, "reward/rejected": -3.359375, "step": 3066 }, { "approx. KL/chosen": 213.0, "approx. KL/rejected": 852.0, "epoch": 0.9199160167966407, "grad_norm": 4.348222732543945, "learning_rate": 1.5858747871656256e-07, "logp/chosen": -278.0, "logp/rejected": -326.0, "loss": 0.56182861328125, "reward/accuracy": 0.6875, "reward/chosen": -1.640625, "reward/margin": 1.625, "reward/rejected": -3.265625, "step": 3067 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 1368.0, "epoch": 0.9202159568086383, "grad_norm": 3.0578861236572266, "learning_rate": 1.5741243587663545e-07, "logp/chosen": -510.0, "logp/rejected": -348.0, "loss": 0.31146240234375, "reward/accuracy": 0.875, "reward/chosen": -1.859375, "reward/margin": 2.875, "reward/rejected": -4.71875, "step": 3068 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 840.0, "epoch": 0.9205158968206358, "grad_norm": 3.6729543209075928, "learning_rate": 1.5624169281442448e-07, "logp/chosen": -450.0, "logp/rejected": -404.0, "loss": 0.37676239013671875, "reward/accuracy": 0.75, "reward/chosen": -1.3359375, "reward/margin": 2.015625, "reward/rejected": -3.359375, "step": 3069 }, { "approx. KL/chosen": 97.0, "approx. KL/rejected": 704.0, "epoch": 0.9208158368326335, "grad_norm": 1.9869754314422607, "learning_rate": 1.5507525056944528e-07, "logp/chosen": -332.0, "logp/rejected": -382.0, "loss": 0.181854248046875, "reward/accuracy": 1.0, "reward/chosen": -0.60546875, "reward/margin": 2.578125, "reward/rejected": -3.171875, "step": 3070 }, { "approx. KL/chosen": 488.0, "approx. KL/rejected": 1012.0, "epoch": 0.9211157768446311, "grad_norm": 4.373155117034912, "learning_rate": 1.5391311017738997e-07, "logp/chosen": -218.0, "logp/rejected": -346.0, "loss": 0.425567626953125, "reward/accuracy": 0.8125, "reward/chosen": -2.546875, "reward/margin": 1.4296875, "reward/rejected": -3.96875, "step": 3071 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 596.0, "epoch": 0.9214157168566287, "grad_norm": 4.4387736320495605, "learning_rate": 1.5275527267013536e-07, "logp/chosen": -374.0, "logp/rejected": -380.0, "loss": 0.60455322265625, "reward/accuracy": 0.5625, "reward/chosen": -1.8671875, "reward/margin": 0.9375, "reward/rejected": -2.8125, "step": 3072 }, { "approx. KL/chosen": 206.0, "approx. KL/rejected": 510.0, "epoch": 0.9217156568686262, "grad_norm": 3.7749929428100586, "learning_rate": 1.5160173907573405e-07, "logp/chosen": -410.0, "logp/rejected": -460.0, "loss": 0.51873779296875, "reward/accuracy": 0.6875, "reward/chosen": -1.6015625, "reward/margin": 1.03125, "reward/rejected": -2.640625, "step": 3073 }, { "approx. KL/chosen": 212.0, "approx. KL/rejected": 784.0, "epoch": 0.9220155968806238, "grad_norm": 3.20051908493042, "learning_rate": 1.5045251041842234e-07, "logp/chosen": -382.0, "logp/rejected": -326.0, "loss": 0.272125244140625, "reward/accuracy": 0.875, "reward/chosen": -1.4375, "reward/margin": 2.34375, "reward/rejected": -3.78125, "step": 3074 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 872.0, "epoch": 0.9223155368926215, "grad_norm": 3.606400489807129, "learning_rate": 1.493075877186073e-07, "logp/chosen": -302.0, "logp/rejected": -306.0, "loss": 0.3528289794921875, "reward/accuracy": 0.8125, "reward/chosen": -1.6328125, "reward/margin": 2.015625, "reward/rejected": -3.640625, "step": 3075 }, { "approx. KL/chosen": 199.0, "approx. KL/rejected": 712.0, "epoch": 0.9226154769046191, "grad_norm": 3.6435067653656006, "learning_rate": 1.4816697199287855e-07, "logp/chosen": -316.0, "logp/rejected": -352.0, "loss": 0.375732421875, "reward/accuracy": 0.8125, "reward/chosen": -1.6796875, "reward/margin": 1.6640625, "reward/rejected": -3.34375, "step": 3076 }, { "approx. KL/chosen": 147.0, "approx. KL/rejected": 1088.0, "epoch": 0.9229154169166167, "grad_norm": 2.569993495941162, "learning_rate": 1.4703066425399759e-07, "logp/chosen": -294.0, "logp/rejected": -334.0, "loss": 0.2800254821777344, "reward/accuracy": 0.8125, "reward/chosen": -1.203125, "reward/margin": 2.734375, "reward/rejected": -3.9375, "step": 3077 }, { "approx. KL/chosen": 292.0, "approx. KL/rejected": 1040.0, "epoch": 0.9232153569286142, "grad_norm": 4.269279956817627, "learning_rate": 1.4589866551090405e-07, "logp/chosen": -274.0, "logp/rejected": -278.0, "loss": 0.2855224609375, "reward/accuracy": 0.9375, "reward/chosen": -1.8828125, "reward/margin": 2.296875, "reward/rejected": -4.1875, "step": 3078 }, { "approx. KL/chosen": 346.0, "approx. KL/rejected": 800.0, "epoch": 0.9235152969406119, "grad_norm": 4.271581172943115, "learning_rate": 1.4477097676870944e-07, "logp/chosen": -320.0, "logp/rejected": -334.0, "loss": 0.5013427734375, "reward/accuracy": 0.8125, "reward/chosen": -2.171875, "reward/margin": 1.34375, "reward/rejected": -3.515625, "step": 3079 }, { "approx. KL/chosen": 400.0, "approx. KL/rejected": 848.0, "epoch": 0.9238152369526095, "grad_norm": 5.393860340118408, "learning_rate": 1.436475990286984e-07, "logp/chosen": -376.0, "logp/rejected": -302.0, "loss": 0.552001953125, "reward/accuracy": 0.75, "reward/chosen": -2.125, "reward/margin": 1.2578125, "reward/rejected": -3.390625, "step": 3080 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 1000.0, "epoch": 0.9241151769646071, "grad_norm": 3.4854960441589355, "learning_rate": 1.425285332883275e-07, "logp/chosen": -324.0, "logp/rejected": -360.0, "loss": 0.33709716796875, "reward/accuracy": 0.9375, "reward/chosen": -1.8515625, "reward/margin": 2.125, "reward/rejected": -3.96875, "step": 3081 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 636.0, "epoch": 0.9244151169766047, "grad_norm": 3.46140193939209, "learning_rate": 1.41413780541228e-07, "logp/chosen": -392.0, "logp/rejected": -436.0, "loss": 0.33868408203125, "reward/accuracy": 0.875, "reward/chosen": -1.6484375, "reward/margin": 1.453125, "reward/rejected": -3.09375, "step": 3082 }, { "approx. KL/chosen": 138.0, "approx. KL/rejected": 386.0, "epoch": 0.9247150569886022, "grad_norm": 3.9127957820892334, "learning_rate": 1.4030334177719596e-07, "logp/chosen": -438.0, "logp/rejected": -450.0, "loss": 0.61004638671875, "reward/accuracy": 0.75, "reward/chosen": -1.4296875, "reward/margin": 0.9921875, "reward/rejected": -2.421875, "step": 3083 }, { "approx. KL/chosen": 253.0, "approx. KL/rejected": 584.0, "epoch": 0.9250149970005999, "grad_norm": 4.624712944030762, "learning_rate": 1.391972179822032e-07, "logp/chosen": -354.0, "logp/rejected": -340.0, "loss": 0.7525634765625, "reward/accuracy": 0.6875, "reward/chosen": -1.9375, "reward/margin": 0.796875, "reward/rejected": -2.734375, "step": 3084 }, { "approx. KL/chosen": 572.0, "approx. KL/rejected": 1184.0, "epoch": 0.9253149370125975, "grad_norm": 7.365598201751709, "learning_rate": 1.3809541013838467e-07, "logp/chosen": -390.0, "logp/rejected": -396.0, "loss": 0.538482666015625, "reward/accuracy": 0.6875, "reward/chosen": -2.75, "reward/margin": 1.6015625, "reward/rejected": -4.34375, "step": 3085 }, { "approx. KL/chosen": 310.0, "approx. KL/rejected": 852.0, "epoch": 0.9256148770245951, "grad_norm": 3.837226152420044, "learning_rate": 1.3699791922404672e-07, "logp/chosen": -340.0, "logp/rejected": -406.0, "loss": 0.4598541259765625, "reward/accuracy": 0.75, "reward/chosen": -2.03125, "reward/margin": 1.25, "reward/rejected": -3.28125, "step": 3086 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 832.0, "epoch": 0.9259148170365927, "grad_norm": 6.315606117248535, "learning_rate": 1.359047462136609e-07, "logp/chosen": -386.0, "logp/rejected": -342.0, "loss": 0.661865234375, "reward/accuracy": 0.6875, "reward/chosen": -1.984375, "reward/margin": 1.46875, "reward/rejected": -3.453125, "step": 3087 }, { "approx. KL/chosen": 188.0, "approx. KL/rejected": 684.0, "epoch": 0.9262147570485902, "grad_norm": 2.874753952026367, "learning_rate": 1.3481589207786693e-07, "logp/chosen": -292.0, "logp/rejected": -314.0, "loss": 0.45281982421875, "reward/accuracy": 0.6875, "reward/chosen": -1.609375, "reward/margin": 1.5703125, "reward/rejected": -3.1875, "step": 3088 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 756.0, "epoch": 0.9265146970605879, "grad_norm": 3.730426549911499, "learning_rate": 1.33731357783467e-07, "logp/chosen": -282.0, "logp/rejected": -270.0, "loss": 0.47385406494140625, "reward/accuracy": 0.875, "reward/chosen": -1.640625, "reward/margin": 1.7578125, "reward/rejected": -3.390625, "step": 3089 }, { "approx. KL/chosen": 378.0, "approx. KL/rejected": 776.0, "epoch": 0.9268146370725855, "grad_norm": 4.902387619018555, "learning_rate": 1.326511442934292e-07, "logp/chosen": -237.0, "logp/rejected": -224.0, "loss": 0.723876953125, "reward/accuracy": 0.6875, "reward/chosen": -2.078125, "reward/margin": 1.1484375, "reward/rejected": -3.234375, "step": 3090 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 1232.0, "epoch": 0.9271145770845831, "grad_norm": 3.5758748054504395, "learning_rate": 1.3157525256688508e-07, "logp/chosen": -232.0, "logp/rejected": -324.0, "loss": 0.260711669921875, "reward/accuracy": 0.875, "reward/chosen": -1.875, "reward/margin": 2.703125, "reward/rejected": -4.5625, "step": 3091 }, { "approx. KL/chosen": 228.0, "approx. KL/rejected": 792.0, "epoch": 0.9274145170965807, "grad_norm": 3.7635722160339355, "learning_rate": 1.3050368355912891e-07, "logp/chosen": -276.0, "logp/rejected": -318.0, "loss": 0.45206451416015625, "reward/accuracy": 0.75, "reward/chosen": -1.90625, "reward/margin": 1.515625, "reward/rejected": -3.421875, "step": 3092 }, { "approx. KL/chosen": 143.0, "approx. KL/rejected": 724.0, "epoch": 0.9277144571085782, "grad_norm": 3.2208802700042725, "learning_rate": 1.2943643822161621e-07, "logp/chosen": -382.0, "logp/rejected": -428.0, "loss": 0.300811767578125, "reward/accuracy": 0.875, "reward/chosen": -1.1484375, "reward/margin": 2.171875, "reward/rejected": -3.328125, "step": 3093 }, { "approx. KL/chosen": 161.0, "approx. KL/rejected": 964.0, "epoch": 0.9280143971205759, "grad_norm": 3.397111654281616, "learning_rate": 1.2837351750196515e-07, "logp/chosen": -248.0, "logp/rejected": -249.0, "loss": 0.36712646484375, "reward/accuracy": 0.75, "reward/chosen": -1.328125, "reward/margin": 2.296875, "reward/rejected": -3.625, "step": 3094 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 772.0, "epoch": 0.9283143371325735, "grad_norm": 3.9602863788604736, "learning_rate": 1.2731492234395015e-07, "logp/chosen": -352.0, "logp/rejected": -388.0, "loss": 0.50616455078125, "reward/accuracy": 0.625, "reward/chosen": -1.9765625, "reward/margin": 1.234375, "reward/rejected": -3.203125, "step": 3095 }, { "approx. KL/chosen": 120.5, "approx. KL/rejected": 684.0, "epoch": 0.9286142771445711, "grad_norm": 2.8722078800201416, "learning_rate": 1.2626065368750994e-07, "logp/chosen": -380.0, "logp/rejected": -374.0, "loss": 0.31903076171875, "reward/accuracy": 0.8125, "reward/chosen": -1.203125, "reward/margin": 1.875, "reward/rejected": -3.078125, "step": 3096 }, { "approx. KL/chosen": 316.0, "approx. KL/rejected": 708.0, "epoch": 0.9289142171565686, "grad_norm": 4.674255847930908, "learning_rate": 1.2521071246873851e-07, "logp/chosen": -422.0, "logp/rejected": -360.0, "loss": 0.547119140625, "reward/accuracy": 0.625, "reward/chosen": -2.09375, "reward/margin": 1.3984375, "reward/rejected": -3.5, "step": 3097 }, { "approx. KL/chosen": 436.0, "approx. KL/rejected": 956.0, "epoch": 0.9292141571685663, "grad_norm": 4.430501937866211, "learning_rate": 1.2416509961988843e-07, "logp/chosen": -318.0, "logp/rejected": -406.0, "loss": 0.54437255859375, "reward/accuracy": 0.625, "reward/chosen": -2.34375, "reward/margin": 1.40625, "reward/rejected": -3.75, "step": 3098 }, { "approx. KL/chosen": 418.0, "approx. KL/rejected": 676.0, "epoch": 0.9295140971805639, "grad_norm": 5.602142333984375, "learning_rate": 1.231238160693682e-07, "logp/chosen": -256.0, "logp/rejected": -340.0, "loss": 0.8898239135742188, "reward/accuracy": 0.6875, "reward/chosen": -2.03125, "reward/margin": 1.109375, "reward/rejected": -3.140625, "step": 3099 }, { "approx. KL/chosen": 180.0, "approx. KL/rejected": 748.0, "epoch": 0.9298140371925615, "grad_norm": 3.2832915782928467, "learning_rate": 1.220868627417432e-07, "logp/chosen": -426.0, "logp/rejected": -428.0, "loss": 0.29632568359375, "reward/accuracy": 0.875, "reward/chosen": -1.515625, "reward/margin": 2.015625, "reward/rejected": -3.515625, "step": 3100 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 1008.0, "epoch": 0.9301139772045591, "grad_norm": 3.5476362705230713, "learning_rate": 1.2105424055773418e-07, "logp/chosen": -386.0, "logp/rejected": -404.0, "loss": 0.333099365234375, "reward/accuracy": 0.8125, "reward/chosen": -1.84375, "reward/margin": 2.171875, "reward/rejected": -4.03125, "step": 3101 }, { "approx. KL/chosen": 420.0, "approx. KL/rejected": 1160.0, "epoch": 0.9304139172165566, "grad_norm": 4.626169681549072, "learning_rate": 1.200259504342155e-07, "logp/chosen": -247.0, "logp/rejected": -396.0, "loss": 0.441497802734375, "reward/accuracy": 0.75, "reward/chosen": -2.34375, "reward/margin": 1.9609375, "reward/rejected": -4.3125, "step": 3102 }, { "approx. KL/chosen": 240.0, "approx. KL/rejected": 418.0, "epoch": 0.9307138572285543, "grad_norm": 4.543946743011475, "learning_rate": 1.1900199328421624e-07, "logp/chosen": -342.0, "logp/rejected": -296.0, "loss": 0.62738037109375, "reward/accuracy": 0.625, "reward/chosen": -1.7109375, "reward/margin": 0.8046875, "reward/rejected": -2.515625, "step": 3103 }, { "approx. KL/chosen": 256.0, "approx. KL/rejected": 484.0, "epoch": 0.9310137972405519, "grad_norm": 4.8451642990112305, "learning_rate": 1.1798237001691581e-07, "logp/chosen": -446.0, "logp/rejected": -406.0, "loss": 0.614501953125, "reward/accuracy": 0.625, "reward/chosen": -1.53125, "reward/margin": 0.9375, "reward/rejected": -2.46875, "step": 3104 }, { "approx. KL/chosen": 310.0, "approx. KL/rejected": 820.0, "epoch": 0.9313137372525495, "grad_norm": 3.118489980697632, "learning_rate": 1.1696708153764779e-07, "logp/chosen": -326.0, "logp/rejected": -420.0, "loss": 0.30718994140625, "reward/accuracy": 0.875, "reward/chosen": -1.9609375, "reward/margin": 1.78125, "reward/rejected": -3.734375, "step": 3105 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 796.0, "epoch": 0.9316136772645471, "grad_norm": 3.604525566101074, "learning_rate": 1.1595612874789608e-07, "logp/chosen": -288.0, "logp/rejected": -370.0, "loss": 0.42626953125, "reward/accuracy": 0.8125, "reward/chosen": -1.96875, "reward/margin": 1.21875, "reward/rejected": -3.1875, "step": 3106 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 732.0, "epoch": 0.9319136172765446, "grad_norm": 4.953159809112549, "learning_rate": 1.1494951254529485e-07, "logp/chosen": -510.0, "logp/rejected": -438.0, "loss": 0.6170845031738281, "reward/accuracy": 0.75, "reward/chosen": -1.671875, "reward/margin": 1.6171875, "reward/rejected": -3.296875, "step": 3107 }, { "approx. KL/chosen": 346.0, "approx. KL/rejected": 1176.0, "epoch": 0.9322135572885423, "grad_norm": 2.857581377029419, "learning_rate": 1.139472338236286e-07, "logp/chosen": -416.0, "logp/rejected": -394.0, "loss": 0.267974853515625, "reward/accuracy": 0.875, "reward/chosen": -2.109375, "reward/margin": 2.28125, "reward/rejected": -4.40625, "step": 3108 }, { "approx. KL/chosen": 229.0, "approx. KL/rejected": 868.0, "epoch": 0.9325134973005399, "grad_norm": 3.4194042682647705, "learning_rate": 1.1294929347282768e-07, "logp/chosen": -308.0, "logp/rejected": -340.0, "loss": 0.3792266845703125, "reward/accuracy": 0.8125, "reward/chosen": -1.703125, "reward/margin": 1.8125, "reward/rejected": -3.515625, "step": 3109 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 900.0, "epoch": 0.9328134373125375, "grad_norm": 6.562281608581543, "learning_rate": 1.1195569237897441e-07, "logp/chosen": -338.0, "logp/rejected": -448.0, "loss": 0.621337890625, "reward/accuracy": 0.875, "reward/chosen": -1.8671875, "reward/margin": 1.6484375, "reward/rejected": -3.515625, "step": 3110 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 588.0, "epoch": 0.9331133773245351, "grad_norm": 5.321910858154297, "learning_rate": 1.1096643142429531e-07, "logp/chosen": -478.0, "logp/rejected": -370.0, "loss": 0.5745849609375, "reward/accuracy": 0.8125, "reward/chosen": -1.8203125, "reward/margin": 1.203125, "reward/rejected": -3.015625, "step": 3111 }, { "approx. KL/chosen": 99.0, "approx. KL/rejected": 764.0, "epoch": 0.9334133173365327, "grad_norm": 2.692150831222534, "learning_rate": 1.0998151148716496e-07, "logp/chosen": -388.0, "logp/rejected": -382.0, "loss": 0.271148681640625, "reward/accuracy": 0.8125, "reward/chosen": -0.98828125, "reward/margin": 2.421875, "reward/rejected": -3.421875, "step": 3112 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 1176.0, "epoch": 0.9337132573485303, "grad_norm": 4.424884796142578, "learning_rate": 1.0900093344210217e-07, "logp/chosen": -318.0, "logp/rejected": -272.0, "loss": 0.43689727783203125, "reward/accuracy": 0.6875, "reward/chosen": -2.125, "reward/margin": 1.9921875, "reward/rejected": -4.125, "step": 3113 }, { "approx. KL/chosen": 292.0, "approx. KL/rejected": 980.0, "epoch": 0.9340131973605279, "grad_norm": 3.806358814239502, "learning_rate": 1.0802469815977157e-07, "logp/chosen": -336.0, "logp/rejected": -344.0, "loss": 0.472412109375, "reward/accuracy": 0.8125, "reward/chosen": -2.078125, "reward/margin": 1.75, "reward/rejected": -3.828125, "step": 3114 }, { "approx. KL/chosen": 350.0, "approx. KL/rejected": 664.0, "epoch": 0.9343131373725255, "grad_norm": 4.59316349029541, "learning_rate": 1.0705280650698036e-07, "logp/chosen": -358.0, "logp/rejected": -324.0, "loss": 0.47650146484375, "reward/accuracy": 0.6875, "reward/chosen": -1.96875, "reward/margin": 1.2421875, "reward/rejected": -3.203125, "step": 3115 }, { "approx. KL/chosen": 376.0, "approx. KL/rejected": 824.0, "epoch": 0.9346130773845231, "grad_norm": 3.9777655601501465, "learning_rate": 1.0608525934668157e-07, "logp/chosen": -462.0, "logp/rejected": -390.0, "loss": 0.44451904296875, "reward/accuracy": 0.6875, "reward/chosen": -2.0625, "reward/margin": 1.375, "reward/rejected": -3.4375, "step": 3116 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 502.0, "epoch": 0.9349130173965207, "grad_norm": 4.941709518432617, "learning_rate": 1.05122057537968e-07, "logp/chosen": -199.0, "logp/rejected": -248.0, "loss": 0.64111328125, "reward/accuracy": 0.625, "reward/chosen": -1.3671875, "reward/margin": 1.078125, "reward/rejected": -2.453125, "step": 3117 }, { "approx. KL/chosen": 380.0, "approx. KL/rejected": 1072.0, "epoch": 0.9352129574085183, "grad_norm": 4.984161376953125, "learning_rate": 1.0416320193607554e-07, "logp/chosen": -500.0, "logp/rejected": -376.0, "loss": 0.4951171875, "reward/accuracy": 0.875, "reward/chosen": -2.03125, "reward/margin": 2.234375, "reward/rejected": -4.25, "step": 3118 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 800.0, "epoch": 0.9355128974205159, "grad_norm": 3.373159885406494, "learning_rate": 1.0320869339238094e-07, "logp/chosen": -274.0, "logp/rejected": -328.0, "loss": 0.43182373046875, "reward/accuracy": 0.75, "reward/chosen": -1.5546875, "reward/margin": 1.7265625, "reward/rejected": -3.28125, "step": 3119 }, { "approx. KL/chosen": 223.0, "approx. KL/rejected": 880.0, "epoch": 0.9358128374325135, "grad_norm": 4.239603042602539, "learning_rate": 1.0225853275440122e-07, "logp/chosen": -253.0, "logp/rejected": -334.0, "loss": 0.594970703125, "reward/accuracy": 0.625, "reward/chosen": -1.7421875, "reward/margin": 1.5703125, "reward/rejected": -3.3125, "step": 3120 }, { "approx. KL/chosen": 450.0, "approx. KL/rejected": 1048.0, "epoch": 0.936112777444511, "grad_norm": 5.340919494628906, "learning_rate": 1.0131272086579214e-07, "logp/chosen": -374.0, "logp/rejected": -308.0, "loss": 0.7196846008300781, "reward/accuracy": 0.6875, "reward/chosen": -1.984375, "reward/margin": 1.828125, "reward/rejected": -3.8125, "step": 3121 }, { "approx. KL/chosen": 160.0, "approx. KL/rejected": 856.0, "epoch": 0.9364127174565087, "grad_norm": 2.8315603733062744, "learning_rate": 1.0037125856635021e-07, "logp/chosen": -380.0, "logp/rejected": -324.0, "loss": 0.3284454345703125, "reward/accuracy": 0.875, "reward/chosen": -1.3984375, "reward/margin": 2.171875, "reward/rejected": -3.5625, "step": 3122 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 1096.0, "epoch": 0.9367126574685063, "grad_norm": 4.5031585693359375, "learning_rate": 9.943414669200624e-08, "logp/chosen": -247.0, "logp/rejected": -282.0, "loss": 0.40485382080078125, "reward/accuracy": 0.625, "reward/chosen": -2.03125, "reward/margin": 1.859375, "reward/rejected": -3.890625, "step": 3123 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 704.0, "epoch": 0.9370125974805039, "grad_norm": 4.407084941864014, "learning_rate": 9.850138607483184e-08, "logp/chosen": -182.0, "logp/rejected": -302.0, "loss": 0.479034423828125, "reward/accuracy": 0.75, "reward/chosen": -1.734375, "reward/margin": 1.4296875, "reward/rejected": -3.171875, "step": 3124 }, { "approx. KL/chosen": 456.0, "approx. KL/rejected": 684.0, "epoch": 0.9373125374925015, "grad_norm": 5.559093952178955, "learning_rate": 9.757297754303285e-08, "logp/chosen": -552.0, "logp/rejected": -456.0, "loss": 0.592315673828125, "reward/accuracy": 0.75, "reward/chosen": -2.34375, "reward/margin": 0.96875, "reward/rejected": -3.3125, "step": 3125 }, { "approx. KL/chosen": 68.5, "approx. KL/rejected": 1256.0, "epoch": 0.937612477504499, "grad_norm": 0.8054547905921936, "learning_rate": 9.664892192095265e-08, "logp/chosen": -258.0, "logp/rejected": -310.0, "loss": 0.11852836608886719, "reward/accuracy": 1.0, "reward/chosen": -0.87890625, "reward/margin": 3.59375, "reward/rejected": -4.46875, "step": 3126 }, { "approx. KL/chosen": 144.0, "approx. KL/rejected": 712.0, "epoch": 0.9379124175164967, "grad_norm": 3.7777791023254395, "learning_rate": 9.572922002906881e-08, "logp/chosen": -394.0, "logp/rejected": -372.0, "loss": 0.416748046875, "reward/accuracy": 0.6875, "reward/chosen": -1.28125, "reward/margin": 1.765625, "reward/rejected": -3.046875, "step": 3127 }, { "approx. KL/chosen": 107.0, "approx. KL/rejected": 576.0, "epoch": 0.9382123575284943, "grad_norm": 2.4598324298858643, "learning_rate": 9.481387268399144e-08, "logp/chosen": -268.0, "logp/rejected": -274.0, "loss": 0.269927978515625, "reward/accuracy": 0.875, "reward/chosen": -1.2109375, "reward/margin": 1.8125, "reward/rejected": -3.015625, "step": 3128 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 612.0, "epoch": 0.9385122975404919, "grad_norm": 4.287142753601074, "learning_rate": 9.390288069846709e-08, "logp/chosen": -532.0, "logp/rejected": -354.0, "loss": 0.58795166015625, "reward/accuracy": 0.75, "reward/chosen": -1.8046875, "reward/margin": 1.1953125, "reward/rejected": -3.0, "step": 3129 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 740.0, "epoch": 0.9388122375524895, "grad_norm": 3.9403634071350098, "learning_rate": 9.299624488137371e-08, "logp/chosen": -338.0, "logp/rejected": -336.0, "loss": 0.4407958984375, "reward/accuracy": 0.75, "reward/chosen": -1.6015625, "reward/margin": 1.515625, "reward/rejected": -3.125, "step": 3130 }, { "approx. KL/chosen": 131.0, "approx. KL/rejected": 804.0, "epoch": 0.9391121775644871, "grad_norm": 2.3193578720092773, "learning_rate": 9.209396603772125e-08, "logp/chosen": -386.0, "logp/rejected": -300.0, "loss": 0.203704833984375, "reward/accuracy": 0.875, "reward/chosen": -1.234375, "reward/margin": 2.4375, "reward/rejected": -3.671875, "step": 3131 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 728.0, "epoch": 0.9394121175764847, "grad_norm": 4.283134460449219, "learning_rate": 9.119604496865164e-08, "logp/chosen": -238.0, "logp/rejected": -408.0, "loss": 0.4522705078125, "reward/accuracy": 0.6875, "reward/chosen": -1.8828125, "reward/margin": 1.3671875, "reward/rejected": -3.25, "step": 3132 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 676.0, "epoch": 0.9397120575884823, "grad_norm": 3.951582193374634, "learning_rate": 9.030248247143602e-08, "logp/chosen": -448.0, "logp/rejected": -372.0, "loss": 0.495697021484375, "reward/accuracy": 0.8125, "reward/chosen": -1.5078125, "reward/margin": 1.53125, "reward/rejected": -3.046875, "step": 3133 }, { "approx. KL/chosen": 354.0, "approx. KL/rejected": 584.0, "epoch": 0.9400119976004799, "grad_norm": 7.052683353424072, "learning_rate": 8.941327933947863e-08, "logp/chosen": -193.0, "logp/rejected": -189.0, "loss": 0.887451171875, "reward/accuracy": 0.6875, "reward/chosen": -1.9765625, "reward/margin": 0.8046875, "reward/rejected": -2.78125, "step": 3134 }, { "approx. KL/chosen": 211.0, "approx. KL/rejected": 1024.0, "epoch": 0.9403119376124776, "grad_norm": 2.0892434120178223, "learning_rate": 8.852843636231013e-08, "logp/chosen": -245.0, "logp/rejected": -292.0, "loss": 0.18118762969970703, "reward/accuracy": 0.9375, "reward/chosen": -1.421875, "reward/margin": 2.734375, "reward/rejected": -4.15625, "step": 3135 }, { "approx. KL/chosen": 154.0, "approx. KL/rejected": 900.0, "epoch": 0.9406118776244751, "grad_norm": 3.451313018798828, "learning_rate": 8.764795432558981e-08, "logp/chosen": -380.0, "logp/rejected": -288.0, "loss": 0.45197296142578125, "reward/accuracy": 0.75, "reward/chosen": -1.3125, "reward/margin": 2.203125, "reward/rejected": -3.515625, "step": 3136 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 964.0, "epoch": 0.9409118176364727, "grad_norm": 2.594501256942749, "learning_rate": 8.677183401110734e-08, "logp/chosen": -194.0, "logp/rejected": -326.0, "loss": 0.2560882568359375, "reward/accuracy": 0.75, "reward/chosen": -1.1640625, "reward/margin": 2.796875, "reward/rejected": -3.953125, "step": 3137 }, { "approx. KL/chosen": 134.0, "approx. KL/rejected": 736.0, "epoch": 0.9412117576484703, "grad_norm": 4.305449485778809, "learning_rate": 8.590007619677709e-08, "logp/chosen": -368.0, "logp/rejected": -380.0, "loss": 0.42519378662109375, "reward/accuracy": 0.625, "reward/chosen": -1.3203125, "reward/margin": 1.84375, "reward/rejected": -3.171875, "step": 3138 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 540.0, "epoch": 0.9415116976604679, "grad_norm": 6.864510536193848, "learning_rate": 8.503268165663991e-08, "logp/chosen": -512.0, "logp/rejected": -350.0, "loss": 0.5853424072265625, "reward/accuracy": 0.75, "reward/chosen": -1.8125, "reward/margin": 1.171875, "reward/rejected": -2.984375, "step": 3139 }, { "approx. KL/chosen": 520.0, "approx. KL/rejected": 832.0, "epoch": 0.9418116376724655, "grad_norm": 6.616754055023193, "learning_rate": 8.416965116086528e-08, "logp/chosen": -312.0, "logp/rejected": -396.0, "loss": 0.7010498046875, "reward/accuracy": 0.6875, "reward/chosen": -2.328125, "reward/margin": 1.1796875, "reward/rejected": -3.515625, "step": 3140 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 498.0, "epoch": 0.9421115776844631, "grad_norm": 4.483715534210205, "learning_rate": 8.331098547574468e-08, "logp/chosen": -336.0, "logp/rejected": -482.0, "loss": 0.598541259765625, "reward/accuracy": 0.75, "reward/chosen": -1.6640625, "reward/margin": 1.0703125, "reward/rejected": -2.734375, "step": 3141 }, { "approx. KL/chosen": 314.0, "approx. KL/rejected": 988.0, "epoch": 0.9424115176964607, "grad_norm": 4.192022800445557, "learning_rate": 8.245668536369544e-08, "logp/chosen": -354.0, "logp/rejected": -354.0, "loss": 0.45053863525390625, "reward/accuracy": 0.75, "reward/chosen": -1.8125, "reward/margin": 2.0625, "reward/rejected": -3.890625, "step": 3142 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 640.0, "epoch": 0.9427114577084583, "grad_norm": 5.364315509796143, "learning_rate": 8.160675158325748e-08, "logp/chosen": -312.0, "logp/rejected": -416.0, "loss": 0.67724609375, "reward/accuracy": 0.625, "reward/chosen": -1.3203125, "reward/margin": 1.3515625, "reward/rejected": -2.671875, "step": 3143 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 504.0, "epoch": 0.943011397720456, "grad_norm": 3.82084321975708, "learning_rate": 8.076118488909656e-08, "logp/chosen": -260.0, "logp/rejected": -270.0, "loss": 0.5635986328125, "reward/accuracy": 0.75, "reward/chosen": -1.4296875, "reward/margin": 1.171875, "reward/rejected": -2.59375, "step": 3144 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 760.0, "epoch": 0.9433113377324535, "grad_norm": 5.019459247589111, "learning_rate": 7.991998603199824e-08, "logp/chosen": -280.0, "logp/rejected": -340.0, "loss": 0.40972900390625, "reward/accuracy": 0.75, "reward/chosen": -1.9921875, "reward/margin": 1.3828125, "reward/rejected": -3.375, "step": 3145 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 968.0, "epoch": 0.9436112777444511, "grad_norm": 2.9381041526794434, "learning_rate": 7.908315575887004e-08, "logp/chosen": -320.0, "logp/rejected": -346.0, "loss": 0.2952117919921875, "reward/accuracy": 0.875, "reward/chosen": -1.671875, "reward/margin": 2.21875, "reward/rejected": -3.890625, "step": 3146 }, { "approx. KL/chosen": 408.0, "approx. KL/rejected": 1400.0, "epoch": 0.9439112177564487, "grad_norm": 5.030852317810059, "learning_rate": 7.82506948127415e-08, "logp/chosen": -394.0, "logp/rejected": -382.0, "loss": 0.40271759033203125, "reward/accuracy": 0.75, "reward/chosen": -1.6015625, "reward/margin": 2.890625, "reward/rejected": -4.46875, "step": 3147 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 616.0, "epoch": 0.9442111577684463, "grad_norm": 3.324708938598633, "learning_rate": 7.742260393276302e-08, "logp/chosen": -334.0, "logp/rejected": -358.0, "loss": 0.3961639404296875, "reward/accuracy": 0.875, "reward/chosen": -0.8984375, "reward/margin": 2.015625, "reward/rejected": -2.90625, "step": 3148 }, { "approx. KL/chosen": 239.0, "approx. KL/rejected": 696.0, "epoch": 0.944511097780444, "grad_norm": 3.708693265914917, "learning_rate": 7.659888385420366e-08, "logp/chosen": -207.0, "logp/rejected": -198.0, "loss": 0.503387451171875, "reward/accuracy": 0.6875, "reward/chosen": -1.7421875, "reward/margin": 1.46875, "reward/rejected": -3.21875, "step": 3149 }, { "approx. KL/chosen": 183.0, "approx. KL/rejected": 724.0, "epoch": 0.9448110377924415, "grad_norm": 3.8517730236053467, "learning_rate": 7.577953530845172e-08, "logp/chosen": -308.0, "logp/rejected": -326.0, "loss": 0.4840087890625, "reward/accuracy": 0.6875, "reward/chosen": -1.6015625, "reward/margin": 1.5546875, "reward/rejected": -3.15625, "step": 3150 }, { "approx. KL/chosen": 177.0, "approx. KL/rejected": 940.0, "epoch": 0.9451109778044391, "grad_norm": 1.9337302446365356, "learning_rate": 7.496455902301469e-08, "logp/chosen": -370.0, "logp/rejected": -380.0, "loss": 0.19889068603515625, "reward/accuracy": 0.9375, "reward/chosen": -1.265625, "reward/margin": 2.53125, "reward/rejected": -3.796875, "step": 3151 }, { "approx. KL/chosen": 101.0, "approx. KL/rejected": 528.0, "epoch": 0.9454109178164367, "grad_norm": 3.74491024017334, "learning_rate": 7.415395572151706e-08, "logp/chosen": -462.0, "logp/rejected": -472.0, "loss": 0.43865966796875, "reward/accuracy": 0.6875, "reward/chosen": -1.171875, "reward/margin": 1.390625, "reward/rejected": -2.5625, "step": 3152 }, { "approx. KL/chosen": 223.0, "approx. KL/rejected": 856.0, "epoch": 0.9457108578284343, "grad_norm": 3.6594390869140625, "learning_rate": 7.334772612370089e-08, "logp/chosen": -306.0, "logp/rejected": -392.0, "loss": 0.326904296875, "reward/accuracy": 0.75, "reward/chosen": -1.6484375, "reward/margin": 2.1875, "reward/rejected": -3.828125, "step": 3153 }, { "approx. KL/chosen": 474.0, "approx. KL/rejected": 964.0, "epoch": 0.946010797840432, "grad_norm": 6.4785847663879395, "learning_rate": 7.254587094542465e-08, "logp/chosen": -288.0, "logp/rejected": -372.0, "loss": 0.6731815338134766, "reward/accuracy": 0.75, "reward/chosen": -1.90625, "reward/margin": 1.5703125, "reward/rejected": -3.484375, "step": 3154 }, { "approx. KL/chosen": 464.0, "approx. KL/rejected": 864.0, "epoch": 0.9463107378524295, "grad_norm": 6.128223896026611, "learning_rate": 7.174839089866381e-08, "logp/chosen": -320.0, "logp/rejected": -360.0, "loss": 0.420684814453125, "reward/accuracy": 0.75, "reward/chosen": -2.28125, "reward/margin": 1.5, "reward/rejected": -3.78125, "step": 3155 }, { "approx. KL/chosen": 243.0, "approx. KL/rejected": 968.0, "epoch": 0.9466106778644271, "grad_norm": 4.638293743133545, "learning_rate": 7.095528669150753e-08, "logp/chosen": -488.0, "logp/rejected": -404.0, "loss": 0.401519775390625, "reward/accuracy": 0.875, "reward/chosen": -1.78125, "reward/margin": 2.046875, "reward/rejected": -3.828125, "step": 3156 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 366.0, "epoch": 0.9469106178764247, "grad_norm": 4.923223972320557, "learning_rate": 7.016655902815971e-08, "logp/chosen": -304.0, "logp/rejected": -266.0, "loss": 0.67333984375, "reward/accuracy": 0.625, "reward/chosen": -1.8203125, "reward/margin": 0.46875, "reward/rejected": -2.28125, "step": 3157 }, { "approx. KL/chosen": 336.0, "approx. KL/rejected": 520.0, "epoch": 0.9472105578884223, "grad_norm": 3.4638819694519043, "learning_rate": 6.938220860894018e-08, "logp/chosen": -268.0, "logp/rejected": -348.0, "loss": 0.4625244140625, "reward/accuracy": 0.9375, "reward/chosen": -1.9375, "reward/margin": 0.91796875, "reward/rejected": -2.859375, "step": 3158 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 924.0, "epoch": 0.94751049790042, "grad_norm": 5.974075794219971, "learning_rate": 6.860223613027961e-08, "logp/chosen": -294.0, "logp/rejected": -362.0, "loss": 0.3644866943359375, "reward/accuracy": 0.75, "reward/chosen": -1.8125, "reward/margin": 2.015625, "reward/rejected": -3.828125, "step": 3159 }, { "approx. KL/chosen": 324.0, "approx. KL/rejected": 696.0, "epoch": 0.9478104379124175, "grad_norm": 5.174210548400879, "learning_rate": 6.782664228472291e-08, "logp/chosen": -192.0, "logp/rejected": -286.0, "loss": 0.6534423828125, "reward/accuracy": 0.625, "reward/chosen": -1.984375, "reward/margin": 1.390625, "reward/rejected": -3.390625, "step": 3160 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 512.0, "epoch": 0.9481103779244151, "grad_norm": 4.9292402267456055, "learning_rate": 6.70554277609281e-08, "logp/chosen": -300.0, "logp/rejected": -318.0, "loss": 0.59075927734375, "reward/accuracy": 0.6875, "reward/chosen": -1.8671875, "reward/margin": 0.9453125, "reward/rejected": -2.8125, "step": 3161 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 624.0, "epoch": 0.9484103179364127, "grad_norm": 3.3657517433166504, "learning_rate": 6.628859324366243e-08, "logp/chosen": -296.0, "logp/rejected": -278.0, "loss": 0.4251861572265625, "reward/accuracy": 0.75, "reward/chosen": -1.578125, "reward/margin": 1.515625, "reward/rejected": -3.09375, "step": 3162 }, { "approx. KL/chosen": 146.0, "approx. KL/rejected": 844.0, "epoch": 0.9487102579484104, "grad_norm": 2.17977237701416, "learning_rate": 6.552613941380515e-08, "logp/chosen": -280.0, "logp/rejected": -234.0, "loss": 0.19305419921875, "reward/accuracy": 1.0, "reward/chosen": -1.34375, "reward/margin": 2.28125, "reward/rejected": -3.625, "step": 3163 }, { "approx. KL/chosen": 141.0, "approx. KL/rejected": 932.0, "epoch": 0.9490101979604079, "grad_norm": 3.3301916122436523, "learning_rate": 6.476806694834636e-08, "logp/chosen": -354.0, "logp/rejected": -342.0, "loss": 0.3211669921875, "reward/accuracy": 0.8125, "reward/chosen": -1.3046875, "reward/margin": 2.25, "reward/rejected": -3.5625, "step": 3164 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 748.0, "epoch": 0.9493101379724055, "grad_norm": 3.962754011154175, "learning_rate": 6.4014376520386e-08, "logp/chosen": -346.0, "logp/rejected": -354.0, "loss": 0.38451385498046875, "reward/accuracy": 0.75, "reward/chosen": -1.390625, "reward/margin": 2.125, "reward/rejected": -3.515625, "step": 3165 }, { "approx. KL/chosen": 212.0, "approx. KL/rejected": 824.0, "epoch": 0.9496100779844031, "grad_norm": 2.217709541320801, "learning_rate": 6.326506879913208e-08, "logp/chosen": -318.0, "logp/rejected": -412.0, "loss": 0.1820068359375, "reward/accuracy": 0.9375, "reward/chosen": -1.65625, "reward/margin": 2.09375, "reward/rejected": -3.75, "step": 3166 }, { "approx. KL/chosen": 250.0, "approx. KL/rejected": 620.0, "epoch": 0.9499100179964007, "grad_norm": 4.71640157699585, "learning_rate": 6.252014444990184e-08, "logp/chosen": -380.0, "logp/rejected": -310.0, "loss": 0.491973876953125, "reward/accuracy": 0.6875, "reward/chosen": -1.5703125, "reward/margin": 1.4375, "reward/rejected": -3.015625, "step": 3167 }, { "approx. KL/chosen": 242.0, "approx. KL/rejected": 616.0, "epoch": 0.9502099580083984, "grad_norm": 4.655571460723877, "learning_rate": 6.177960413412009e-08, "logp/chosen": -416.0, "logp/rejected": -388.0, "loss": 0.69744873046875, "reward/accuracy": 0.625, "reward/chosen": -1.6796875, "reward/margin": 1.2265625, "reward/rejected": -2.90625, "step": 3168 }, { "approx. KL/chosen": 402.0, "approx. KL/rejected": 1072.0, "epoch": 0.9505098980203959, "grad_norm": 4.165834903717041, "learning_rate": 6.104344850932031e-08, "logp/chosen": -276.0, "logp/rejected": -288.0, "loss": 0.523681640625, "reward/accuracy": 0.875, "reward/chosen": -2.046875, "reward/margin": 1.7890625, "reward/rejected": -3.828125, "step": 3169 }, { "approx. KL/chosen": 200.0, "approx. KL/rejected": 720.0, "epoch": 0.9508098380323935, "grad_norm": 2.949563503265381, "learning_rate": 6.03116782291413e-08, "logp/chosen": -380.0, "logp/rejected": -378.0, "loss": 0.258544921875, "reward/accuracy": 0.9375, "reward/chosen": -1.6015625, "reward/margin": 1.7109375, "reward/rejected": -3.3125, "step": 3170 }, { "approx. KL/chosen": 368.0, "approx. KL/rejected": 612.0, "epoch": 0.9511097780443911, "grad_norm": 6.022570610046387, "learning_rate": 5.958429394332832e-08, "logp/chosen": -380.0, "logp/rejected": -414.0, "loss": 0.82177734375, "reward/accuracy": 0.4375, "reward/chosen": -2.296875, "reward/margin": 0.52734375, "reward/rejected": -2.828125, "step": 3171 }, { "approx. KL/chosen": 292.0, "approx. KL/rejected": 1088.0, "epoch": 0.9514097180563887, "grad_norm": 4.028537750244141, "learning_rate": 5.886129629773252e-08, "logp/chosen": -388.0, "logp/rejected": -388.0, "loss": 0.42474365234375, "reward/accuracy": 0.8125, "reward/chosen": -1.5078125, "reward/margin": 2.234375, "reward/rejected": -3.734375, "step": 3172 }, { "approx. KL/chosen": 270.0, "approx. KL/rejected": 632.0, "epoch": 0.9517096580683864, "grad_norm": 4.918576240539551, "learning_rate": 5.8142685934310384e-08, "logp/chosen": -448.0, "logp/rejected": -384.0, "loss": 0.587890625, "reward/accuracy": 0.625, "reward/chosen": -2.015625, "reward/margin": 0.9609375, "reward/rejected": -2.96875, "step": 3173 }, { "approx. KL/chosen": 326.0, "approx. KL/rejected": 828.0, "epoch": 0.9520095980803839, "grad_norm": 4.7082600593566895, "learning_rate": 5.742846349112208e-08, "logp/chosen": -328.0, "logp/rejected": -304.0, "loss": 0.6021728515625, "reward/accuracy": 0.8125, "reward/chosen": -1.6484375, "reward/margin": 1.8125, "reward/rejected": -3.453125, "step": 3174 }, { "approx. KL/chosen": 241.0, "approx. KL/rejected": 388.0, "epoch": 0.9523095380923815, "grad_norm": 5.461798191070557, "learning_rate": 5.67186296023331e-08, "logp/chosen": -338.0, "logp/rejected": -336.0, "loss": 0.5457763671875, "reward/accuracy": 0.8125, "reward/chosen": -1.3203125, "reward/margin": 1.125, "reward/rejected": -2.4375, "step": 3175 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 1008.0, "epoch": 0.9526094781043791, "grad_norm": 3.613534688949585, "learning_rate": 5.601318489821095e-08, "logp/chosen": -318.0, "logp/rejected": -326.0, "loss": 0.3382110595703125, "reward/accuracy": 0.875, "reward/chosen": -2.03125, "reward/margin": 2.125, "reward/rejected": -4.15625, "step": 3176 }, { "approx. KL/chosen": 249.0, "approx. KL/rejected": 636.0, "epoch": 0.9529094181163767, "grad_norm": 4.751700401306152, "learning_rate": 5.531213000512681e-08, "logp/chosen": -294.0, "logp/rejected": -326.0, "loss": 0.5332794189453125, "reward/accuracy": 0.5625, "reward/chosen": -1.7890625, "reward/margin": 1.21875, "reward/rejected": -3.015625, "step": 3177 }, { "approx. KL/chosen": 280.0, "approx. KL/rejected": 576.0, "epoch": 0.9532093581283744, "grad_norm": 4.800091743469238, "learning_rate": 5.461546554555275e-08, "logp/chosen": -290.0, "logp/rejected": -300.0, "loss": 0.742919921875, "reward/accuracy": 0.625, "reward/chosen": -1.8515625, "reward/margin": 0.97265625, "reward/rejected": -2.828125, "step": 3178 }, { "approx. KL/chosen": 177.0, "approx. KL/rejected": 1256.0, "epoch": 0.9535092981403719, "grad_norm": 2.1704037189483643, "learning_rate": 5.392319213806452e-08, "logp/chosen": -370.0, "logp/rejected": -380.0, "loss": 0.18911361694335938, "reward/accuracy": 0.9375, "reward/chosen": -1.2109375, "reward/margin": 3.015625, "reward/rejected": -4.21875, "step": 3179 }, { "approx. KL/chosen": 231.0, "approx. KL/rejected": 510.0, "epoch": 0.9538092381523695, "grad_norm": 3.922330617904663, "learning_rate": 5.323531039733765e-08, "logp/chosen": -254.0, "logp/rejected": -248.0, "loss": 0.5150146484375, "reward/accuracy": 0.8125, "reward/chosen": -1.578125, "reward/margin": 1.2734375, "reward/rejected": -2.859375, "step": 3180 }, { "approx. KL/chosen": 306.0, "approx. KL/rejected": 836.0, "epoch": 0.9541091781643671, "grad_norm": 4.774270534515381, "learning_rate": 5.255182093414857e-08, "logp/chosen": -318.0, "logp/rejected": -322.0, "loss": 0.6580810546875, "reward/accuracy": 0.8125, "reward/chosen": -1.9453125, "reward/margin": 1.78125, "reward/rejected": -3.734375, "step": 3181 }, { "approx. KL/chosen": 324.0, "approx. KL/rejected": 816.0, "epoch": 0.9544091181763648, "grad_norm": 3.6014115810394287, "learning_rate": 5.187272435537405e-08, "logp/chosen": -490.0, "logp/rejected": -390.0, "loss": 0.4192352294921875, "reward/accuracy": 0.875, "reward/chosen": -2.21875, "reward/margin": 1.4296875, "reward/rejected": -3.65625, "step": 3182 }, { "approx. KL/chosen": 181.0, "approx. KL/rejected": 672.0, "epoch": 0.9547090581883624, "grad_norm": 3.9115703105926514, "learning_rate": 5.119802126398954e-08, "logp/chosen": -302.0, "logp/rejected": -282.0, "loss": 0.463958740234375, "reward/accuracy": 0.8125, "reward/chosen": -1.515625, "reward/margin": 1.5078125, "reward/rejected": -3.015625, "step": 3183 }, { "approx. KL/chosen": 892.0, "approx. KL/rejected": 1280.0, "epoch": 0.9550089982003599, "grad_norm": 10.002387046813965, "learning_rate": 5.052771225907027e-08, "logp/chosen": -430.0, "logp/rejected": -392.0, "loss": 0.83807373046875, "reward/accuracy": 0.6875, "reward/chosen": -3.03125, "reward/margin": 1.34375, "reward/rejected": -4.375, "step": 3184 }, { "approx. KL/chosen": 432.0, "approx. KL/rejected": 808.0, "epoch": 0.9553089382123575, "grad_norm": 4.068079948425293, "learning_rate": 4.986179793579071e-08, "logp/chosen": -256.0, "logp/rejected": -268.0, "loss": 0.4024658203125, "reward/accuracy": 0.8125, "reward/chosen": -2.40625, "reward/margin": 1.328125, "reward/rejected": -3.734375, "step": 3185 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 1256.0, "epoch": 0.9556088782243551, "grad_norm": 2.128899097442627, "learning_rate": 4.920027888542123e-08, "logp/chosen": -255.0, "logp/rejected": -292.0, "loss": 0.18359375, "reward/accuracy": 1.0, "reward/chosen": -1.484375, "reward/margin": 3.046875, "reward/rejected": -4.53125, "step": 3186 }, { "approx. KL/chosen": 364.0, "approx. KL/rejected": 1048.0, "epoch": 0.9559088182363528, "grad_norm": 5.590826511383057, "learning_rate": 4.85431556953303e-08, "logp/chosen": -466.0, "logp/rejected": -508.0, "loss": 0.3729705810546875, "reward/accuracy": 0.75, "reward/chosen": -1.7421875, "reward/margin": 2.28125, "reward/rejected": -4.03125, "step": 3187 }, { "approx. KL/chosen": 288.0, "approx. KL/rejected": 1384.0, "epoch": 0.9562087582483503, "grad_norm": 5.1672587394714355, "learning_rate": 4.7890428948984544e-08, "logp/chosen": -274.0, "logp/rejected": -444.0, "loss": 0.2974090576171875, "reward/accuracy": 0.875, "reward/chosen": -1.796875, "reward/margin": 2.859375, "reward/rejected": -4.65625, "step": 3188 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 1000.0, "epoch": 0.9565086982603479, "grad_norm": 3.0931873321533203, "learning_rate": 4.724209922594647e-08, "logp/chosen": -302.0, "logp/rejected": -336.0, "loss": 0.328033447265625, "reward/accuracy": 0.75, "reward/chosen": -1.5625, "reward/margin": 2.390625, "reward/rejected": -3.953125, "step": 3189 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 860.0, "epoch": 0.9568086382723455, "grad_norm": 5.376116752624512, "learning_rate": 4.659816710187171e-08, "logp/chosen": -374.0, "logp/rejected": -342.0, "loss": 0.5303573608398438, "reward/accuracy": 0.6875, "reward/chosen": -1.609375, "reward/margin": 1.7890625, "reward/rejected": -3.390625, "step": 3190 }, { "approx. KL/chosen": 346.0, "approx. KL/rejected": 648.0, "epoch": 0.9571085782843431, "grad_norm": 6.459821701049805, "learning_rate": 4.5958633148515674e-08, "logp/chosen": -350.0, "logp/rejected": -496.0, "loss": 0.8870849609375, "reward/accuracy": 0.5, "reward/chosen": -2.25, "reward/margin": 0.478515625, "reward/rejected": -2.734375, "step": 3191 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 720.0, "epoch": 0.9574085182963408, "grad_norm": 8.172962188720703, "learning_rate": 4.5323497933725814e-08, "logp/chosen": -326.0, "logp/rejected": -398.0, "loss": 0.6463623046875, "reward/accuracy": 0.6875, "reward/chosen": -2.015625, "reward/margin": 1.2265625, "reward/rejected": -3.234375, "step": 3192 }, { "approx. KL/chosen": 360.0, "approx. KL/rejected": 604.0, "epoch": 0.9577084583083383, "grad_norm": 4.910424709320068, "learning_rate": 4.469276202144379e-08, "logp/chosen": -350.0, "logp/rejected": -390.0, "loss": 0.59490966796875, "reward/accuracy": 0.625, "reward/chosen": -1.9140625, "reward/margin": 0.953125, "reward/rejected": -2.875, "step": 3193 }, { "approx. KL/chosen": 376.0, "approx. KL/rejected": 768.0, "epoch": 0.9580083983203359, "grad_norm": 4.787553787231445, "learning_rate": 4.406642597170663e-08, "logp/chosen": -392.0, "logp/rejected": -456.0, "loss": 0.496063232421875, "reward/accuracy": 0.6875, "reward/chosen": -2.328125, "reward/margin": 1.3046875, "reward/rejected": -3.640625, "step": 3194 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 668.0, "epoch": 0.9583083383323335, "grad_norm": 3.223705530166626, "learning_rate": 4.344449034064335e-08, "logp/chosen": -324.0, "logp/rejected": -324.0, "loss": 0.34979248046875, "reward/accuracy": 0.75, "reward/chosen": -1.390625, "reward/margin": 1.703125, "reward/rejected": -3.09375, "step": 3195 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 968.0, "epoch": 0.9586082783443312, "grad_norm": 3.4638915061950684, "learning_rate": 4.2826955680476676e-08, "logp/chosen": -384.0, "logp/rejected": -342.0, "loss": 0.307861328125, "reward/accuracy": 0.875, "reward/chosen": -1.265625, "reward/margin": 2.375, "reward/rejected": -3.640625, "step": 3196 }, { "approx. KL/chosen": 292.0, "approx. KL/rejected": 840.0, "epoch": 0.9589082183563288, "grad_norm": 4.0990824699401855, "learning_rate": 4.221382253952078e-08, "logp/chosen": -318.0, "logp/rejected": -356.0, "loss": 0.52001953125, "reward/accuracy": 0.625, "reward/chosen": -1.9375, "reward/margin": 1.3515625, "reward/rejected": -3.296875, "step": 3197 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 532.0, "epoch": 0.9592081583683263, "grad_norm": 5.000025749206543, "learning_rate": 4.160509146218239e-08, "logp/chosen": -316.0, "logp/rejected": -298.0, "loss": 0.6820755004882812, "reward/accuracy": 0.6875, "reward/chosen": -1.984375, "reward/margin": 0.86328125, "reward/rejected": -2.84375, "step": 3198 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 608.0, "epoch": 0.9595080983803239, "grad_norm": 2.7989132404327393, "learning_rate": 4.100076298895972e-08, "logp/chosen": -274.0, "logp/rejected": -204.0, "loss": 0.403045654296875, "reward/accuracy": 0.8125, "reward/chosen": -1.328125, "reward/margin": 1.6953125, "reward/rejected": -3.015625, "step": 3199 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 700.0, "epoch": 0.9598080383923215, "grad_norm": 3.552927255630493, "learning_rate": 4.0400837656440763e-08, "logp/chosen": -346.0, "logp/rejected": -316.0, "loss": 0.378204345703125, "reward/accuracy": 0.875, "reward/chosen": -1.421875, "reward/margin": 1.6015625, "reward/rejected": -3.03125, "step": 3200 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 788.0, "epoch": 0.9601079784043192, "grad_norm": 5.3608269691467285, "learning_rate": 3.980531599730553e-08, "logp/chosen": -304.0, "logp/rejected": -284.0, "loss": 0.626312255859375, "reward/accuracy": 0.8125, "reward/chosen": -1.9453125, "reward/margin": 1.640625, "reward/rejected": -3.59375, "step": 3201 }, { "approx. KL/chosen": 328.0, "approx. KL/rejected": 700.0, "epoch": 0.9604079184163168, "grad_norm": 4.025485515594482, "learning_rate": 3.921419854032216e-08, "logp/chosen": -402.0, "logp/rejected": -282.0, "loss": 0.416290283203125, "reward/accuracy": 0.75, "reward/chosen": -1.90625, "reward/margin": 1.5390625, "reward/rejected": -3.453125, "step": 3202 }, { "approx. KL/chosen": 121.5, "approx. KL/rejected": 784.0, "epoch": 0.9607078584283143, "grad_norm": 2.586143970489502, "learning_rate": 3.862748581035025e-08, "logp/chosen": -270.0, "logp/rejected": -308.0, "loss": 0.260955810546875, "reward/accuracy": 0.9375, "reward/chosen": -1.2421875, "reward/margin": 2.25, "reward/rejected": -3.5, "step": 3203 }, { "approx. KL/chosen": 366.0, "approx. KL/rejected": 800.0, "epoch": 0.9610077984403119, "grad_norm": 4.478282451629639, "learning_rate": 3.804517832833699e-08, "logp/chosen": -314.0, "logp/rejected": -306.0, "loss": 0.7740478515625, "reward/accuracy": 0.6875, "reward/chosen": -2.328125, "reward/margin": 1.0859375, "reward/rejected": -3.40625, "step": 3204 }, { "approx. KL/chosen": 424.0, "approx. KL/rejected": 744.0, "epoch": 0.9613077384523095, "grad_norm": 6.197026252746582, "learning_rate": 3.746727661131877e-08, "logp/chosen": -362.0, "logp/rejected": -326.0, "loss": 0.631103515625, "reward/accuracy": 0.5625, "reward/chosen": -1.9609375, "reward/margin": 1.28125, "reward/rejected": -3.25, "step": 3205 }, { "approx. KL/chosen": 314.0, "approx. KL/rejected": 616.0, "epoch": 0.9616076784643072, "grad_norm": 3.686279535293579, "learning_rate": 3.689378117241904e-08, "logp/chosen": -360.0, "logp/rejected": -276.0, "loss": 0.442474365234375, "reward/accuracy": 0.875, "reward/chosen": -1.7734375, "reward/margin": 1.234375, "reward/rejected": -3.015625, "step": 3206 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 616.0, "epoch": 0.9619076184763048, "grad_norm": 4.992669105529785, "learning_rate": 3.6324692520851025e-08, "logp/chosen": -318.0, "logp/rejected": -278.0, "loss": 0.650482177734375, "reward/accuracy": 0.875, "reward/chosen": -1.65625, "reward/margin": 1.3125, "reward/rejected": -2.96875, "step": 3207 }, { "approx. KL/chosen": 482.0, "approx. KL/rejected": 640.0, "epoch": 0.9622075584883023, "grad_norm": 7.374763488769531, "learning_rate": 3.5760011161911636e-08, "logp/chosen": -352.0, "logp/rejected": -332.0, "loss": 0.81298828125, "reward/accuracy": 0.6875, "reward/chosen": -2.4375, "reward/margin": 0.94921875, "reward/rejected": -3.390625, "step": 3208 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 656.0, "epoch": 0.9625074985002999, "grad_norm": 3.3166747093200684, "learning_rate": 3.519973759698925e-08, "logp/chosen": -310.0, "logp/rejected": -320.0, "loss": 0.38232421875, "reward/accuracy": 0.8125, "reward/chosen": -1.5, "reward/margin": 1.65625, "reward/rejected": -3.15625, "step": 3209 }, { "approx. KL/chosen": 224.0, "approx. KL/rejected": 820.0, "epoch": 0.9628074385122976, "grad_norm": 6.515950679779053, "learning_rate": 3.4643872323553154e-08, "logp/chosen": -298.0, "logp/rejected": -330.0, "loss": 0.5159072875976562, "reward/accuracy": 0.75, "reward/chosen": -1.5625, "reward/margin": 1.9609375, "reward/rejected": -3.53125, "step": 3210 }, { "approx. KL/chosen": 274.0, "approx. KL/rejected": 624.0, "epoch": 0.9631073785242952, "grad_norm": 3.6669745445251465, "learning_rate": 3.4092415835162987e-08, "logp/chosen": -402.0, "logp/rejected": -326.0, "loss": 0.34368896484375, "reward/accuracy": 0.875, "reward/chosen": -1.90625, "reward/margin": 1.40625, "reward/rejected": -3.3125, "step": 3211 }, { "approx. KL/chosen": 446.0, "approx. KL/rejected": 1064.0, "epoch": 0.9634073185362927, "grad_norm": 4.895343780517578, "learning_rate": 3.3545368621461514e-08, "logp/chosen": -360.0, "logp/rejected": -360.0, "loss": 0.5145263671875, "reward/accuracy": 0.6875, "reward/chosen": -2.125, "reward/margin": 1.546875, "reward/rejected": -3.671875, "step": 3212 }, { "approx. KL/chosen": 135.0, "approx. KL/rejected": 584.0, "epoch": 0.9637072585482903, "grad_norm": 3.3415610790252686, "learning_rate": 3.3002731168177424e-08, "logp/chosen": -446.0, "logp/rejected": -312.0, "loss": 0.395538330078125, "reward/accuracy": 0.75, "reward/chosen": -1.3046875, "reward/margin": 1.6796875, "reward/rejected": -2.984375, "step": 3213 }, { "approx. KL/chosen": 278.0, "approx. KL/rejected": 484.0, "epoch": 0.9640071985602879, "grad_norm": 6.115284442901611, "learning_rate": 3.246450395712253e-08, "logp/chosen": -308.0, "logp/rejected": -278.0, "loss": 0.5908203125, "reward/accuracy": 0.75, "reward/chosen": -1.65625, "reward/margin": 1.1640625, "reward/rejected": -2.828125, "step": 3214 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 896.0, "epoch": 0.9643071385722856, "grad_norm": 3.9838829040527344, "learning_rate": 3.193068746619399e-08, "logp/chosen": -304.0, "logp/rejected": -342.0, "loss": 0.578125, "reward/accuracy": 0.75, "reward/chosen": -1.5546875, "reward/margin": 1.65625, "reward/rejected": -3.21875, "step": 3215 }, { "approx. KL/chosen": 140.0, "approx. KL/rejected": 536.0, "epoch": 0.9646070785842832, "grad_norm": 3.893441915512085, "learning_rate": 3.140128216937266e-08, "logp/chosen": -434.0, "logp/rejected": -392.0, "loss": 0.4572906494140625, "reward/accuracy": 0.625, "reward/chosen": -1.3046875, "reward/margin": 1.5390625, "reward/rejected": -2.84375, "step": 3216 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 840.0, "epoch": 0.9649070185962807, "grad_norm": 2.3479719161987305, "learning_rate": 3.087628853672253e-08, "logp/chosen": -360.0, "logp/rejected": -250.0, "loss": 0.2176513671875, "reward/accuracy": 0.9375, "reward/chosen": -1.78125, "reward/margin": 2.0, "reward/rejected": -3.78125, "step": 3217 }, { "approx. KL/chosen": 205.0, "approx. KL/rejected": 788.0, "epoch": 0.9652069586082783, "grad_norm": 3.052203893661499, "learning_rate": 3.035570703438962e-08, "logp/chosen": -444.0, "logp/rejected": -490.0, "loss": 0.33367919921875, "reward/accuracy": 0.9375, "reward/chosen": -1.4453125, "reward/margin": 2.0625, "reward/rejected": -3.515625, "step": 3218 }, { "approx. KL/chosen": 310.0, "approx. KL/rejected": 836.0, "epoch": 0.9655068986202759, "grad_norm": 4.7057085037231445, "learning_rate": 2.983953812460305e-08, "logp/chosen": -310.0, "logp/rejected": -251.0, "loss": 0.614654541015625, "reward/accuracy": 0.6875, "reward/chosen": -1.9140625, "reward/margin": 1.5078125, "reward/rejected": -3.421875, "step": 3219 }, { "approx. KL/chosen": 129.0, "approx. KL/rejected": 1096.0, "epoch": 0.9658068386322736, "grad_norm": 1.729284405708313, "learning_rate": 2.932778226567401e-08, "logp/chosen": -528.0, "logp/rejected": -424.0, "loss": 0.1346893310546875, "reward/accuracy": 1.0, "reward/chosen": -1.2578125, "reward/margin": 2.984375, "reward/rejected": -4.25, "step": 3220 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 1152.0, "epoch": 0.9661067786442712, "grad_norm": 1.9074865579605103, "learning_rate": 2.882043991199568e-08, "logp/chosen": -338.0, "logp/rejected": -284.0, "loss": 0.22175216674804688, "reward/accuracy": 0.9375, "reward/chosen": -1.609375, "reward/margin": 2.71875, "reward/rejected": -4.34375, "step": 3221 }, { "approx. KL/chosen": 213.0, "approx. KL/rejected": 1032.0, "epoch": 0.9664067186562687, "grad_norm": 3.3132574558258057, "learning_rate": 2.8317511514041607e-08, "logp/chosen": -320.0, "logp/rejected": -306.0, "loss": 0.24633026123046875, "reward/accuracy": 0.875, "reward/chosen": -1.65625, "reward/margin": 2.5, "reward/rejected": -4.15625, "step": 3222 }, { "approx. KL/chosen": 132.0, "approx. KL/rejected": 444.0, "epoch": 0.9667066586682663, "grad_norm": 4.1578874588012695, "learning_rate": 2.7818997518366808e-08, "logp/chosen": -245.0, "logp/rejected": -224.0, "loss": 0.42877197265625, "reward/accuracy": 0.8125, "reward/chosen": -1.0234375, "reward/margin": 1.3984375, "reward/rejected": -2.421875, "step": 3223 }, { "approx. KL/chosen": 101.5, "approx. KL/rejected": 748.0, "epoch": 0.967006598680264, "grad_norm": 2.7907698154449463, "learning_rate": 2.7324898367605545e-08, "logp/chosen": -336.0, "logp/rejected": -314.0, "loss": 0.286865234375, "reward/accuracy": 0.8125, "reward/chosen": -1.0703125, "reward/margin": 2.296875, "reward/rejected": -3.375, "step": 3224 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 472.0, "epoch": 0.9673065386922616, "grad_norm": 4.14032506942749, "learning_rate": 2.6835214500474106e-08, "logp/chosen": -314.0, "logp/rejected": -396.0, "loss": 0.497650146484375, "reward/accuracy": 0.75, "reward/chosen": -1.375, "reward/margin": 1.390625, "reward/rejected": -2.765625, "step": 3225 }, { "approx. KL/chosen": 179.0, "approx. KL/rejected": 980.0, "epoch": 0.9676064787042592, "grad_norm": 2.5289344787597656, "learning_rate": 2.63499463517658e-08, "logp/chosen": -318.0, "logp/rejected": -348.0, "loss": 0.2403411865234375, "reward/accuracy": 0.875, "reward/chosen": -1.5546875, "reward/margin": 2.390625, "reward/rejected": -3.953125, "step": 3226 }, { "approx. KL/chosen": 340.0, "approx. KL/rejected": 700.0, "epoch": 0.9679064187162567, "grad_norm": 5.343683242797852, "learning_rate": 2.5869094352356516e-08, "logp/chosen": -322.0, "logp/rejected": -366.0, "loss": 0.74853515625, "reward/accuracy": 0.4375, "reward/chosen": -2.265625, "reward/margin": 0.6953125, "reward/rejected": -2.953125, "step": 3227 }, { "approx. KL/chosen": 237.0, "approx. KL/rejected": 660.0, "epoch": 0.9682063587282543, "grad_norm": 4.4013848304748535, "learning_rate": 2.539265892919751e-08, "logp/chosen": -358.0, "logp/rejected": -412.0, "loss": 0.45050048828125, "reward/accuracy": 0.75, "reward/chosen": -1.5546875, "reward/margin": 1.5625, "reward/rejected": -3.109375, "step": 3228 }, { "approx. KL/chosen": 149.0, "approx. KL/rejected": 892.0, "epoch": 0.968506298740252, "grad_norm": 3.33663272857666, "learning_rate": 2.4920640505320947e-08, "logp/chosen": -286.0, "logp/rejected": -240.0, "loss": 0.349609375, "reward/accuracy": 0.875, "reward/chosen": -1.484375, "reward/margin": 1.9453125, "reward/rejected": -3.421875, "step": 3229 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 684.0, "epoch": 0.9688062387522496, "grad_norm": 2.5069892406463623, "learning_rate": 2.4453039499836574e-08, "logp/chosen": -384.0, "logp/rejected": -278.0, "loss": 0.3040313720703125, "reward/accuracy": 0.8125, "reward/chosen": -1.0625, "reward/margin": 2.046875, "reward/rejected": -3.109375, "step": 3230 }, { "approx. KL/chosen": 402.0, "approx. KL/rejected": 940.0, "epoch": 0.9691061787642472, "grad_norm": 6.185847282409668, "learning_rate": 2.3989856327930604e-08, "logp/chosen": -392.0, "logp/rejected": -344.0, "loss": 0.8177490234375, "reward/accuracy": 0.6875, "reward/chosen": -2.34375, "reward/margin": 1.3125, "reward/rejected": -3.65625, "step": 3231 }, { "approx. KL/chosen": 128.0, "approx. KL/rejected": 1288.0, "epoch": 0.9694061187762447, "grad_norm": 2.851783514022827, "learning_rate": 2.353109140086851e-08, "logp/chosen": -388.0, "logp/rejected": -434.0, "loss": 0.3345794677734375, "reward/accuracy": 0.75, "reward/chosen": -1.1484375, "reward/margin": 2.984375, "reward/rejected": -4.125, "step": 3232 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 876.0, "epoch": 0.9697060587882423, "grad_norm": 3.698869466781616, "learning_rate": 2.3076745125991674e-08, "logp/chosen": -416.0, "logp/rejected": -312.0, "loss": 0.31353759765625, "reward/accuracy": 0.8125, "reward/chosen": -1.75, "reward/margin": 2.0, "reward/rejected": -3.75, "step": 3233 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 1240.0, "epoch": 0.97000599880024, "grad_norm": 3.723581314086914, "learning_rate": 2.262681790671739e-08, "logp/chosen": -364.0, "logp/rejected": -362.0, "loss": 0.290008544921875, "reward/accuracy": 0.8125, "reward/chosen": -1.578125, "reward/margin": 2.828125, "reward/rejected": -4.40625, "step": 3234 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 652.0, "epoch": 0.9703059388122376, "grad_norm": 3.791323661804199, "learning_rate": 2.2181310142541657e-08, "logp/chosen": -233.0, "logp/rejected": -270.0, "loss": 0.4374847412109375, "reward/accuracy": 0.75, "reward/chosen": -1.75, "reward/margin": 1.4921875, "reward/rejected": -3.25, "step": 3235 }, { "approx. KL/chosen": 296.0, "approx. KL/rejected": 868.0, "epoch": 0.9706058788242351, "grad_norm": 2.848104238510132, "learning_rate": 2.1740222229033602e-08, "logp/chosen": -458.0, "logp/rejected": -372.0, "loss": 0.31787109375, "reward/accuracy": 0.875, "reward/chosen": -1.734375, "reward/margin": 1.90625, "reward/rejected": -3.640625, "step": 3236 }, { "approx. KL/chosen": 139.0, "approx. KL/rejected": 812.0, "epoch": 0.9709058188362327, "grad_norm": 2.062654972076416, "learning_rate": 2.1303554557839945e-08, "logp/chosen": -270.0, "logp/rejected": -306.0, "loss": 0.16522216796875, "reward/accuracy": 0.9375, "reward/chosen": -1.234375, "reward/margin": 2.53125, "reward/rejected": -3.765625, "step": 3237 }, { "approx. KL/chosen": 153.0, "approx. KL/rejected": 1064.0, "epoch": 0.9712057588482303, "grad_norm": 6.228185176849365, "learning_rate": 2.0871307516681648e-08, "logp/chosen": -352.0, "logp/rejected": -422.0, "loss": 0.36151123046875, "reward/accuracy": 0.8125, "reward/chosen": -1.25, "reward/margin": 2.234375, "reward/rejected": -3.46875, "step": 3238 }, { "approx. KL/chosen": 151.0, "approx. KL/rejected": 912.0, "epoch": 0.971505698860228, "grad_norm": 3.533698320388794, "learning_rate": 2.0443481489355045e-08, "logp/chosen": -308.0, "logp/rejected": -326.0, "loss": 0.2863578796386719, "reward/accuracy": 0.875, "reward/chosen": -1.4765625, "reward/margin": 2.296875, "reward/rejected": -3.765625, "step": 3239 }, { "approx. KL/chosen": 544.0, "approx. KL/rejected": 948.0, "epoch": 0.9718056388722256, "grad_norm": 5.848905563354492, "learning_rate": 2.0020076855730154e-08, "logp/chosen": -366.0, "logp/rejected": -318.0, "loss": 0.597625732421875, "reward/accuracy": 0.6875, "reward/chosen": -2.734375, "reward/margin": 1.25, "reward/rejected": -3.984375, "step": 3240 }, { "approx. KL/chosen": 204.0, "approx. KL/rejected": 928.0, "epoch": 0.9721055788842231, "grad_norm": 2.4649627208709717, "learning_rate": 1.9601093991752918e-08, "logp/chosen": -374.0, "logp/rejected": -388.0, "loss": 0.256134033203125, "reward/accuracy": 0.875, "reward/chosen": -1.5, "reward/margin": 2.203125, "reward/rejected": -3.703125, "step": 3241 }, { "approx. KL/chosen": 458.0, "approx. KL/rejected": 510.0, "epoch": 0.9724055188962207, "grad_norm": 7.583611965179443, "learning_rate": 1.9186533269440755e-08, "logp/chosen": -318.0, "logp/rejected": -376.0, "loss": 0.7431640625, "reward/accuracy": 0.8125, "reward/chosen": -2.046875, "reward/margin": 0.88671875, "reward/rejected": -2.9375, "step": 3242 }, { "approx. KL/chosen": 251.0, "approx. KL/rejected": 636.0, "epoch": 0.9727054589082184, "grad_norm": 4.451012134552002, "learning_rate": 1.8776395056886996e-08, "logp/chosen": -416.0, "logp/rejected": -392.0, "loss": 0.4739990234375, "reward/accuracy": 0.75, "reward/chosen": -1.7734375, "reward/margin": 1.3828125, "reward/rejected": -3.15625, "step": 3243 }, { "approx. KL/chosen": 294.0, "approx. KL/rejected": 692.0, "epoch": 0.973005398920216, "grad_norm": 3.054064989089966, "learning_rate": 1.8370679718256457e-08, "logp/chosen": -308.0, "logp/rejected": -262.0, "loss": 0.45538330078125, "reward/accuracy": 0.8125, "reward/chosen": -1.8125, "reward/margin": 1.2890625, "reward/rejected": -3.109375, "step": 3244 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 660.0, "epoch": 0.9733053389322136, "grad_norm": 3.8577611446380615, "learning_rate": 1.7969387613787636e-08, "logp/chosen": -436.0, "logp/rejected": -288.0, "loss": 0.4556884765625, "reward/accuracy": 0.75, "reward/chosen": -2.0, "reward/margin": 1.1328125, "reward/rejected": -3.140625, "step": 3245 }, { "approx. KL/chosen": 460.0, "approx. KL/rejected": 1184.0, "epoch": 0.9736052789442111, "grad_norm": 4.200345039367676, "learning_rate": 1.7572519099791076e-08, "logp/chosen": -342.0, "logp/rejected": -454.0, "loss": 0.5676231384277344, "reward/accuracy": 0.75, "reward/chosen": -2.46875, "reward/margin": 1.5703125, "reward/rejected": -4.03125, "step": 3246 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 756.0, "epoch": 0.9739052189562087, "grad_norm": 3.298501968383789, "learning_rate": 1.7180074528651003e-08, "logp/chosen": -374.0, "logp/rejected": -382.0, "loss": 0.422332763671875, "reward/accuracy": 0.875, "reward/chosen": -1.953125, "reward/margin": 1.5625, "reward/rejected": -3.515625, "step": 3247 }, { "approx. KL/chosen": 340.0, "approx. KL/rejected": 1304.0, "epoch": 0.9742051589682064, "grad_norm": 3.3899037837982178, "learning_rate": 1.6792054248821467e-08, "logp/chosen": -360.0, "logp/rejected": -348.0, "loss": 0.3161773681640625, "reward/accuracy": 0.8125, "reward/chosen": -2.328125, "reward/margin": 2.25, "reward/rejected": -4.59375, "step": 3248 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 608.0, "epoch": 0.974505098980204, "grad_norm": 4.316955089569092, "learning_rate": 1.6408458604830203e-08, "logp/chosen": -344.0, "logp/rejected": -326.0, "loss": 0.5294189453125, "reward/accuracy": 0.625, "reward/chosen": -1.5234375, "reward/margin": 1.4296875, "reward/rejected": -2.953125, "step": 3249 }, { "approx. KL/chosen": 83.0, "approx. KL/rejected": 760.0, "epoch": 0.9748050389922016, "grad_norm": 3.217494487762451, "learning_rate": 1.6029287937274206e-08, "logp/chosen": -366.0, "logp/rejected": -454.0, "loss": 0.321502685546875, "reward/accuracy": 0.875, "reward/chosen": -1.0703125, "reward/margin": 2.34375, "reward/rejected": -3.421875, "step": 3250 }, { "approx. KL/chosen": 203.0, "approx. KL/rejected": 632.0, "epoch": 0.9751049790041991, "grad_norm": 3.940065860748291, "learning_rate": 1.5654542582823062e-08, "logp/chosen": -350.0, "logp/rejected": -328.0, "loss": 0.455596923828125, "reward/accuracy": 0.75, "reward/chosen": -1.796875, "reward/margin": 1.2734375, "reward/rejected": -3.078125, "step": 3251 }, { "approx. KL/chosen": 312.0, "approx. KL/rejected": 940.0, "epoch": 0.9754049190161967, "grad_norm": 3.883795976638794, "learning_rate": 1.528422287421727e-08, "logp/chosen": -262.0, "logp/rejected": -294.0, "loss": 0.4659423828125, "reward/accuracy": 0.8125, "reward/chosen": -2.15625, "reward/margin": 1.6796875, "reward/rejected": -3.828125, "step": 3252 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 1120.0, "epoch": 0.9757048590281944, "grad_norm": 4.337996006011963, "learning_rate": 1.491832914026603e-08, "logp/chosen": -424.0, "logp/rejected": -394.0, "loss": 0.4129638671875, "reward/accuracy": 0.75, "reward/chosen": -1.953125, "reward/margin": 2.1875, "reward/rejected": -4.125, "step": 3253 }, { "approx. KL/chosen": 149.0, "approx. KL/rejected": 1008.0, "epoch": 0.976004799040192, "grad_norm": 2.679914712905884, "learning_rate": 1.4556861705850023e-08, "logp/chosen": -332.0, "logp/rejected": -294.0, "loss": 0.3242340087890625, "reward/accuracy": 0.875, "reward/chosen": -1.4609375, "reward/margin": 2.234375, "reward/rejected": -3.703125, "step": 3254 }, { "approx. KL/chosen": 376.0, "approx. KL/rejected": 972.0, "epoch": 0.9763047390521895, "grad_norm": 3.748629570007324, "learning_rate": 1.4199820891919735e-08, "logp/chosen": -394.0, "logp/rejected": -296.0, "loss": 0.375, "reward/accuracy": 0.8125, "reward/chosen": -2.375, "reward/margin": 1.6953125, "reward/rejected": -4.0625, "step": 3255 }, { "approx. KL/chosen": 284.0, "approx. KL/rejected": 1032.0, "epoch": 0.9766046790641871, "grad_norm": 3.0304453372955322, "learning_rate": 1.3847207015494912e-08, "logp/chosen": -308.0, "logp/rejected": -320.0, "loss": 0.3674354553222656, "reward/accuracy": 0.75, "reward/chosen": -1.96875, "reward/margin": 1.890625, "reward/rejected": -3.859375, "step": 3256 }, { "approx. KL/chosen": 83.5, "approx. KL/rejected": 600.0, "epoch": 0.9769046190761848, "grad_norm": 3.4972825050354004, "learning_rate": 1.3499020389664552e-08, "logp/chosen": -312.0, "logp/rejected": -392.0, "loss": 0.3912239074707031, "reward/accuracy": 0.6875, "reward/chosen": -0.94921875, "reward/margin": 1.9453125, "reward/rejected": -2.90625, "step": 3257 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 856.0, "epoch": 0.9772045590881824, "grad_norm": 4.383887767791748, "learning_rate": 1.3155261323586355e-08, "logp/chosen": -372.0, "logp/rejected": -376.0, "loss": 0.60693359375, "reward/accuracy": 0.6875, "reward/chosen": -1.6171875, "reward/margin": 1.453125, "reward/rejected": -3.0625, "step": 3258 }, { "approx. KL/chosen": 149.0, "approx. KL/rejected": 776.0, "epoch": 0.97750449910018, "grad_norm": 2.179185152053833, "learning_rate": 1.281593012248783e-08, "logp/chosen": -378.0, "logp/rejected": -350.0, "loss": 0.2362060546875, "reward/accuracy": 0.9375, "reward/chosen": -0.96484375, "reward/margin": 2.296875, "reward/rejected": -3.265625, "step": 3259 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 988.0, "epoch": 0.9778044391121775, "grad_norm": 2.105471134185791, "learning_rate": 1.2481027087664077e-08, "logp/chosen": -362.0, "logp/rejected": -332.0, "loss": 0.20262527465820312, "reward/accuracy": 0.875, "reward/chosen": -1.3671875, "reward/margin": 2.671875, "reward/rejected": -4.03125, "step": 3260 }, { "approx. KL/chosen": 177.0, "approx. KL/rejected": 992.0, "epoch": 0.9781043791241751, "grad_norm": 3.0391273498535156, "learning_rate": 1.2150552516478341e-08, "logp/chosen": -326.0, "logp/rejected": -280.0, "loss": 0.3250732421875, "reward/accuracy": 0.8125, "reward/chosen": -1.2734375, "reward/margin": 2.359375, "reward/rejected": -3.640625, "step": 3261 }, { "approx. KL/chosen": 294.0, "approx. KL/rejected": 968.0, "epoch": 0.9784043191361728, "grad_norm": 4.648551940917969, "learning_rate": 1.1824506702362016e-08, "logp/chosen": -458.0, "logp/rejected": -436.0, "loss": 0.442657470703125, "reward/accuracy": 0.875, "reward/chosen": -1.6953125, "reward/margin": 2.296875, "reward/rejected": -4.0, "step": 3262 }, { "approx. KL/chosen": 216.0, "approx. KL/rejected": 470.0, "epoch": 0.9787042591481704, "grad_norm": 3.7996833324432373, "learning_rate": 1.1502889934814077e-08, "logp/chosen": -322.0, "logp/rejected": -296.0, "loss": 0.53851318359375, "reward/accuracy": 0.6875, "reward/chosen": -1.6484375, "reward/margin": 1.046875, "reward/rejected": -2.703125, "step": 3263 }, { "approx. KL/chosen": 358.0, "approx. KL/rejected": 1232.0, "epoch": 0.979004199160168, "grad_norm": 3.9517738819122314, "learning_rate": 1.1185702499401652e-08, "logp/chosen": -330.0, "logp/rejected": -368.0, "loss": 0.38250732421875, "reward/accuracy": 0.75, "reward/chosen": -2.265625, "reward/margin": 2.015625, "reward/rejected": -4.28125, "step": 3264 }, { "approx. KL/chosen": 146.0, "approx. KL/rejected": 528.0, "epoch": 0.9793041391721655, "grad_norm": 3.4495534896850586, "learning_rate": 1.0872944677757235e-08, "logp/chosen": -320.0, "logp/rejected": -292.0, "loss": 0.4671630859375, "reward/accuracy": 0.6875, "reward/chosen": -1.4921875, "reward/margin": 1.3046875, "reward/rejected": -2.796875, "step": 3265 }, { "approx. KL/chosen": 456.0, "approx. KL/rejected": 712.0, "epoch": 0.9796040791841631, "grad_norm": 7.709106922149658, "learning_rate": 1.056461674758258e-08, "logp/chosen": -292.0, "logp/rejected": -278.0, "loss": 0.84698486328125, "reward/accuracy": 0.6875, "reward/chosen": -1.921875, "reward/margin": 1.28125, "reward/rejected": -3.203125, "step": 3266 }, { "approx. KL/chosen": 123.0, "approx. KL/rejected": 1020.0, "epoch": 0.9799040191961608, "grad_norm": 2.3210084438323975, "learning_rate": 1.0260718982644247e-08, "logp/chosen": -356.0, "logp/rejected": -348.0, "loss": 0.18768310546875, "reward/accuracy": 0.9375, "reward/chosen": -1.375, "reward/margin": 2.625, "reward/rejected": -4.0, "step": 3267 }, { "approx. KL/chosen": 190.0, "approx. KL/rejected": 904.0, "epoch": 0.9802039592081584, "grad_norm": 2.905275344848633, "learning_rate": 9.961251652775283e-09, "logp/chosen": -288.0, "logp/rejected": -446.0, "loss": 0.30915069580078125, "reward/accuracy": 0.75, "reward/chosen": -1.6328125, "reward/margin": 2.203125, "reward/rejected": -3.84375, "step": 3268 }, { "approx. KL/chosen": 158.0, "approx. KL/rejected": 848.0, "epoch": 0.980503899220156, "grad_norm": 2.3697545528411865, "learning_rate": 9.666215023876325e-09, "logp/chosen": -292.0, "logp/rejected": -396.0, "loss": 0.2782135009765625, "reward/accuracy": 0.875, "reward/chosen": -1.484375, "reward/margin": 2.296875, "reward/rejected": -3.78125, "step": 3269 }, { "approx. KL/chosen": 201.0, "approx. KL/rejected": 864.0, "epoch": 0.9808038392321535, "grad_norm": 2.467602491378784, "learning_rate": 9.375609357912817e-09, "logp/chosen": -336.0, "logp/rejected": -342.0, "loss": 0.25030517578125, "reward/accuracy": 0.875, "reward/chosen": -1.3359375, "reward/margin": 2.34375, "reward/rejected": -3.6875, "step": 3270 }, { "approx. KL/chosen": 226.0, "approx. KL/rejected": 680.0, "epoch": 0.9811037792441512, "grad_norm": 6.036736965179443, "learning_rate": 9.08943491291614e-09, "logp/chosen": -444.0, "logp/rejected": -344.0, "loss": 0.568359375, "reward/accuracy": 0.75, "reward/chosen": -1.4375, "reward/margin": 1.546875, "reward/rejected": -2.984375, "step": 3271 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 1112.0, "epoch": 0.9814037192561488, "grad_norm": 3.9293477535247803, "learning_rate": 8.807691942983586e-09, "logp/chosen": -266.0, "logp/rejected": -282.0, "loss": 0.30838775634765625, "reward/accuracy": 0.875, "reward/chosen": -1.7421875, "reward/margin": 2.59375, "reward/rejected": -4.3125, "step": 3272 }, { "approx. KL/chosen": 282.0, "approx. KL/rejected": 688.0, "epoch": 0.9817036592681464, "grad_norm": 6.023789405822754, "learning_rate": 8.53038069827672e-09, "logp/chosen": -298.0, "logp/rejected": -368.0, "loss": 0.510162353515625, "reward/accuracy": 0.75, "reward/chosen": -1.8203125, "reward/margin": 1.5390625, "reward/rejected": -3.359375, "step": 3273 }, { "approx. KL/chosen": 422.0, "approx. KL/rejected": 696.0, "epoch": 0.982003599280144, "grad_norm": 3.683166265487671, "learning_rate": 8.257501425022463e-09, "logp/chosen": -292.0, "logp/rejected": -242.0, "loss": 0.40966796875, "reward/accuracy": 0.8125, "reward/chosen": -2.09375, "reward/margin": 1.1328125, "reward/rejected": -3.234375, "step": 3274 }, { "approx. KL/chosen": 178.0, "approx. KL/rejected": 1048.0, "epoch": 0.9823035392921415, "grad_norm": 2.7913918495178223, "learning_rate": 7.989054365514227e-09, "logp/chosen": -202.0, "logp/rejected": -280.0, "loss": 0.28357696533203125, "reward/accuracy": 0.8125, "reward/chosen": -1.3359375, "reward/margin": 2.5625, "reward/rejected": -3.90625, "step": 3275 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 824.0, "epoch": 0.9826034793041392, "grad_norm": 3.796628475189209, "learning_rate": 7.725039758106901e-09, "logp/chosen": -416.0, "logp/rejected": -312.0, "loss": 0.4364166259765625, "reward/accuracy": 0.75, "reward/chosen": -1.4609375, "reward/margin": 1.96875, "reward/rejected": -3.4375, "step": 3276 }, { "approx. KL/chosen": 189.0, "approx. KL/rejected": 732.0, "epoch": 0.9829034193161368, "grad_norm": 3.5914297103881836, "learning_rate": 7.465457837222967e-09, "logp/chosen": -304.0, "logp/rejected": -304.0, "loss": 0.2880859375, "reward/accuracy": 0.8125, "reward/chosen": -1.5390625, "reward/margin": 1.875, "reward/rejected": -3.421875, "step": 3277 }, { "approx. KL/chosen": 510.0, "approx. KL/rejected": 868.0, "epoch": 0.9832033593281344, "grad_norm": 5.74415397644043, "learning_rate": 7.2103088333463915e-09, "logp/chosen": -258.0, "logp/rejected": -264.0, "loss": 0.7864990234375, "reward/accuracy": 0.625, "reward/chosen": -2.640625, "reward/margin": 0.91015625, "reward/rejected": -3.5625, "step": 3278 }, { "approx. KL/chosen": 120.0, "approx. KL/rejected": 1056.0, "epoch": 0.9835032993401319, "grad_norm": 2.4807536602020264, "learning_rate": 6.959592973026508e-09, "logp/chosen": -476.0, "logp/rejected": -388.0, "loss": 0.21759033203125, "reward/accuracy": 0.875, "reward/chosen": -1.03125, "reward/margin": 2.625, "reward/rejected": -3.640625, "step": 3279 }, { "approx. KL/chosen": 168.0, "approx. KL/rejected": 904.0, "epoch": 0.9838032393521295, "grad_norm": 3.9507789611816406, "learning_rate": 6.713310478876911e-09, "logp/chosen": -448.0, "logp/rejected": -418.0, "loss": 0.44049072265625, "reward/accuracy": 0.875, "reward/chosen": -1.515625, "reward/margin": 1.9140625, "reward/rejected": -3.4375, "step": 3280 }, { "approx. KL/chosen": 378.0, "approx. KL/rejected": 704.0, "epoch": 0.9841031793641272, "grad_norm": 4.8831892013549805, "learning_rate": 6.4714615695737895e-09, "logp/chosen": -207.0, "logp/rejected": -350.0, "loss": 0.48211669921875, "reward/accuracy": 0.75, "reward/chosen": -2.078125, "reward/margin": 1.296875, "reward/rejected": -3.375, "step": 3281 }, { "approx. KL/chosen": 300.0, "approx. KL/rejected": 824.0, "epoch": 0.9844031193761248, "grad_norm": 3.0308339595794678, "learning_rate": 6.2340464598564796e-09, "logp/chosen": -326.0, "logp/rejected": -280.0, "loss": 0.3888206481933594, "reward/accuracy": 0.875, "reward/chosen": -1.6875, "reward/margin": 2.078125, "reward/rejected": -3.765625, "step": 3282 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 1020.0, "epoch": 0.9847030593881224, "grad_norm": 4.050581932067871, "learning_rate": 6.0010653605285794e-09, "logp/chosen": -284.0, "logp/rejected": -298.0, "loss": 0.492584228515625, "reward/accuracy": 0.6875, "reward/chosen": -1.9375, "reward/margin": 2.046875, "reward/rejected": -3.984375, "step": 3283 }, { "approx. KL/chosen": 191.0, "approx. KL/rejected": 836.0, "epoch": 0.9850029994001199, "grad_norm": 4.470670223236084, "learning_rate": 5.77251847845628e-09, "logp/chosen": -245.0, "logp/rejected": -330.0, "loss": 0.5341796875, "reward/accuracy": 0.5625, "reward/chosen": -1.671875, "reward/margin": 1.6015625, "reward/rejected": -3.265625, "step": 3284 }, { "approx. KL/chosen": 207.0, "approx. KL/rejected": 640.0, "epoch": 0.9853029394121176, "grad_norm": 5.014088153839111, "learning_rate": 5.548406016567809e-09, "logp/chosen": -380.0, "logp/rejected": -360.0, "loss": 0.58404541015625, "reward/accuracy": 0.6875, "reward/chosen": -1.734375, "reward/margin": 1.4296875, "reward/rejected": -3.15625, "step": 3285 }, { "approx. KL/chosen": 118.5, "approx. KL/rejected": 612.0, "epoch": 0.9856028794241152, "grad_norm": 3.799739360809326, "learning_rate": 5.3287281738551025e-09, "logp/chosen": -300.0, "logp/rejected": -300.0, "loss": 0.3896484375, "reward/accuracy": 0.8125, "reward/chosen": -1.21875, "reward/margin": 1.734375, "reward/rejected": -2.953125, "step": 3286 }, { "approx. KL/chosen": 308.0, "approx. KL/rejected": 908.0, "epoch": 0.9859028194361128, "grad_norm": 4.548401832580566, "learning_rate": 5.113485145372132e-09, "logp/chosen": -378.0, "logp/rejected": -402.0, "loss": 0.3430023193359375, "reward/accuracy": 0.8125, "reward/chosen": -1.7421875, "reward/margin": 2.078125, "reward/rejected": -3.8125, "step": 3287 }, { "approx. KL/chosen": 258.0, "approx. KL/rejected": 688.0, "epoch": 0.9862027594481104, "grad_norm": 5.124297142028809, "learning_rate": 4.902677122234912e-09, "logp/chosen": -298.0, "logp/rejected": -298.0, "loss": 0.481842041015625, "reward/accuracy": 0.6875, "reward/chosen": -1.6953125, "reward/margin": 1.65625, "reward/rejected": -3.359375, "step": 3288 }, { "approx. KL/chosen": 243.0, "approx. KL/rejected": 1004.0, "epoch": 0.9865026994601079, "grad_norm": 3.7452104091644287, "learning_rate": 4.69630429162149e-09, "logp/chosen": -426.0, "logp/rejected": -462.0, "loss": 0.4433574676513672, "reward/accuracy": 0.75, "reward/chosen": -1.921875, "reward/margin": 1.921875, "reward/rejected": -3.84375, "step": 3289 }, { "approx. KL/chosen": 106.5, "approx. KL/rejected": 796.0, "epoch": 0.9868026394721056, "grad_norm": 2.994262933731079, "learning_rate": 4.4943668367730676e-09, "logp/chosen": -346.0, "logp/rejected": -364.0, "loss": 0.271759033203125, "reward/accuracy": 0.8125, "reward/chosen": -0.98828125, "reward/margin": 2.421875, "reward/rejected": -3.421875, "step": 3290 }, { "approx. KL/chosen": 328.0, "approx. KL/rejected": 612.0, "epoch": 0.9871025794841032, "grad_norm": 4.715109348297119, "learning_rate": 4.296864936991219e-09, "logp/chosen": -366.0, "logp/rejected": -392.0, "loss": 0.5860595703125, "reward/accuracy": 0.6875, "reward/chosen": -2.140625, "reward/margin": 1.15625, "reward/rejected": -3.296875, "step": 3291 }, { "approx. KL/chosen": 370.0, "approx. KL/rejected": 732.0, "epoch": 0.9874025194961008, "grad_norm": 5.280590534210205, "learning_rate": 4.103798767640111e-09, "logp/chosen": -258.0, "logp/rejected": -400.0, "loss": 0.7666015625, "reward/accuracy": 0.5625, "reward/chosen": -2.453125, "reward/margin": 0.69921875, "reward/rejected": -3.140625, "step": 3292 }, { "approx. KL/chosen": 264.0, "approx. KL/rejected": 1004.0, "epoch": 0.9877024595080984, "grad_norm": 2.638679265975952, "learning_rate": 3.915168500144284e-09, "logp/chosen": -176.0, "logp/rejected": -314.0, "loss": 0.30792236328125, "reward/accuracy": 0.9375, "reward/chosen": -1.9453125, "reward/margin": 2.03125, "reward/rejected": -3.96875, "step": 3293 }, { "approx. KL/chosen": 246.0, "approx. KL/rejected": 792.0, "epoch": 0.9880023995200959, "grad_norm": 2.541393995285034, "learning_rate": 3.73097430199032e-09, "logp/chosen": -270.0, "logp/rejected": -328.0, "loss": 0.2103271484375, "reward/accuracy": 1.0, "reward/chosen": -1.71875, "reward/margin": 1.953125, "reward/rejected": -3.671875, "step": 3294 }, { "approx. KL/chosen": 342.0, "approx. KL/rejected": 768.0, "epoch": 0.9883023395320936, "grad_norm": 5.425806522369385, "learning_rate": 3.551216336726837e-09, "logp/chosen": -302.0, "logp/rejected": -288.0, "loss": 0.96368408203125, "reward/accuracy": 0.6875, "reward/chosen": -2.09375, "reward/margin": 0.90234375, "reward/rejected": -3.0, "step": 3295 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 840.0, "epoch": 0.9886022795440912, "grad_norm": 4.3975419998168945, "learning_rate": 3.375894763961718e-09, "logp/chosen": -326.0, "logp/rejected": -298.0, "loss": 0.68817138671875, "reward/accuracy": 0.625, "reward/chosen": -1.2109375, "reward/margin": 2.09375, "reward/rejected": -3.3125, "step": 3296 }, { "approx. KL/chosen": 173.0, "approx. KL/rejected": 420.0, "epoch": 0.9889022195560888, "grad_norm": 4.717715740203857, "learning_rate": 3.2050097393648837e-09, "logp/chosen": -388.0, "logp/rejected": -332.0, "loss": 0.5892333984375, "reward/accuracy": 0.6875, "reward/chosen": -1.5234375, "reward/margin": 0.99609375, "reward/rejected": -2.515625, "step": 3297 }, { "approx. KL/chosen": 223.0, "approx. KL/rejected": 972.0, "epoch": 0.9892021595680864, "grad_norm": 4.227132320404053, "learning_rate": 3.0385614146666298e-09, "logp/chosen": -398.0, "logp/rejected": -356.0, "loss": 0.329833984375, "reward/accuracy": 0.8125, "reward/chosen": -1.4453125, "reward/margin": 2.53125, "reward/rejected": -3.96875, "step": 3298 }, { "approx. KL/chosen": 227.0, "approx. KL/rejected": 856.0, "epoch": 0.989502099580084, "grad_norm": 4.698919773101807, "learning_rate": 2.8765499376581794e-09, "logp/chosen": -434.0, "logp/rejected": -432.0, "loss": 0.35846710205078125, "reward/accuracy": 0.875, "reward/chosen": -1.609375, "reward/margin": 2.078125, "reward/rejected": -3.6875, "step": 3299 }, { "approx. KL/chosen": 340.0, "approx. KL/rejected": 1160.0, "epoch": 0.9898020395920816, "grad_norm": 4.956527233123779, "learning_rate": 2.7189754521905755e-09, "logp/chosen": -390.0, "logp/rejected": -472.0, "loss": 0.704559326171875, "reward/accuracy": 0.5625, "reward/chosen": -1.8671875, "reward/margin": 1.578125, "reward/rejected": -3.4375, "step": 3300 }, { "approx. KL/chosen": 356.0, "approx. KL/rejected": 724.0, "epoch": 0.9901019796040792, "grad_norm": 4.584836959838867, "learning_rate": 2.565838098175788e-09, "logp/chosen": -372.0, "logp/rejected": -326.0, "loss": 0.557861328125, "reward/accuracy": 0.5625, "reward/chosen": -2.171875, "reward/margin": 0.9140625, "reward/rejected": -3.078125, "step": 3301 }, { "approx. KL/chosen": 358.0, "approx. KL/rejected": 828.0, "epoch": 0.9904019196160768, "grad_norm": 4.565250873565674, "learning_rate": 2.417138011585607e-09, "logp/chosen": -264.0, "logp/rejected": -288.0, "loss": 0.5892333984375, "reward/accuracy": 0.625, "reward/chosen": -2.171875, "reward/margin": 1.2265625, "reward/rejected": -3.40625, "step": 3302 }, { "approx. KL/chosen": 214.0, "approx. KL/rejected": 668.0, "epoch": 0.9907018596280743, "grad_norm": 2.5210537910461426, "learning_rate": 2.2728753244533054e-09, "logp/chosen": -402.0, "logp/rejected": -386.0, "loss": 0.2923583984375, "reward/accuracy": 0.9375, "reward/chosen": -1.609375, "reward/margin": 1.5703125, "reward/rejected": -3.1875, "step": 3303 }, { "approx. KL/chosen": 156.0, "approx. KL/rejected": 936.0, "epoch": 0.991001799640072, "grad_norm": 2.7304019927978516, "learning_rate": 2.1330501648697542e-09, "logp/chosen": -374.0, "logp/rejected": -372.0, "loss": 0.2289276123046875, "reward/accuracy": 0.875, "reward/chosen": -1.1875, "reward/margin": 2.46875, "reward/rejected": -3.65625, "step": 3304 }, { "approx. KL/chosen": 163.0, "approx. KL/rejected": 1376.0, "epoch": 0.9913017396520696, "grad_norm": 3.7827959060668945, "learning_rate": 1.997662656987864e-09, "logp/chosen": -245.0, "logp/rejected": -356.0, "loss": 0.2944984436035156, "reward/accuracy": 0.75, "reward/chosen": -1.234375, "reward/margin": 3.0, "reward/rejected": -4.21875, "step": 3305 }, { "approx. KL/chosen": 286.0, "approx. KL/rejected": 856.0, "epoch": 0.9916016796640672, "grad_norm": 2.7353639602661133, "learning_rate": 1.8667129210186984e-09, "logp/chosen": -350.0, "logp/rejected": -456.0, "loss": 0.267578125, "reward/accuracy": 0.875, "reward/chosen": -1.765625, "reward/margin": 1.7734375, "reward/rejected": -3.53125, "step": 3306 }, { "approx. KL/chosen": 480.0, "approx. KL/rejected": 744.0, "epoch": 0.9919016196760648, "grad_norm": 6.139124393463135, "learning_rate": 1.740201073234249e-09, "logp/chosen": -340.0, "logp/rejected": -312.0, "loss": 0.84197998046875, "reward/accuracy": 0.5, "reward/chosen": -2.25, "reward/margin": 1.09375, "reward/rejected": -3.328125, "step": 3307 }, { "approx. KL/chosen": 268.0, "approx. KL/rejected": 520.0, "epoch": 0.9922015596880623, "grad_norm": 6.091853618621826, "learning_rate": 1.6181272259652158e-09, "logp/chosen": -278.0, "logp/rejected": -358.0, "loss": 0.7745361328125, "reward/accuracy": 0.625, "reward/chosen": -1.7421875, "reward/margin": 1.078125, "reward/rejected": -2.828125, "step": 3308 }, { "approx. KL/chosen": 332.0, "approx. KL/rejected": 636.0, "epoch": 0.99250149970006, "grad_norm": 6.359967231750488, "learning_rate": 1.5004914876026733e-09, "logp/chosen": -344.0, "logp/rejected": -334.0, "loss": 0.585205078125, "reward/accuracy": 0.75, "reward/chosen": -1.9296875, "reward/margin": 1.1328125, "reward/rejected": -3.0625, "step": 3309 }, { "approx. KL/chosen": 276.0, "approx. KL/rejected": 1080.0, "epoch": 0.9928014397120576, "grad_norm": 2.701035737991333, "learning_rate": 1.3872939625958482e-09, "logp/chosen": -356.0, "logp/rejected": -516.0, "loss": 0.28643798828125, "reward/accuracy": 0.9375, "reward/chosen": -1.671875, "reward/margin": 2.125, "reward/rejected": -3.796875, "step": 3310 }, { "approx. KL/chosen": 218.0, "approx. KL/rejected": 848.0, "epoch": 0.9931013797240552, "grad_norm": 2.8467953205108643, "learning_rate": 1.278534751454341e-09, "logp/chosen": -264.0, "logp/rejected": -247.0, "loss": 0.3323974609375, "reward/accuracy": 0.875, "reward/chosen": -1.640625, "reward/margin": 2.125, "reward/rejected": -3.765625, "step": 3311 }, { "approx. KL/chosen": 354.0, "approx. KL/rejected": 636.0, "epoch": 0.9934013197360528, "grad_norm": 6.692195892333984, "learning_rate": 1.174213950745351e-09, "logp/chosen": -292.0, "logp/rejected": -352.0, "loss": 1.0792236328125, "reward/accuracy": 0.5, "reward/chosen": -2.109375, "reward/margin": 0.412109375, "reward/rejected": -2.53125, "step": 3312 }, { "approx. KL/chosen": 262.0, "approx. KL/rejected": 680.0, "epoch": 0.9937012597480503, "grad_norm": 3.659738540649414, "learning_rate": 1.074331653098115e-09, "logp/chosen": -231.0, "logp/rejected": -264.0, "loss": 0.40338134765625, "reward/accuracy": 0.8125, "reward/chosen": -1.71875, "reward/margin": 1.703125, "reward/rejected": -3.421875, "step": 3313 }, { "approx. KL/chosen": 234.0, "approx. KL/rejected": 1024.0, "epoch": 0.994001199760048, "grad_norm": 2.430600166320801, "learning_rate": 9.78887947197249e-10, "logp/chosen": -484.0, "logp/rejected": -438.0, "loss": 0.1724853515625, "reward/accuracy": 1.0, "reward/chosen": -1.6484375, "reward/margin": 2.5, "reward/rejected": -4.15625, "step": 3314 }, { "approx. KL/chosen": 230.0, "approx. KL/rejected": 540.0, "epoch": 0.9943011397720456, "grad_norm": 4.527289867401123, "learning_rate": 8.878829177894066e-10, "logp/chosen": -282.0, "logp/rejected": -356.0, "loss": 0.580810546875, "reward/accuracy": 0.625, "reward/chosen": -2.0, "reward/margin": 0.9375, "reward/rejected": -2.9375, "step": 3315 }, { "approx. KL/chosen": 238.0, "approx. KL/rejected": 596.0, "epoch": 0.9946010797840432, "grad_norm": 3.4183430671691895, "learning_rate": 8.013166456777299e-10, "logp/chosen": -344.0, "logp/rejected": -262.0, "loss": 0.4580078125, "reward/accuracy": 0.6875, "reward/chosen": -1.765625, "reward/margin": 1.1953125, "reward/rejected": -2.96875, "step": 3316 }, { "approx. KL/chosen": 148.0, "approx. KL/rejected": 1112.0, "epoch": 0.9949010197960408, "grad_norm": 3.164811611175537, "learning_rate": 7.191892077262896e-10, "logp/chosen": -434.0, "logp/rejected": -454.0, "loss": 0.3751678466796875, "reward/accuracy": 0.875, "reward/chosen": -1.28125, "reward/margin": 2.359375, "reward/rejected": -3.640625, "step": 3317 }, { "approx. KL/chosen": 176.0, "approx. KL/rejected": 1608.0, "epoch": 0.9952009598080384, "grad_norm": 1.182651400566101, "learning_rate": 6.415006768556442e-10, "logp/chosen": -282.0, "logp/rejected": -420.0, "loss": 0.08211612701416016, "reward/accuracy": 1.0, "reward/chosen": -1.4296875, "reward/margin": 3.90625, "reward/rejected": -5.34375, "step": 3318 }, { "approx. KL/chosen": 169.0, "approx. KL/rejected": 1016.0, "epoch": 0.995500899820036, "grad_norm": 4.038011074066162, "learning_rate": 5.682511220472808e-10, "logp/chosen": -324.0, "logp/rejected": -390.0, "loss": 0.44194793701171875, "reward/accuracy": 0.75, "reward/chosen": -1.328125, "reward/margin": 2.28125, "reward/rejected": -3.609375, "step": 3319 }, { "approx. KL/chosen": 320.0, "approx. KL/rejected": 1000.0, "epoch": 0.9958008398320336, "grad_norm": 2.813540458679199, "learning_rate": 4.994406083391745e-10, "logp/chosen": -326.0, "logp/rejected": -342.0, "loss": 0.25537109375, "reward/accuracy": 1.0, "reward/chosen": -1.7578125, "reward/margin": 2.21875, "reward/rejected": -3.96875, "step": 3320 }, { "approx. KL/chosen": 86.0, "approx. KL/rejected": 1104.0, "epoch": 0.9961007798440312, "grad_norm": 3.0387332439422607, "learning_rate": 4.3506919682911875e-10, "logp/chosen": -342.0, "logp/rejected": -352.0, "loss": 0.3124980926513672, "reward/accuracy": 0.8125, "reward/chosen": -0.8984375, "reward/margin": 3.09375, "reward/rejected": -3.984375, "step": 3321 }, { "approx. KL/chosen": 292.0, "approx. KL/rejected": 652.0, "epoch": 0.9964007198560288, "grad_norm": 6.170619964599609, "learning_rate": 3.7513694467361526e-10, "logp/chosen": -252.0, "logp/rejected": -318.0, "loss": 0.5742034912109375, "reward/accuracy": 0.75, "reward/chosen": -1.5625, "reward/margin": 1.453125, "reward/rejected": -3.015625, "step": 3322 }, { "approx. KL/chosen": 332.0, "approx. KL/rejected": 816.0, "epoch": 0.9967006598680264, "grad_norm": 5.144645690917969, "learning_rate": 3.196439050862088e-10, "logp/chosen": -400.0, "logp/rejected": -350.0, "loss": 0.61590576171875, "reward/accuracy": 0.75, "reward/chosen": -1.953125, "reward/margin": 1.578125, "reward/rejected": -3.53125, "step": 3323 }, { "approx. KL/chosen": 384.0, "approx. KL/rejected": 964.0, "epoch": 0.997000599880024, "grad_norm": 4.565888404846191, "learning_rate": 2.685901273402625e-10, "logp/chosen": -436.0, "logp/rejected": -470.0, "loss": 0.590087890625, "reward/accuracy": 0.6875, "reward/chosen": -2.3125, "reward/margin": 1.2734375, "reward/rejected": -3.578125, "step": 3324 }, { "approx. KL/chosen": 220.0, "approx. KL/rejected": 956.0, "epoch": 0.9973005398920216, "grad_norm": 3.587700605392456, "learning_rate": 2.2197565676673748e-10, "logp/chosen": -384.0, "logp/rejected": -378.0, "loss": 0.45135498046875, "reward/accuracy": 0.75, "reward/chosen": -1.7109375, "reward/margin": 2.09375, "reward/rejected": -3.796875, "step": 3325 }, { "approx. KL/chosen": 194.0, "approx. KL/rejected": 760.0, "epoch": 0.9976004799040192, "grad_norm": 3.2838079929351807, "learning_rate": 1.7980053475474823e-10, "logp/chosen": -228.0, "logp/rejected": -268.0, "loss": 0.4345245361328125, "reward/accuracy": 0.75, "reward/chosen": -1.46875, "reward/margin": 1.8359375, "reward/rejected": -3.296875, "step": 3326 }, { "approx. KL/chosen": 208.0, "approx. KL/rejected": 728.0, "epoch": 0.9979004199160167, "grad_norm": 2.902029514312744, "learning_rate": 1.4206479875267244e-10, "logp/chosen": -334.0, "logp/rejected": -386.0, "loss": 0.243682861328125, "reward/accuracy": 0.9375, "reward/chosen": -1.59375, "reward/margin": 1.953125, "reward/rejected": -3.546875, "step": 3327 }, { "approx. KL/chosen": 164.0, "approx. KL/rejected": 888.0, "epoch": 0.9982003599280144, "grad_norm": 3.6194114685058594, "learning_rate": 1.0876848226537562e-10, "logp/chosen": -474.0, "logp/rejected": -404.0, "loss": 0.357666015625, "reward/accuracy": 0.8125, "reward/chosen": -1.4453125, "reward/margin": 1.9921875, "reward/rejected": -3.4375, "step": 3328 }, { "approx. KL/chosen": 226.0, "approx. KL/rejected": 944.0, "epoch": 0.998500299940012, "grad_norm": 4.404518127441406, "learning_rate": 7.991161485809695e-11, "logp/chosen": -360.0, "logp/rejected": -292.0, "loss": 0.40380859375, "reward/accuracy": 0.75, "reward/chosen": -1.734375, "reward/margin": 1.8125, "reward/rejected": -3.546875, "step": 3329 }, { "approx. KL/chosen": 98.0, "approx. KL/rejected": 684.0, "epoch": 0.9988002399520096, "grad_norm": 2.6953561305999756, "learning_rate": 5.549422215256339e-11, "logp/chosen": -324.0, "logp/rejected": -348.0, "loss": 0.39337158203125, "reward/accuracy": 0.75, "reward/chosen": -1.1875, "reward/margin": 1.84375, "reward/rejected": -3.03125, "step": 3330 }, { "approx. KL/chosen": 225.0, "approx. KL/rejected": 462.0, "epoch": 0.9991001799640072, "grad_norm": 5.008880138397217, "learning_rate": 3.551632582921016e-11, "logp/chosen": -356.0, "logp/rejected": -306.0, "loss": 0.76513671875, "reward/accuracy": 0.5, "reward/chosen": -1.671875, "reward/margin": 0.7109375, "reward/rejected": -2.375, "step": 3331 }, { "approx. KL/chosen": 344.0, "approx. KL/rejected": 1304.0, "epoch": 0.9994001199760048, "grad_norm": 3.0373878479003906, "learning_rate": 1.997794362662564e-11, "logp/chosen": -328.0, "logp/rejected": -406.0, "loss": 0.277984619140625, "reward/accuracy": 0.875, "reward/chosen": -1.828125, "reward/margin": 2.5625, "reward/rejected": -4.40625, "step": 3332 }, { "approx. KL/chosen": 235.0, "approx. KL/rejected": 652.0, "epoch": 0.9997000599880024, "grad_norm": 3.8260269165039062, "learning_rate": 8.879089340996238e-12, "logp/chosen": -284.0, "logp/rejected": -290.0, "loss": 0.5714111328125, "reward/accuracy": 0.875, "reward/chosen": -1.875, "reward/margin": 1.3046875, "reward/rejected": -3.1875, "step": 3333 }, { "approx. KL/chosen": 244.0, "approx. KL/rejected": 1264.0, "epoch": 1.0, "grad_norm": 5.060642242431641, "learning_rate": 2.219772827771749e-12, "logp/chosen": -270.0, "logp/rejected": -354.0, "loss": 0.2887153625488281, "reward/accuracy": 0.875, "reward/chosen": -1.6875, "reward/margin": 3.078125, "reward/rejected": -4.75, "step": 3334 }, { "epoch": 1.0, "eval_approx. KL/chosen": 258.7083333333333, "eval_approx. KL/rejected": 804.2222222222222, "eval_logp/chosen": -342.27777777777777, "eval_logp/rejected": -348.2037037037037, "eval_loss": 0.4545944929122925, "eval_reward/accuracy": 0.7818287037037037, "eval_reward/chosen": -1.7077546296296295, "eval_reward/margin": 1.6935402199074074, "eval_reward/rejected": -3.3994502314814814, "eval_runtime": 23.0802, "eval_samples_per_second": 74.306, "eval_steps_per_second": 4.679, "step": 3334 } ], "logging_steps": 1.0, "max_steps": 3334, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 0, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 8.246347381688238e+17, "train_batch_size": 2, "trial_name": null, "trial_params": null }