| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.9870967741935484, |
| "eval_steps": 500, |
| "global_step": 308, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.03225806451612903, |
| "grad_norm": 11.747716903686523, |
| "learning_rate": 2.2086148233646147e-06, |
| "logits/chosen": -6.872015476226807, |
| "logits/rejected": -8.557083129882812, |
| "logps/chosen": -200.8333282470703, |
| "logps/rejected": -126.05131530761719, |
| "loss": 3.7785, |
| "rewards/accuracies": 0.5296875238418579, |
| "rewards/chosen": 0.6622785925865173, |
| "rewards/margins": 0.7673968076705933, |
| "rewards/rejected": -0.10511825233697891, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.06451612903225806, |
| "grad_norm": 11.248347282409668, |
| "learning_rate": 4.9693833525703835e-06, |
| "logits/chosen": -6.8620195388793945, |
| "logits/rejected": -8.925396919250488, |
| "logps/chosen": -194.3622283935547, |
| "logps/rejected": -129.8681182861328, |
| "loss": 3.8044, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.45987948775291443, |
| "rewards/margins": 0.4599760174751282, |
| "rewards/rejected": -9.66489315032959e-05, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.0967741935483871, |
| "grad_norm": 13.071877479553223, |
| "learning_rate": 7.730151881776152e-06, |
| "logits/chosen": -6.489249229431152, |
| "logits/rejected": -8.789779663085938, |
| "logps/chosen": -202.84713745117188, |
| "logps/rejected": -129.98233032226562, |
| "loss": 3.9664, |
| "rewards/accuracies": 0.526562511920929, |
| "rewards/chosen": 0.5739307999610901, |
| "rewards/margins": 0.5547459721565247, |
| "rewards/rejected": 0.019184818491339684, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.12903225806451613, |
| "grad_norm": 8.609030723571777, |
| "learning_rate": 1.049092041098192e-05, |
| "logits/chosen": -6.255059719085693, |
| "logits/rejected": -9.302863121032715, |
| "logps/chosen": -176.4971466064453, |
| "logps/rejected": -134.4936981201172, |
| "loss": 2.6269, |
| "rewards/accuracies": 0.628125011920929, |
| "rewards/chosen": 1.0335595607757568, |
| "rewards/margins": 2.3927571773529053, |
| "rewards/rejected": -1.3591976165771484, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.16129032258064516, |
| "grad_norm": 7.000743389129639, |
| "learning_rate": 1.325168894018769e-05, |
| "logits/chosen": -5.956324100494385, |
| "logits/rejected": -9.702302932739258, |
| "logps/chosen": -169.1029052734375, |
| "logps/rejected": -165.97474670410156, |
| "loss": 1.6851, |
| "rewards/accuracies": 0.8109375238418579, |
| "rewards/chosen": 2.0764987468719482, |
| "rewards/margins": 5.65482234954834, |
| "rewards/rejected": -3.5783238410949707, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.1935483870967742, |
| "grad_norm": 2.4143693447113037, |
| "learning_rate": 1.601245746939346e-05, |
| "logits/chosen": -6.569132328033447, |
| "logits/rejected": -10.802966117858887, |
| "logps/chosen": -172.4666290283203, |
| "logps/rejected": -244.1662139892578, |
| "loss": 0.3164, |
| "rewards/accuracies": 0.932812511920929, |
| "rewards/chosen": 2.330841541290283, |
| "rewards/margins": 13.70948600769043, |
| "rewards/rejected": -11.378644943237305, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.22580645161290322, |
| "grad_norm": 0.06904491782188416, |
| "learning_rate": 1.8773225998599227e-05, |
| "logits/chosen": -8.068090438842773, |
| "logits/rejected": -11.996877670288086, |
| "logps/chosen": -184.2109832763672, |
| "logps/rejected": -375.8692932128906, |
| "loss": 0.0338, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": 2.172847032546997, |
| "rewards/margins": 27.02886962890625, |
| "rewards/rejected": -24.85602569580078, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.25806451612903225, |
| "grad_norm": 0.1487932652235031, |
| "learning_rate": 1.9317814719367415e-05, |
| "logits/chosen": -8.399168014526367, |
| "logits/rejected": -12.002328872680664, |
| "logps/chosen": -205.44052124023438, |
| "logps/rejected": -487.5409240722656, |
| "loss": 0.0144, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": -0.532281219959259, |
| "rewards/margins": 35.829750061035156, |
| "rewards/rejected": -36.362030029296875, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.2903225806451613, |
| "grad_norm": 0.6289846301078796, |
| "learning_rate": 1.9287109033995323e-05, |
| "logits/chosen": -8.708343505859375, |
| "logits/rejected": -11.307577133178711, |
| "logps/chosen": -217.4675750732422, |
| "logps/rejected": -556.1575317382812, |
| "loss": 0.0232, |
| "rewards/accuracies": 0.995312511920929, |
| "rewards/chosen": -2.1770095825195312, |
| "rewards/margins": 40.92938995361328, |
| "rewards/rejected": -43.10639190673828, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.3225806451612903, |
| "grad_norm": 1.6672333478927612, |
| "learning_rate": 1.923288991518115e-05, |
| "logits/chosen": -9.274807929992676, |
| "logits/rejected": -11.517184257507324, |
| "logps/chosen": -216.9584503173828, |
| "logps/rejected": -567.5177612304688, |
| "loss": 0.0117, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5761454105377197, |
| "rewards/margins": 41.563804626464844, |
| "rewards/rejected": -44.13995361328125, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.3548387096774194, |
| "grad_norm": 0.1163749024271965, |
| "learning_rate": 1.915533421442089e-05, |
| "logits/chosen": -8.658121109008789, |
| "logits/rejected": -10.576135635375977, |
| "logps/chosen": -204.00003051757812, |
| "logps/rejected": -574.2200317382812, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": -1.2570573091506958, |
| "rewards/margins": 43.9434700012207, |
| "rewards/rejected": -45.200523376464844, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.3870967741935484, |
| "grad_norm": 0.0025108063127845526, |
| "learning_rate": 1.9054694902290306e-05, |
| "logits/chosen": -7.479229927062988, |
| "logits/rejected": -8.57958698272705, |
| "logps/chosen": -195.77511596679688, |
| "logps/rejected": -588.6607666015625, |
| "loss": 0.0084, |
| "rewards/accuracies": 0.996874988079071, |
| "rewards/chosen": -0.20098623633384705, |
| "rewards/margins": 46.062320709228516, |
| "rewards/rejected": -46.263309478759766, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.41935483870967744, |
| "grad_norm": 0.05390551686286926, |
| "learning_rate": 1.893130024330745e-05, |
| "logits/chosen": -5.855751991271973, |
| "logits/rejected": -6.480991363525391, |
| "logps/chosen": -181.93038940429688, |
| "logps/rejected": -579.6819458007812, |
| "loss": 0.0106, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 1.1638758182525635, |
| "rewards/margins": 46.36396408081055, |
| "rewards/rejected": -45.20009231567383, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.45161290322580644, |
| "grad_norm": 0.008025813847780228, |
| "learning_rate": 1.8785552725202064e-05, |
| "logits/chosen": -4.496771335601807, |
| "logits/rejected": -4.330361843109131, |
| "logps/chosen": -177.8521728515625, |
| "logps/rejected": -569.95166015625, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5159854888916016, |
| "rewards/margins": 46.066001892089844, |
| "rewards/rejected": -44.550018310546875, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.4838709677419355, |
| "grad_norm": 0.2902831435203552, |
| "learning_rate": 1.8617927746084266e-05, |
| "logits/chosen": -3.704448699951172, |
| "logits/rejected": -3.2316722869873047, |
| "logps/chosen": -173.41717529296875, |
| "logps/rejected": -592.05908203125, |
| "loss": 0.0017, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.7170642614364624, |
| "rewards/margins": 47.78697967529297, |
| "rewards/rejected": -46.069915771484375, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.5161290322580645, |
| "grad_norm": 1.5575469732284546, |
| "learning_rate": 1.8428972063794692e-05, |
| "logits/chosen": -3.1886916160583496, |
| "logits/rejected": -2.6457602977752686, |
| "logps/chosen": -166.82505798339844, |
| "logps/rejected": -598.5891723632812, |
| "loss": 0.0504, |
| "rewards/accuracies": 0.995312511920929, |
| "rewards/chosen": 1.4597384929656982, |
| "rewards/margins": 48.52386474609375, |
| "rewards/rejected": -47.064125061035156, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.5483870967741935, |
| "grad_norm": 6.104778731241822e-05, |
| "learning_rate": 1.8219302012494108e-05, |
| "logits/chosen": -3.119097948074341, |
| "logits/rejected": -2.784574508666992, |
| "logps/chosen": -176.2117462158203, |
| "logps/rejected": -601.7147827148438, |
| "loss": 0.0034, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 1.8994001150131226, |
| "rewards/margins": 49.24987030029297, |
| "rewards/rejected": -47.35047149658203, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.5806451612903226, |
| "grad_norm": 0.021236000582575798, |
| "learning_rate": 1.7989601492309465e-05, |
| "logits/chosen": -2.805873394012451, |
| "logits/rejected": -2.618786334991455, |
| "logps/chosen": -170.4734649658203, |
| "logps/rejected": -609.2696533203125, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.078625202178955, |
| "rewards/margins": 50.50914764404297, |
| "rewards/rejected": -48.43052673339844, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.6129032258064516, |
| "grad_norm": 0.006399027071893215, |
| "learning_rate": 1.774061973859382e-05, |
| "logits/chosen": -2.691558361053467, |
| "logits/rejected": -2.4888315200805664, |
| "logps/chosen": -177.30880737304688, |
| "logps/rejected": -640.2526245117188, |
| "loss": 0.0002, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.453687906265259, |
| "rewards/margins": 53.2391471862793, |
| "rewards/rejected": -50.78545379638672, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.6451612903225806, |
| "grad_norm": 0.000686797546222806, |
| "learning_rate": 1.747316887807638e-05, |
| "logits/chosen": -2.461819887161255, |
| "logits/rejected": -2.338575601577759, |
| "logps/chosen": -170.4715118408203, |
| "logps/rejected": -635.7724609375, |
| "loss": 0.0015, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.5472875833511353, |
| "rewards/margins": 52.81101608276367, |
| "rewards/rejected": -51.26373291015625, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.6774193548387096, |
| "grad_norm": 0.0314163975417614, |
| "learning_rate": 1.718812127987388e-05, |
| "logits/chosen": -2.4023430347442627, |
| "logits/rejected": -2.294923782348633, |
| "logps/chosen": -169.93972778320312, |
| "logps/rejected": -621.2903442382812, |
| "loss": 0.0081, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 1.7308235168457031, |
| "rewards/margins": 51.537498474121094, |
| "rewards/rejected": -49.806678771972656, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.7096774193548387, |
| "grad_norm": 1.8648837804794312, |
| "learning_rate": 1.688640671000386e-05, |
| "logits/chosen": -2.5111083984375, |
| "logits/rejected": -2.4942150115966797, |
| "logps/chosen": -186.0556182861328, |
| "logps/rejected": -643.71826171875, |
| "loss": 0.0384, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 1.8740406036376953, |
| "rewards/margins": 53.93976974487305, |
| "rewards/rejected": -52.06572341918945, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.7419354838709677, |
| "grad_norm": 0.2866516709327698, |
| "learning_rate": 1.656900929868119e-05, |
| "logits/chosen": -2.851760149002075, |
| "logits/rejected": -2.680408477783203, |
| "logps/chosen": -180.36245727539062, |
| "logps/rejected": -650.8028564453125, |
| "loss": 0.0048, |
| "rewards/accuracies": 0.996874988079071, |
| "rewards/chosen": 1.7712488174438477, |
| "rewards/margins": 54.01469039916992, |
| "rewards/rejected": -52.243446350097656, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.7741935483870968, |
| "grad_norm": 0.00485634570941329, |
| "learning_rate": 1.6236964330289824e-05, |
| "logits/chosen": -3.038351535797119, |
| "logits/rejected": -2.873201847076416, |
| "logps/chosen": -176.7779541015625, |
| "logps/rejected": -638.7794799804688, |
| "loss": 0.0015, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 2.315247058868408, |
| "rewards/margins": 53.3264045715332, |
| "rewards/rejected": -51.01116180419922, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.8064516129032258, |
| "grad_norm": 0.00011157651897519827, |
| "learning_rate": 1.5891354866500355e-05, |
| "logits/chosen": -3.0750269889831543, |
| "logits/rejected": -2.8404810428619385, |
| "logps/chosen": -166.00732421875, |
| "logps/rejected": -626.678466796875, |
| "loss": 0.001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8882861137390137, |
| "rewards/margins": 53.35065841674805, |
| "rewards/rejected": -50.46237564086914, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.8387096774193549, |
| "grad_norm": 0.037098996341228485, |
| "learning_rate": 1.5533308213547954e-05, |
| "logits/chosen": -2.93206787109375, |
| "logits/rejected": -2.821136474609375, |
| "logps/chosen": -163.5557098388672, |
| "logps/rejected": -600.4147338867188, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8543925285339355, |
| "rewards/margins": 50.14447784423828, |
| "rewards/rejected": -47.29008865356445, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.8709677419354839, |
| "grad_norm": 6.515044515253976e-05, |
| "learning_rate": 1.5163992245193735e-05, |
| "logits/chosen": -2.9613232612609863, |
| "logits/rejected": -2.806081771850586, |
| "logps/chosen": -167.47032165527344, |
| "logps/rejected": -627.2801513671875, |
| "loss": 0.0021, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 2.517486095428467, |
| "rewards/margins": 52.39359664916992, |
| "rewards/rejected": -49.87610626220703, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.9032258064516129, |
| "grad_norm": 7.156098581617698e-05, |
| "learning_rate": 1.4784611593363314e-05, |
| "logits/chosen": -2.8768656253814697, |
| "logits/rejected": -2.8222408294677734, |
| "logps/chosen": -163.4990234375, |
| "logps/rejected": -613.9205932617188, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8345298767089844, |
| "rewards/margins": 51.45341110229492, |
| "rewards/rejected": -48.6188850402832, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.9354838709677419, |
| "grad_norm": 1.8226394331577467e-06, |
| "learning_rate": 1.439640371888783e-05, |
| "logits/chosen": -2.739189624786377, |
| "logits/rejected": -2.763517379760742, |
| "logps/chosen": -162.78518676757812, |
| "logps/rejected": -597.4986572265625, |
| "loss": 0.0027, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 3.1442718505859375, |
| "rewards/margins": 50.352046966552734, |
| "rewards/rejected": -47.2077751159668, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.967741935483871, |
| "grad_norm": 6.970116373850033e-06, |
| "learning_rate": 1.4000634875163929e-05, |
| "logits/chosen": -2.6427102088928223, |
| "logits/rejected": -2.703989028930664, |
| "logps/chosen": -173.3591766357422, |
| "logps/rejected": -622.0789794921875, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.122755527496338, |
| "rewards/margins": 51.858497619628906, |
| "rewards/rejected": -49.73574447631836, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.9935483870967742, |
| "eval_logits/chosen": -2.4111573696136475, |
| "eval_logits/rejected": -2.533914566040039, |
| "eval_logps/chosen": -121.9600601196289, |
| "eval_logps/rejected": -688.9767456054688, |
| "eval_loss": 0.0001295939728152007, |
| "eval_rewards/accuracies": 1.0, |
| "eval_rewards/chosen": -0.3132941722869873, |
| "eval_rewards/margins": 61.79844284057617, |
| "eval_rewards/rejected": -62.11174011230469, |
| "eval_runtime": 13.3156, |
| "eval_samples_per_second": 15.02, |
| "eval_steps_per_second": 7.51, |
| "step": 154 |
| }, |
| { |
| "epoch": 1.0, |
| "grad_norm": 0.1435479074716568, |
| "learning_rate": 1.3598595977898393e-05, |
| "logits/chosen": -2.470811128616333, |
| "logits/rejected": -2.687915086746216, |
| "logps/chosen": -176.60227966308594, |
| "logps/rejected": -602.3994750976562, |
| "loss": 0.003, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 1.7994766235351562, |
| "rewards/margins": 50.05354309082031, |
| "rewards/rejected": -48.254066467285156, |
| "step": 155 |
| }, |
| { |
| "epoch": 1.032258064516129, |
| "grad_norm": 4.26612663269043, |
| "learning_rate": 1.3191598394409396e-05, |
| "logits/chosen": -2.4601125717163086, |
| "logits/rejected": -2.6306862831115723, |
| "logps/chosen": -182.5096893310547, |
| "logps/rejected": -619.1905517578125, |
| "loss": 0.0084, |
| "rewards/accuracies": 0.996874988079071, |
| "rewards/chosen": 1.3070443868637085, |
| "rewards/margins": 50.604530334472656, |
| "rewards/rejected": -49.2974853515625, |
| "step": 160 |
| }, |
| { |
| "epoch": 1.064516129032258, |
| "grad_norm": 0.07364771515130997, |
| "learning_rate": 1.2780969666218915e-05, |
| "logits/chosen": -2.371699094772339, |
| "logits/rejected": -2.5179049968719482, |
| "logps/chosen": -171.65237426757812, |
| "logps/rejected": -626.7579345703125, |
| "loss": 0.0002, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.8574281930923462, |
| "rewards/margins": 52.002235412597656, |
| "rewards/rejected": -50.144805908203125, |
| "step": 165 |
| }, |
| { |
| "epoch": 1.096774193548387, |
| "grad_norm": 2.0316293239593506, |
| "learning_rate": 1.2368049178888188e-05, |
| "logits/chosen": -2.3688201904296875, |
| "logits/rejected": -2.4845595359802246, |
| "logps/chosen": -167.29345703125, |
| "logps/rejected": -637.6082763671875, |
| "loss": 0.0059, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 2.473865032196045, |
| "rewards/margins": 53.60892868041992, |
| "rewards/rejected": -51.13506317138672, |
| "step": 170 |
| }, |
| { |
| "epoch": 1.129032258064516, |
| "grad_norm": 5.519531896425178e-06, |
| "learning_rate": 1.1954183793220426e-05, |
| "logits/chosen": -2.4739153385162354, |
| "logits/rejected": -2.5887179374694824, |
| "logps/chosen": -168.55252075195312, |
| "logps/rejected": -629.6751098632812, |
| "loss": 0.0003, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.837996006011963, |
| "rewards/margins": 54.0001106262207, |
| "rewards/rejected": -50.16211700439453, |
| "step": 175 |
| }, |
| { |
| "epoch": 1.1612903225806452, |
| "grad_norm": 4.96319489684538e-06, |
| "learning_rate": 1.154072345208082e-05, |
| "logits/chosen": -2.675598382949829, |
| "logits/rejected": -2.770794153213501, |
| "logps/chosen": -167.05870056152344, |
| "logps/rejected": -619.7980346679688, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.837059736251831, |
| "rewards/margins": 52.89777755737305, |
| "rewards/rejected": -49.06071853637695, |
| "step": 180 |
| }, |
| { |
| "epoch": 1.1935483870967742, |
| "grad_norm": 2.3110576876206324e-05, |
| "learning_rate": 1.1129016777163497e-05, |
| "logits/chosen": -2.7444872856140137, |
| "logits/rejected": -2.823488235473633, |
| "logps/chosen": -158.0244140625, |
| "logps/rejected": -612.7073974609375, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 3.5461955070495605, |
| "rewards/margins": 52.555580139160156, |
| "rewards/rejected": -49.00938034057617, |
| "step": 185 |
| }, |
| { |
| "epoch": 1.2258064516129032, |
| "grad_norm": 0.00015435948444064707, |
| "learning_rate": 1.0720406670067866e-05, |
| "logits/chosen": -3.0422451496124268, |
| "logits/rejected": -3.078159809112549, |
| "logps/chosen": -162.6065216064453, |
| "logps/rejected": -648.2110595703125, |
| "loss": 0.009, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 2.963984966278076, |
| "rewards/margins": 54.6415901184082, |
| "rewards/rejected": -51.6776123046875, |
| "step": 190 |
| }, |
| { |
| "epoch": 1.2580645161290323, |
| "grad_norm": 0.23506908118724823, |
| "learning_rate": 1.0316225932032739e-05, |
| "logits/chosen": -3.1033904552459717, |
| "logits/rejected": -3.1072916984558105, |
| "logps/chosen": -162.05966186523438, |
| "logps/rejected": -627.5680541992188, |
| "loss": 0.0017, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 3.406363010406494, |
| "rewards/margins": 53.60102462768555, |
| "rewards/rejected": -50.194664001464844, |
| "step": 195 |
| }, |
| { |
| "epoch": 1.2903225806451613, |
| "grad_norm": 0.0005963341682218015, |
| "learning_rate": 9.917792916615701e-06, |
| "logits/chosen": -3.057863473892212, |
| "logits/rejected": -3.170236349105835, |
| "logps/chosen": -172.58238220214844, |
| "logps/rejected": -642.7207641601562, |
| "loss": 0.0015, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 3.198392629623413, |
| "rewards/margins": 54.85893630981445, |
| "rewards/rejected": -51.66054153442383, |
| "step": 200 |
| }, |
| { |
| "epoch": 1.3225806451612903, |
| "grad_norm": 3.264209590270184e-05, |
| "learning_rate": 9.526407229497827e-06, |
| "logits/chosen": -3.2984588146209717, |
| "logits/rejected": -3.360607624053955, |
| "logps/chosen": -163.1930694580078, |
| "logps/rejected": -648.0612182617188, |
| "loss": 0.0016, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.657193660736084, |
| "rewards/margins": 54.8690071105957, |
| "rewards/rejected": -52.211814880371094, |
| "step": 205 |
| }, |
| { |
| "epoch": 1.3548387096774195, |
| "grad_norm": 4.916101943308604e-07, |
| "learning_rate": 9.143345489440102e-06, |
| "logits/chosen": -3.411468982696533, |
| "logits/rejected": -3.4234490394592285, |
| "logps/chosen": -162.50698852539062, |
| "logps/rejected": -648.9544677734375, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.4228503704071045, |
| "rewards/margins": 56.02128982543945, |
| "rewards/rejected": -52.59843826293945, |
| "step": 210 |
| }, |
| { |
| "epoch": 1.3870967741935485, |
| "grad_norm": 3.84959868426904e-08, |
| "learning_rate": 8.769857164218336e-06, |
| "logits/chosen": -3.3674731254577637, |
| "logits/rejected": -3.465867519378662, |
| "logps/chosen": -156.56558227539062, |
| "logps/rejected": -662.9464721679688, |
| "loss": 0.014, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 3.516038179397583, |
| "rewards/margins": 56.7149772644043, |
| "rewards/rejected": -53.198936462402344, |
| "step": 215 |
| }, |
| { |
| "epoch": 1.4193548387096775, |
| "grad_norm": 1.2234114365128335e-05, |
| "learning_rate": 8.407160495119008e-06, |
| "logits/chosen": -3.2030277252197266, |
| "logits/rejected": -3.3925929069519043, |
| "logps/chosen": -170.909423828125, |
| "logps/rejected": -667.1768798828125, |
| "loss": 0.0004, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.376603364944458, |
| "rewards/margins": 57.6746826171875, |
| "rewards/rejected": -54.2980842590332, |
| "step": 220 |
| }, |
| { |
| "epoch": 1.4516129032258065, |
| "grad_norm": 0.19691216945648193, |
| "learning_rate": 8.056438523289393e-06, |
| "logits/chosen": -3.271845579147339, |
| "logits/rejected": -3.299424409866333, |
| "logps/chosen": -165.4629364013672, |
| "logps/rejected": -647.8621826171875, |
| "loss": 0.0047, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 2.843123435974121, |
| "rewards/margins": 54.9563102722168, |
| "rewards/rejected": -52.113182067871094, |
| "step": 225 |
| }, |
| { |
| "epoch": 1.4838709677419355, |
| "grad_norm": 0.0007676176610402763, |
| "learning_rate": 7.718835230903237e-06, |
| "logits/chosen": -3.305957794189453, |
| "logits/rejected": -3.3377411365509033, |
| "logps/chosen": -168.66317749023438, |
| "logps/rejected": -664.5413818359375, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.4339606761932373, |
| "rewards/margins": 57.25107955932617, |
| "rewards/rejected": -53.81711959838867, |
| "step": 230 |
| }, |
| { |
| "epoch": 1.5161290322580645, |
| "grad_norm": 1.1214526239200495e-05, |
| "learning_rate": 7.3954518097287005e-06, |
| "logits/chosen": -3.360619068145752, |
| "logits/rejected": -3.381943464279175, |
| "logps/chosen": -170.27052307128906, |
| "logps/rejected": -697.7355346679688, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.744898557662964, |
| "rewards/margins": 60.68017578125, |
| "rewards/rejected": -56.935279846191406, |
| "step": 235 |
| }, |
| { |
| "epoch": 1.5483870967741935, |
| "grad_norm": 0.001958629349246621, |
| "learning_rate": 7.08734306926967e-06, |
| "logits/chosen": -3.3265819549560547, |
| "logits/rejected": -3.33760404586792, |
| "logps/chosen": -158.65939331054688, |
| "logps/rejected": -662.5846557617188, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8199775218963623, |
| "rewards/margins": 56.54740524291992, |
| "rewards/rejected": -53.7274284362793, |
| "step": 240 |
| }, |
| { |
| "epoch": 1.5806451612903225, |
| "grad_norm": 1.1762933354475535e-05, |
| "learning_rate": 6.7955139961964235e-06, |
| "logits/chosen": -3.417844295501709, |
| "logits/rejected": -3.355306625366211, |
| "logps/chosen": -163.48977661132812, |
| "logps/rejected": -671.9425659179688, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.6044082641601562, |
| "rewards/margins": 57.15119171142578, |
| "rewards/rejected": -54.546783447265625, |
| "step": 245 |
| }, |
| { |
| "epoch": 1.6129032258064515, |
| "grad_norm": 0.3113669157028198, |
| "learning_rate": 6.5209164762879876e-06, |
| "logits/chosen": -3.1726534366607666, |
| "logits/rejected": -3.2277159690856934, |
| "logps/chosen": -158.69281005859375, |
| "logps/rejected": -670.3931884765625, |
| "loss": 0.018, |
| "rewards/accuracies": 0.995312511920929, |
| "rewards/chosen": 2.9490036964416504, |
| "rewards/margins": 57.56342315673828, |
| "rewards/rejected": -54.61442184448242, |
| "step": 250 |
| }, |
| { |
| "epoch": 1.6451612903225805, |
| "grad_norm": 0.15534324944019318, |
| "learning_rate": 6.26444618957859e-06, |
| "logits/chosen": -2.9065983295440674, |
| "logits/rejected": -3.0085651874542236, |
| "logps/chosen": -162.54873657226562, |
| "logps/rejected": -677.47412109375, |
| "loss": 0.0006, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.7529945373535156, |
| "rewards/margins": 58.81446075439453, |
| "rewards/rejected": -55.06147003173828, |
| "step": 255 |
| }, |
| { |
| "epoch": 1.6774193548387095, |
| "grad_norm": 0.5288254022598267, |
| "learning_rate": 6.026939688835542e-06, |
| "logits/chosen": -2.7987606525421143, |
| "logits/rejected": -2.935157299041748, |
| "logps/chosen": -156.1405792236328, |
| "logps/rejected": -637.0443115234375, |
| "loss": 0.0059, |
| "rewards/accuracies": 0.996874988079071, |
| "rewards/chosen": 2.6880862712860107, |
| "rewards/margins": 53.83855056762695, |
| "rewards/rejected": -51.15046691894531, |
| "step": 260 |
| }, |
| { |
| "epoch": 1.7096774193548387, |
| "grad_norm": 9.21001483220607e-05, |
| "learning_rate": 5.809171670898002e-06, |
| "logits/chosen": -2.7774410247802734, |
| "logits/rejected": -2.9501843452453613, |
| "logps/chosen": -161.6907501220703, |
| "logps/rejected": -681.8929443359375, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7917118072509766, |
| "rewards/margins": 58.00335693359375, |
| "rewards/rejected": -55.211647033691406, |
| "step": 265 |
| }, |
| { |
| "epoch": 1.7419354838709677, |
| "grad_norm": 0.0019305861787870526, |
| "learning_rate": 5.611852449776906e-06, |
| "logits/chosen": -2.70743989944458, |
| "logits/rejected": -2.9343791007995605, |
| "logps/chosen": -158.28378295898438, |
| "logps/rejected": -656.1748046875, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 2.8768951892852783, |
| "rewards/margins": 56.165191650390625, |
| "rewards/rejected": -53.288291931152344, |
| "step": 270 |
| }, |
| { |
| "epoch": 1.7741935483870968, |
| "grad_norm": 0.05368852615356445, |
| "learning_rate": 5.435625639758302e-06, |
| "logits/chosen": -2.6554648876190186, |
| "logits/rejected": -2.9207005500793457, |
| "logps/chosen": -173.9909210205078, |
| "logps/rejected": -673.0507202148438, |
| "loss": 0.0003, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8360095024108887, |
| "rewards/margins": 57.830604553222656, |
| "rewards/rejected": -54.994598388671875, |
| "step": 275 |
| }, |
| { |
| "epoch": 1.8064516129032258, |
| "grad_norm": 1.7408574422006495e-06, |
| "learning_rate": 5.281066056067326e-06, |
| "logits/chosen": -2.7118172645568848, |
| "logits/rejected": -2.919018268585205, |
| "logps/chosen": -169.70877075195312, |
| "logps/rejected": -689.2927856445312, |
| "loss": 0.008, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 2.9414725303649902, |
| "rewards/margins": 58.81359100341797, |
| "rewards/rejected": -55.87211990356445, |
| "step": 280 |
| }, |
| { |
| "epoch": 1.838709677419355, |
| "grad_norm": 0.02011033147573471, |
| "learning_rate": 5.148677839940424e-06, |
| "logits/chosen": -2.6920559406280518, |
| "logits/rejected": -2.856995105743408, |
| "logps/chosen": -170.58425903320312, |
| "logps/rejected": -718.7882690429688, |
| "loss": 0.0033, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 2.741574764251709, |
| "rewards/margins": 61.97066116333008, |
| "rewards/rejected": -59.22908401489258, |
| "step": 285 |
| }, |
| { |
| "epoch": 1.870967741935484, |
| "grad_norm": 0.07748307287693024, |
| "learning_rate": 5.038892814221403e-06, |
| "logits/chosen": -2.6701674461364746, |
| "logits/rejected": -2.828078031539917, |
| "logps/chosen": -166.5321807861328, |
| "logps/rejected": -721.7613525390625, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.030878782272339, |
| "rewards/margins": 61.87688446044922, |
| "rewards/rejected": -59.84600830078125, |
| "step": 290 |
| }, |
| { |
| "epoch": 1.903225806451613, |
| "grad_norm": 4.640487168217078e-05, |
| "learning_rate": 4.952069074845096e-06, |
| "logits/chosen": -2.712212324142456, |
| "logits/rejected": -2.777492046356201, |
| "logps/chosen": -163.62423706054688, |
| "logps/rejected": -726.83251953125, |
| "loss": 0.0021, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 2.233910083770752, |
| "rewards/margins": 62.41532516479492, |
| "rewards/rejected": -60.18141555786133, |
| "step": 295 |
| }, |
| { |
| "epoch": 1.935483870967742, |
| "grad_norm": 0.00010279910929966718, |
| "learning_rate": 4.888489822802835e-06, |
| "logits/chosen": -2.6591506004333496, |
| "logits/rejected": -2.798229694366455, |
| "logps/chosen": -177.29238891601562, |
| "logps/rejected": -769.4779663085938, |
| "loss": 0.0141, |
| "rewards/accuracies": 0.9984375238418579, |
| "rewards/chosen": 1.5061697959899902, |
| "rewards/margins": 65.76886749267578, |
| "rewards/rejected": -64.2626953125, |
| "step": 300 |
| }, |
| { |
| "epoch": 1.967741935483871, |
| "grad_norm": 1.4639048458775505e-05, |
| "learning_rate": 4.848362440399692e-06, |
| "logits/chosen": -2.73205828666687, |
| "logits/rejected": -2.761648178100586, |
| "logps/chosen": -175.31532287597656, |
| "logps/rejected": -765.2847900390625, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 1.8591419458389282, |
| "rewards/margins": 65.3056869506836, |
| "rewards/rejected": -63.4465446472168, |
| "step": 305 |
| }, |
| { |
| "epoch": 1.9870967741935484, |
| "eval_logits/chosen": -2.4073026180267334, |
| "eval_logits/rejected": -2.258913516998291, |
| "eval_logps/chosen": -124.21800994873047, |
| "eval_logps/rejected": -902.7763061523438, |
| "eval_loss": 9.752377081895247e-06, |
| "eval_rewards/accuracies": 1.0, |
| "eval_rewards/chosen": -0.5390886664390564, |
| "eval_rewards/margins": 82.95260620117188, |
| "eval_rewards/rejected": -83.49169921875, |
| "eval_runtime": 11.9364, |
| "eval_samples_per_second": 16.755, |
| "eval_steps_per_second": 8.378, |
| "step": 308 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 310, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 2, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 64, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
