{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.9870967741935484,
  "eval_steps": 500,
  "global_step": 308,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.03225806451612903,
      "grad_norm": 11.747716903686523,
      "learning_rate": 2.2086148233646147e-06,
      "logits/chosen": -6.872015476226807,
      "logits/rejected": -8.557083129882812,
      "logps/chosen": -200.8333282470703,
      "logps/rejected": -126.05131530761719,
      "loss": 3.7785,
      "rewards/accuracies": 0.5296875238418579,
      "rewards/chosen": 0.6622785925865173,
      "rewards/margins": 0.7673968076705933,
      "rewards/rejected": -0.10511825233697891,
      "step": 5
    },
    {
      "epoch": 0.06451612903225806,
      "grad_norm": 11.248347282409668,
      "learning_rate": 4.9693833525703835e-06,
      "logits/chosen": -6.8620195388793945,
      "logits/rejected": -8.925396919250488,
      "logps/chosen": -194.3622283935547,
      "logps/rejected": -129.8681182861328,
      "loss": 3.8044,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.45987948775291443,
      "rewards/margins": 0.4599760174751282,
      "rewards/rejected": -9.66489315032959e-05,
      "step": 10
    },
    {
      "epoch": 0.0967741935483871,
      "grad_norm": 13.071877479553223,
      "learning_rate": 7.730151881776152e-06,
      "logits/chosen": -6.489249229431152,
      "logits/rejected": -8.789779663085938,
      "logps/chosen": -202.84713745117188,
      "logps/rejected": -129.98233032226562,
      "loss": 3.9664,
      "rewards/accuracies": 0.526562511920929,
      "rewards/chosen": 0.5739307999610901,
      "rewards/margins": 0.5547459721565247,
      "rewards/rejected": 0.019184818491339684,
      "step": 15
    },
    {
      "epoch": 0.12903225806451613,
      "grad_norm": 8.609030723571777,
      "learning_rate": 1.049092041098192e-05,
      "logits/chosen": -6.255059719085693,
      "logits/rejected": -9.302863121032715,
      "logps/chosen": -176.4971466064453,
      "logps/rejected": -134.4936981201172,
      "loss": 2.6269,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": 1.0335595607757568,
      "rewards/margins": 2.3927571773529053,
      "rewards/rejected": -1.3591976165771484,
      "step": 20
    },
    {
      "epoch": 0.16129032258064516,
      "grad_norm": 7.000743389129639,
      "learning_rate": 1.325168894018769e-05,
      "logits/chosen": -5.956324100494385,
      "logits/rejected": -9.702302932739258,
      "logps/chosen": -169.1029052734375,
      "logps/rejected": -165.97474670410156,
      "loss": 1.6851,
      "rewards/accuracies": 0.8109375238418579,
      "rewards/chosen": 2.0764987468719482,
      "rewards/margins": 5.65482234954834,
      "rewards/rejected": -3.5783238410949707,
      "step": 25
    },
    {
      "epoch": 0.1935483870967742,
      "grad_norm": 2.4143693447113037,
      "learning_rate": 1.601245746939346e-05,
      "logits/chosen": -6.569132328033447,
      "logits/rejected": -10.802966117858887,
      "logps/chosen": -172.4666290283203,
      "logps/rejected": -244.1662139892578,
      "loss": 0.3164,
      "rewards/accuracies": 0.932812511920929,
      "rewards/chosen": 2.330841541290283,
      "rewards/margins": 13.70948600769043,
      "rewards/rejected": -11.378644943237305,
      "step": 30
    },
    {
      "epoch": 0.22580645161290322,
      "grad_norm": 0.06904491782188416,
      "learning_rate": 1.8773225998599227e-05,
      "logits/chosen": -8.068090438842773,
      "logits/rejected": -11.996877670288086,
      "logps/chosen": -184.2109832763672,
      "logps/rejected": -375.8692932128906,
      "loss": 0.0338,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": 2.172847032546997,
      "rewards/margins": 27.02886962890625,
      "rewards/rejected": -24.85602569580078,
      "step": 35
    },
    {
      "epoch": 0.25806451612903225,
      "grad_norm": 0.1487932652235031,
      "learning_rate": 1.9317814719367415e-05,
      "logits/chosen": -8.399168014526367,
      "logits/rejected": -12.002328872680664,
      "logps/chosen": -205.44052124023438,
      "logps/rejected": -487.5409240722656,
      "loss": 0.0144,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": -0.532281219959259,
      "rewards/margins": 35.829750061035156,
      "rewards/rejected": -36.362030029296875,
      "step": 40
    },
    {
      "epoch": 0.2903225806451613,
      "grad_norm": 0.6289846301078796,
      "learning_rate": 1.9287109033995323e-05,
      "logits/chosen": -8.708343505859375,
      "logits/rejected": -11.307577133178711,
      "logps/chosen": -217.4675750732422,
      "logps/rejected": -556.1575317382812,
      "loss": 0.0232,
      "rewards/accuracies": 0.995312511920929,
      "rewards/chosen": -2.1770095825195312,
      "rewards/margins": 40.92938995361328,
      "rewards/rejected": -43.10639190673828,
      "step": 45
    },
    {
      "epoch": 0.3225806451612903,
      "grad_norm": 1.6672333478927612,
      "learning_rate": 1.923288991518115e-05,
      "logits/chosen": -9.274807929992676,
      "logits/rejected": -11.517184257507324,
      "logps/chosen": -216.9584503173828,
      "logps/rejected": -567.5177612304688,
      "loss": 0.0117,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5761454105377197,
      "rewards/margins": 41.563804626464844,
      "rewards/rejected": -44.13995361328125,
      "step": 50
    },
    {
      "epoch": 0.3548387096774194,
      "grad_norm": 0.1163749024271965,
      "learning_rate": 1.915533421442089e-05,
      "logits/chosen": -8.658121109008789,
      "logits/rejected": -10.576135635375977,
      "logps/chosen": -204.00003051757812,
      "logps/rejected": -574.2200317382812,
      "loss": 0.0045,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": -1.2570573091506958,
      "rewards/margins": 43.9434700012207,
      "rewards/rejected": -45.200523376464844,
      "step": 55
    },
    {
      "epoch": 0.3870967741935484,
      "grad_norm": 0.0025108063127845526,
      "learning_rate": 1.9054694902290306e-05,
      "logits/chosen": -7.479229927062988,
      "logits/rejected": -8.57958698272705,
      "logps/chosen": -195.77511596679688,
      "logps/rejected": -588.6607666015625,
      "loss": 0.0084,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": -0.20098623633384705,
      "rewards/margins": 46.062320709228516,
      "rewards/rejected": -46.263309478759766,
      "step": 60
    },
    {
      "epoch": 0.41935483870967744,
      "grad_norm": 0.05390551686286926,
      "learning_rate": 1.893130024330745e-05,
      "logits/chosen": -5.855751991271973,
      "logits/rejected": -6.480991363525391,
      "logps/chosen": -181.93038940429688,
      "logps/rejected": -579.6819458007812,
      "loss": 0.0106,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 1.1638758182525635,
      "rewards/margins": 46.36396408081055,
      "rewards/rejected": -45.20009231567383,
      "step": 65
    },
    {
      "epoch": 0.45161290322580644,
      "grad_norm": 0.008025813847780228,
      "learning_rate": 1.8785552725202064e-05,
      "logits/chosen": -4.496771335601807,
      "logits/rejected": -4.330361843109131,
      "logps/chosen": -177.8521728515625,
      "logps/rejected": -569.95166015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5159854888916016,
      "rewards/margins": 46.066001892089844,
      "rewards/rejected": -44.550018310546875,
      "step": 70
    },
    {
      "epoch": 0.4838709677419355,
      "grad_norm": 0.2902831435203552,
      "learning_rate": 1.8617927746084266e-05,
      "logits/chosen": -3.704448699951172,
      "logits/rejected": -3.2316722869873047,
      "logps/chosen": -173.41717529296875,
      "logps/rejected": -592.05908203125,
      "loss": 0.0017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7170642614364624,
      "rewards/margins": 47.78697967529297,
      "rewards/rejected": -46.069915771484375,
      "step": 75
    },
    {
      "epoch": 0.5161290322580645,
      "grad_norm": 1.5575469732284546,
      "learning_rate": 1.8428972063794692e-05,
      "logits/chosen": -3.1886916160583496,
      "logits/rejected": -2.6457602977752686,
      "logps/chosen": -166.82505798339844,
      "logps/rejected": -598.5891723632812,
      "loss": 0.0504,
      "rewards/accuracies": 0.995312511920929,
      "rewards/chosen": 1.4597384929656982,
      "rewards/margins": 48.52386474609375,
      "rewards/rejected": -47.064125061035156,
      "step": 80
    },
    {
      "epoch": 0.5483870967741935,
      "grad_norm": 6.104778731241822e-05,
      "learning_rate": 1.8219302012494108e-05,
      "logits/chosen": -3.119097948074341,
      "logits/rejected": -2.784574508666992,
      "logps/chosen": -176.2117462158203,
      "logps/rejected": -601.7147827148438,
      "loss": 0.0034,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 1.8994001150131226,
      "rewards/margins": 49.24987030029297,
      "rewards/rejected": -47.35047149658203,
      "step": 85
    },
    {
      "epoch": 0.5806451612903226,
      "grad_norm": 0.021236000582575798,
      "learning_rate": 1.7989601492309465e-05,
      "logits/chosen": -2.805873394012451,
      "logits/rejected": -2.618786334991455,
      "logps/chosen": -170.4734649658203,
      "logps/rejected": -609.2696533203125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.078625202178955,
      "rewards/margins": 50.50914764404297,
      "rewards/rejected": -48.43052673339844,
      "step": 90
    },
    {
      "epoch": 0.6129032258064516,
      "grad_norm": 0.006399027071893215,
      "learning_rate": 1.774061973859382e-05,
      "logits/chosen": -2.691558361053467,
      "logits/rejected": -2.4888315200805664,
      "logps/chosen": -177.30880737304688,
      "logps/rejected": -640.2526245117188,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.453687906265259,
      "rewards/margins": 53.2391471862793,
      "rewards/rejected": -50.78545379638672,
      "step": 95
    },
    {
      "epoch": 0.6451612903225806,
      "grad_norm": 0.000686797546222806,
      "learning_rate": 1.747316887807638e-05,
      "logits/chosen": -2.461819887161255,
      "logits/rejected": -2.338575601577759,
      "logps/chosen": -170.4715118408203,
      "logps/rejected": -635.7724609375,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5472875833511353,
      "rewards/margins": 52.81101608276367,
      "rewards/rejected": -51.26373291015625,
      "step": 100
    },
    {
      "epoch": 0.6774193548387096,
      "grad_norm": 0.0314163975417614,
      "learning_rate": 1.718812127987388e-05,
      "logits/chosen": -2.4023430347442627,
      "logits/rejected": -2.294923782348633,
      "logps/chosen": -169.93972778320312,
      "logps/rejected": -621.2903442382812,
      "loss": 0.0081,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 1.7308235168457031,
      "rewards/margins": 51.537498474121094,
      "rewards/rejected": -49.806678771972656,
      "step": 105
    },
    {
      "epoch": 0.7096774193548387,
      "grad_norm": 1.8648837804794312,
      "learning_rate": 1.688640671000386e-05,
      "logits/chosen": -2.5111083984375,
      "logits/rejected": -2.4942150115966797,
      "logps/chosen": -186.0556182861328,
      "logps/rejected": -643.71826171875,
      "loss": 0.0384,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 1.8740406036376953,
      "rewards/margins": 53.93976974487305,
      "rewards/rejected": -52.06572341918945,
      "step": 110
    },
    {
      "epoch": 0.7419354838709677,
      "grad_norm": 0.2866516709327698,
      "learning_rate": 1.656900929868119e-05,
      "logits/chosen": -2.851760149002075,
      "logits/rejected": -2.680408477783203,
      "logps/chosen": -180.36245727539062,
      "logps/rejected": -650.8028564453125,
      "loss": 0.0048,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": 1.7712488174438477,
      "rewards/margins": 54.01469039916992,
      "rewards/rejected": -52.243446350097656,
      "step": 115
    },
    {
      "epoch": 0.7741935483870968,
      "grad_norm": 0.00485634570941329,
      "learning_rate": 1.6236964330289824e-05,
      "logits/chosen": -3.038351535797119,
      "logits/rejected": -2.873201847076416,
      "logps/chosen": -176.7779541015625,
      "logps/rejected": -638.7794799804688,
      "loss": 0.0015,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 2.315247058868408,
      "rewards/margins": 53.3264045715332,
      "rewards/rejected": -51.01116180419922,
      "step": 120
    },
    {
      "epoch": 0.8064516129032258,
      "grad_norm": 0.00011157651897519827,
      "learning_rate": 1.5891354866500355e-05,
      "logits/chosen": -3.0750269889831543,
      "logits/rejected": -2.8404810428619385,
      "logps/chosen": -166.00732421875,
      "logps/rejected": -626.678466796875,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8882861137390137,
      "rewards/margins": 53.35065841674805,
      "rewards/rejected": -50.46237564086914,
      "step": 125
    },
    {
      "epoch": 0.8387096774193549,
      "grad_norm": 0.037098996341228485,
      "learning_rate": 1.5533308213547954e-05,
      "logits/chosen": -2.93206787109375,
      "logits/rejected": -2.821136474609375,
      "logps/chosen": -163.5557098388672,
      "logps/rejected": -600.4147338867188,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8543925285339355,
      "rewards/margins": 50.14447784423828,
      "rewards/rejected": -47.29008865356445,
      "step": 130
    },
    {
      "epoch": 0.8709677419354839,
      "grad_norm": 6.515044515253976e-05,
      "learning_rate": 1.5163992245193735e-05,
      "logits/chosen": -2.9613232612609863,
      "logits/rejected": -2.806081771850586,
      "logps/chosen": -167.47032165527344,
      "logps/rejected": -627.2801513671875,
      "loss": 0.0021,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 2.517486095428467,
      "rewards/margins": 52.39359664916992,
      "rewards/rejected": -49.87610626220703,
      "step": 135
    },
    {
      "epoch": 0.9032258064516129,
      "grad_norm": 7.156098581617698e-05,
      "learning_rate": 1.4784611593363314e-05,
      "logits/chosen": -2.8768656253814697,
      "logits/rejected": -2.8222408294677734,
      "logps/chosen": -163.4990234375,
      "logps/rejected": -613.9205932617188,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8345298767089844,
      "rewards/margins": 51.45341110229492,
      "rewards/rejected": -48.6188850402832,
      "step": 140
    },
    {
      "epoch": 0.9354838709677419,
      "grad_norm": 1.8226394331577467e-06,
      "learning_rate": 1.439640371888783e-05,
      "logits/chosen": -2.739189624786377,
      "logits/rejected": -2.763517379760742,
      "logps/chosen": -162.78518676757812,
      "logps/rejected": -597.4986572265625,
      "loss": 0.0027,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 3.1442718505859375,
      "rewards/margins": 50.352046966552734,
      "rewards/rejected": -47.2077751159668,
      "step": 145
    },
    {
      "epoch": 0.967741935483871,
      "grad_norm": 6.970116373850033e-06,
      "learning_rate": 1.4000634875163929e-05,
      "logits/chosen": -2.6427102088928223,
      "logits/rejected": -2.703989028930664,
      "logps/chosen": -173.3591766357422,
      "logps/rejected": -622.0789794921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.122755527496338,
      "rewards/margins": 51.858497619628906,
      "rewards/rejected": -49.73574447631836,
      "step": 150
    },
    {
      "epoch": 0.9935483870967742,
      "eval_logits/chosen": -2.4111573696136475,
      "eval_logits/rejected": -2.533914566040039,
      "eval_logps/chosen": -121.9600601196289,
      "eval_logps/rejected": -688.9767456054688,
      "eval_loss": 0.0001295939728152007,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.3132941722869873,
      "eval_rewards/margins": 61.79844284057617,
      "eval_rewards/rejected": -62.11174011230469,
      "eval_runtime": 13.3156,
      "eval_samples_per_second": 15.02,
      "eval_steps_per_second": 7.51,
      "step": 154
    },
    {
      "epoch": 1.0,
      "grad_norm": 0.1435479074716568,
      "learning_rate": 1.3598595977898393e-05,
      "logits/chosen": -2.470811128616333,
      "logits/rejected": -2.687915086746216,
      "logps/chosen": -176.60227966308594,
      "logps/rejected": -602.3994750976562,
      "loss": 0.003,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 1.7994766235351562,
      "rewards/margins": 50.05354309082031,
      "rewards/rejected": -48.254066467285156,
      "step": 155
    },
    {
      "epoch": 1.032258064516129,
      "grad_norm": 4.26612663269043,
      "learning_rate": 1.3191598394409396e-05,
      "logits/chosen": -2.4601125717163086,
      "logits/rejected": -2.6306862831115723,
      "logps/chosen": -182.5096893310547,
      "logps/rejected": -619.1905517578125,
      "loss": 0.0084,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": 1.3070443868637085,
      "rewards/margins": 50.604530334472656,
      "rewards/rejected": -49.2974853515625,
      "step": 160
    },
    {
      "epoch": 1.064516129032258,
      "grad_norm": 0.07364771515130997,
      "learning_rate": 1.2780969666218915e-05,
      "logits/chosen": -2.371699094772339,
      "logits/rejected": -2.5179049968719482,
      "logps/chosen": -171.65237426757812,
      "logps/rejected": -626.7579345703125,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8574281930923462,
      "rewards/margins": 52.002235412597656,
      "rewards/rejected": -50.144805908203125,
      "step": 165
    },
    {
      "epoch": 1.096774193548387,
      "grad_norm": 2.0316293239593506,
      "learning_rate": 1.2368049178888188e-05,
      "logits/chosen": -2.3688201904296875,
      "logits/rejected": -2.4845595359802246,
      "logps/chosen": -167.29345703125,
      "logps/rejected": -637.6082763671875,
      "loss": 0.0059,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 2.473865032196045,
      "rewards/margins": 53.60892868041992,
      "rewards/rejected": -51.13506317138672,
      "step": 170
    },
    {
      "epoch": 1.129032258064516,
      "grad_norm": 5.519531896425178e-06,
      "learning_rate": 1.1954183793220426e-05,
      "logits/chosen": -2.4739153385162354,
      "logits/rejected": -2.5887179374694824,
      "logps/chosen": -168.55252075195312,
      "logps/rejected": -629.6751098632812,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.837996006011963,
      "rewards/margins": 54.0001106262207,
      "rewards/rejected": -50.16211700439453,
      "step": 175
    },
    {
      "epoch": 1.1612903225806452,
      "grad_norm": 4.96319489684538e-06,
      "learning_rate": 1.154072345208082e-05,
      "logits/chosen": -2.675598382949829,
      "logits/rejected": -2.770794153213501,
      "logps/chosen": -167.05870056152344,
      "logps/rejected": -619.7980346679688,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.837059736251831,
      "rewards/margins": 52.89777755737305,
      "rewards/rejected": -49.06071853637695,
      "step": 180
    },
    {
      "epoch": 1.1935483870967742,
      "grad_norm": 2.3110576876206324e-05,
      "learning_rate": 1.1129016777163497e-05,
      "logits/chosen": -2.7444872856140137,
      "logits/rejected": -2.823488235473633,
      "logps/chosen": -158.0244140625,
      "logps/rejected": -612.7073974609375,
      "loss": 0.0088,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 3.5461955070495605,
      "rewards/margins": 52.555580139160156,
      "rewards/rejected": -49.00938034057617,
      "step": 185
    },
    {
      "epoch": 1.2258064516129032,
      "grad_norm": 0.00015435948444064707,
      "learning_rate": 1.0720406670067866e-05,
      "logits/chosen": -3.0422451496124268,
      "logits/rejected": -3.078159809112549,
      "logps/chosen": -162.6065216064453,
      "logps/rejected": -648.2110595703125,
      "loss": 0.009,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 2.963984966278076,
      "rewards/margins": 54.6415901184082,
      "rewards/rejected": -51.6776123046875,
      "step": 190
    },
    {
      "epoch": 1.2580645161290323,
      "grad_norm": 0.23506908118724823,
      "learning_rate": 1.0316225932032739e-05,
      "logits/chosen": -3.1033904552459717,
      "logits/rejected": -3.1072916984558105,
      "logps/chosen": -162.05966186523438,
      "logps/rejected": -627.5680541992188,
      "loss": 0.0017,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 3.406363010406494,
      "rewards/margins": 53.60102462768555,
      "rewards/rejected": -50.194664001464844,
      "step": 195
    },
    {
      "epoch": 1.2903225806451613,
      "grad_norm": 0.0005963341682218015,
      "learning_rate": 9.917792916615701e-06,
      "logits/chosen": -3.057863473892212,
      "logits/rejected": -3.170236349105835,
      "logps/chosen": -172.58238220214844,
      "logps/rejected": -642.7207641601562,
      "loss": 0.0015,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 3.198392629623413,
      "rewards/margins": 54.85893630981445,
      "rewards/rejected": -51.66054153442383,
      "step": 200
    },
    {
      "epoch": 1.3225806451612903,
      "grad_norm": 3.264209590270184e-05,
      "learning_rate": 9.526407229497827e-06,
      "logits/chosen": -3.2984588146209717,
      "logits/rejected": -3.360607624053955,
      "logps/chosen": -163.1930694580078,
      "logps/rejected": -648.0612182617188,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.657193660736084,
      "rewards/margins": 54.8690071105957,
      "rewards/rejected": -52.211814880371094,
      "step": 205
    },
    {
      "epoch": 1.3548387096774195,
      "grad_norm": 4.916101943308604e-07,
      "learning_rate": 9.143345489440102e-06,
      "logits/chosen": -3.411468982696533,
      "logits/rejected": -3.4234490394592285,
      "logps/chosen": -162.50698852539062,
      "logps/rejected": -648.9544677734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4228503704071045,
      "rewards/margins": 56.02128982543945,
      "rewards/rejected": -52.59843826293945,
      "step": 210
    },
    {
      "epoch": 1.3870967741935485,
      "grad_norm": 3.84959868426904e-08,
      "learning_rate": 8.769857164218336e-06,
      "logits/chosen": -3.3674731254577637,
      "logits/rejected": -3.465867519378662,
      "logps/chosen": -156.56558227539062,
      "logps/rejected": -662.9464721679688,
      "loss": 0.014,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 3.516038179397583,
      "rewards/margins": 56.7149772644043,
      "rewards/rejected": -53.198936462402344,
      "step": 215
    },
    {
      "epoch": 1.4193548387096775,
      "grad_norm": 1.2234114365128335e-05,
      "learning_rate": 8.407160495119008e-06,
      "logits/chosen": -3.2030277252197266,
      "logits/rejected": -3.3925929069519043,
      "logps/chosen": -170.909423828125,
      "logps/rejected": -667.1768798828125,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.376603364944458,
      "rewards/margins": 57.6746826171875,
      "rewards/rejected": -54.2980842590332,
      "step": 220
    },
    {
      "epoch": 1.4516129032258065,
      "grad_norm": 0.19691216945648193,
      "learning_rate": 8.056438523289393e-06,
      "logits/chosen": -3.271845579147339,
      "logits/rejected": -3.299424409866333,
      "logps/chosen": -165.4629364013672,
      "logps/rejected": -647.8621826171875,
      "loss": 0.0047,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 2.843123435974121,
      "rewards/margins": 54.9563102722168,
      "rewards/rejected": -52.113182067871094,
      "step": 225
    },
    {
      "epoch": 1.4838709677419355,
      "grad_norm": 0.0007676176610402763,
      "learning_rate": 7.718835230903237e-06,
      "logits/chosen": -3.305957794189453,
      "logits/rejected": -3.3377411365509033,
      "logps/chosen": -168.66317749023438,
      "logps/rejected": -664.5413818359375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4339606761932373,
      "rewards/margins": 57.25107955932617,
      "rewards/rejected": -53.81711959838867,
      "step": 230
    },
    {
      "epoch": 1.5161290322580645,
      "grad_norm": 1.1214526239200495e-05,
      "learning_rate": 7.3954518097287005e-06,
      "logits/chosen": -3.360619068145752,
      "logits/rejected": -3.381943464279175,
      "logps/chosen": -170.27052307128906,
      "logps/rejected": -697.7355346679688,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.744898557662964,
      "rewards/margins": 60.68017578125,
      "rewards/rejected": -56.935279846191406,
      "step": 235
    },
    {
      "epoch": 1.5483870967741935,
      "grad_norm": 0.001958629349246621,
      "learning_rate": 7.08734306926967e-06,
      "logits/chosen": -3.3265819549560547,
      "logits/rejected": -3.33760404586792,
      "logps/chosen": -158.65939331054688,
      "logps/rejected": -662.5846557617188,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8199775218963623,
      "rewards/margins": 56.54740524291992,
      "rewards/rejected": -53.7274284362793,
      "step": 240
    },
    {
      "epoch": 1.5806451612903225,
      "grad_norm": 1.1762933354475535e-05,
      "learning_rate": 6.7955139961964235e-06,
      "logits/chosen": -3.417844295501709,
      "logits/rejected": -3.355306625366211,
      "logps/chosen": -163.48977661132812,
      "logps/rejected": -671.9425659179688,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6044082641601562,
      "rewards/margins": 57.15119171142578,
      "rewards/rejected": -54.546783447265625,
      "step": 245
    },
    {
      "epoch": 1.6129032258064515,
      "grad_norm": 0.3113669157028198,
      "learning_rate": 6.5209164762879876e-06,
      "logits/chosen": -3.1726534366607666,
      "logits/rejected": -3.2277159690856934,
      "logps/chosen": -158.69281005859375,
      "logps/rejected": -670.3931884765625,
      "loss": 0.018,
      "rewards/accuracies": 0.995312511920929,
      "rewards/chosen": 2.9490036964416504,
      "rewards/margins": 57.56342315673828,
      "rewards/rejected": -54.61442184448242,
      "step": 250
    },
    {
      "epoch": 1.6451612903225805,
      "grad_norm": 0.15534324944019318,
      "learning_rate": 6.26444618957859e-06,
      "logits/chosen": -2.9065983295440674,
      "logits/rejected": -3.0085651874542236,
      "logps/chosen": -162.54873657226562,
      "logps/rejected": -677.47412109375,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7529945373535156,
      "rewards/margins": 58.81446075439453,
      "rewards/rejected": -55.06147003173828,
      "step": 255
    },
    {
      "epoch": 1.6774193548387095,
      "grad_norm": 0.5288254022598267,
      "learning_rate": 6.026939688835542e-06,
      "logits/chosen": -2.7987606525421143,
      "logits/rejected": -2.935157299041748,
      "logps/chosen": -156.1405792236328,
      "logps/rejected": -637.0443115234375,
      "loss": 0.0059,
      "rewards/accuracies": 0.996874988079071,
      "rewards/chosen": 2.6880862712860107,
      "rewards/margins": 53.83855056762695,
      "rewards/rejected": -51.15046691894531,
      "step": 260
    },
    {
      "epoch": 1.7096774193548387,
      "grad_norm": 9.21001483220607e-05,
      "learning_rate": 5.809171670898002e-06,
      "logits/chosen": -2.7774410247802734,
      "logits/rejected": -2.9501843452453613,
      "logps/chosen": -161.6907501220703,
      "logps/rejected": -681.8929443359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7917118072509766,
      "rewards/margins": 58.00335693359375,
      "rewards/rejected": -55.211647033691406,
      "step": 265
    },
    {
      "epoch": 1.7419354838709677,
      "grad_norm": 0.0019305861787870526,
      "learning_rate": 5.611852449776906e-06,
      "logits/chosen": -2.70743989944458,
      "logits/rejected": -2.9343791007995605,
      "logps/chosen": -158.28378295898438,
      "logps/rejected": -656.1748046875,
      "loss": 0.0044,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 2.8768951892852783,
      "rewards/margins": 56.165191650390625,
      "rewards/rejected": -53.288291931152344,
      "step": 270
    },
    {
      "epoch": 1.7741935483870968,
      "grad_norm": 0.05368852615356445,
      "learning_rate": 5.435625639758302e-06,
      "logits/chosen": -2.6554648876190186,
      "logits/rejected": -2.9207005500793457,
      "logps/chosen": -173.9909210205078,
      "logps/rejected": -673.0507202148438,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8360095024108887,
      "rewards/margins": 57.830604553222656,
      "rewards/rejected": -54.994598388671875,
      "step": 275
    },
    {
      "epoch": 1.8064516129032258,
      "grad_norm": 1.7408574422006495e-06,
      "learning_rate": 5.281066056067326e-06,
      "logits/chosen": -2.7118172645568848,
      "logits/rejected": -2.919018268585205,
      "logps/chosen": -169.70877075195312,
      "logps/rejected": -689.2927856445312,
      "loss": 0.008,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 2.9414725303649902,
      "rewards/margins": 58.81359100341797,
      "rewards/rejected": -55.87211990356445,
      "step": 280
    },
    {
      "epoch": 1.838709677419355,
      "grad_norm": 0.02011033147573471,
      "learning_rate": 5.148677839940424e-06,
      "logits/chosen": -2.6920559406280518,
      "logits/rejected": -2.856995105743408,
      "logps/chosen": -170.58425903320312,
      "logps/rejected": -718.7882690429688,
      "loss": 0.0033,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 2.741574764251709,
      "rewards/margins": 61.97066116333008,
      "rewards/rejected": -59.22908401489258,
      "step": 285
    },
    {
      "epoch": 1.870967741935484,
      "grad_norm": 0.07748307287693024,
      "learning_rate": 5.038892814221403e-06,
      "logits/chosen": -2.6701674461364746,
      "logits/rejected": -2.828078031539917,
      "logps/chosen": -166.5321807861328,
      "logps/rejected": -721.7613525390625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.030878782272339,
      "rewards/margins": 61.87688446044922,
      "rewards/rejected": -59.84600830078125,
      "step": 290
    },
    {
      "epoch": 1.903225806451613,
      "grad_norm": 4.640487168217078e-05,
      "learning_rate": 4.952069074845096e-06,
      "logits/chosen": -2.712212324142456,
      "logits/rejected": -2.777492046356201,
      "logps/chosen": -163.62423706054688,
      "logps/rejected": -726.83251953125,
      "loss": 0.0021,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 2.233910083770752,
      "rewards/margins": 62.41532516479492,
      "rewards/rejected": -60.18141555786133,
      "step": 295
    },
    {
      "epoch": 1.935483870967742,
      "grad_norm": 0.00010279910929966718,
      "learning_rate": 4.888489822802835e-06,
      "logits/chosen": -2.6591506004333496,
      "logits/rejected": -2.798229694366455,
      "logps/chosen": -177.29238891601562,
      "logps/rejected": -769.4779663085938,
      "loss": 0.0141,
      "rewards/accuracies": 0.9984375238418579,
      "rewards/chosen": 1.5061697959899902,
      "rewards/margins": 65.76886749267578,
      "rewards/rejected": -64.2626953125,
      "step": 300
    },
    {
      "epoch": 1.967741935483871,
      "grad_norm": 1.4639048458775505e-05,
      "learning_rate": 4.848362440399692e-06,
      "logits/chosen": -2.73205828666687,
      "logits/rejected": -2.761648178100586,
      "logps/chosen": -175.31532287597656,
      "logps/rejected": -765.2847900390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8591419458389282,
      "rewards/margins": 65.3056869506836,
      "rewards/rejected": -63.4465446472168,
      "step": 305
    },
    {
      "epoch": 1.9870967741935484,
      "eval_logits/chosen": -2.4073026180267334,
      "eval_logits/rejected": -2.258913516998291,
      "eval_logps/chosen": -124.21800994873047,
      "eval_logps/rejected": -902.7763061523438,
      "eval_loss": 9.752377081895247e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -0.5390886664390564,
      "eval_rewards/margins": 82.95260620117188,
      "eval_rewards/rejected": -83.49169921875,
      "eval_runtime": 11.9364,
      "eval_samples_per_second": 16.755,
      "eval_steps_per_second": 8.378,
      "step": 308
    }
  ],
  "logging_steps": 5,
  "max_steps": 310,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 64,
  "trial_name": null,
  "trial_params": null
}