{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9989708404802744,
  "eval_steps": 500,
  "global_step": 273,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.003659233847913093,
      "grad_norm": 346.33831787109375,
      "learning_rate": 1.9926739926739927e-06,
      "logits/chosen": -0.64453125,
      "logits/rejected": -0.62890625,
      "logps/chosen": -162.0,
      "logps/rejected": -163.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.007318467695826186,
      "grad_norm": 344.1603698730469,
      "learning_rate": 1.9853479853479855e-06,
      "logits/chosen": -0.609375,
      "logits/rejected": -0.625,
      "logps/chosen": -139.0,
      "logps/rejected": -188.0,
      "loss": 0.7092,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.00860595703125,
      "rewards/margins": -0.038330078125,
      "rewards/rejected": 0.0296630859375,
      "step": 2
    },
    {
      "epoch": 0.010977701543739279,
      "grad_norm": 321.5267028808594,
      "learning_rate": 1.978021978021978e-06,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.6171875,
      "logps/chosen": -145.0,
      "logps/rejected": -188.0,
      "loss": 0.6973,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": -0.005462646484375,
      "rewards/margins": -0.00860595703125,
      "rewards/rejected": 0.0031280517578125,
      "step": 3
    },
    {
      "epoch": 0.014636935391652372,
      "grad_norm": 338.7471923828125,
      "learning_rate": 1.9706959706959706e-06,
      "logits/chosen": -0.5625,
      "logits/rejected": -0.60546875,
      "logps/chosen": -160.0,
      "logps/rejected": -189.0,
      "loss": 0.6735,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.0218505859375,
      "rewards/margins": 0.035888671875,
      "rewards/rejected": -0.01409912109375,
      "step": 4
    },
    {
      "epoch": 0.018296169239565466,
      "grad_norm": 315.54010009765625,
      "learning_rate": 1.9633699633699634e-06,
      "logits/chosen": -0.55859375,
      "logits/rejected": -0.58203125,
      "logps/chosen": -181.0,
      "logps/rejected": -195.0,
      "loss": 0.6837,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.0031280517578125,
      "rewards/margins": 0.0218505859375,
      "rewards/rejected": -0.018798828125,
      "step": 5
    },
    {
      "epoch": 0.021955403087478557,
      "grad_norm": 321.4845886230469,
      "learning_rate": 1.9560439560439557e-06,
      "logits/chosen": -0.6484375,
      "logits/rejected": -0.6484375,
      "logps/chosen": -177.0,
      "logps/rejected": -179.0,
      "loss": 0.6964,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0234375,
      "rewards/margins": -7.62939453125e-06,
      "rewards/rejected": -0.0234375,
      "step": 6
    },
    {
      "epoch": 0.025614636935391653,
      "grad_norm": 330.94757080078125,
      "learning_rate": 1.9487179487179485e-06,
      "logits/chosen": -0.640625,
      "logits/rejected": -0.63671875,
      "logps/chosen": -145.0,
      "logps/rejected": -167.0,
      "loss": 0.6996,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0218505859375,
      "rewards/margins": -0.01251220703125,
      "rewards/rejected": -0.0093994140625,
      "step": 7
    },
    {
      "epoch": 0.029273870783304744,
      "grad_norm": 348.64080810546875,
      "learning_rate": 1.9413919413919413e-06,
      "logits/chosen": -0.59765625,
      "logits/rejected": -0.609375,
      "logps/chosen": -169.0,
      "logps/rejected": -186.0,
      "loss": 0.692,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.016357421875,
      "rewards/margins": 0.01019287109375,
      "rewards/rejected": -0.026611328125,
      "step": 8
    },
    {
      "epoch": 0.03293310463121784,
      "grad_norm": 331.8152770996094,
      "learning_rate": 1.934065934065934e-06,
      "logits/chosen": -0.66015625,
      "logits/rejected": -0.640625,
      "logps/chosen": -173.0,
      "logps/rejected": -182.0,
      "loss": 0.6917,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.039794921875,
      "rewards/margins": 0.0015106201171875,
      "rewards/rejected": -0.04150390625,
      "step": 9
    },
    {
      "epoch": 0.03659233847913093,
      "grad_norm": 341.0348205566406,
      "learning_rate": 1.926739926739927e-06,
      "logits/chosen": -0.58984375,
      "logits/rejected": -0.6171875,
      "logps/chosen": -156.0,
      "logps/rejected": -216.0,
      "loss": 0.6727,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.034423828125,
      "rewards/margins": 0.042236328125,
      "rewards/rejected": -0.07666015625,
      "step": 10
    },
    {
      "epoch": 0.04025157232704402,
      "grad_norm": 314.3051452636719,
      "learning_rate": 1.9194139194139196e-06,
      "logits/chosen": -0.65234375,
      "logits/rejected": -0.64453125,
      "logps/chosen": -152.0,
      "logps/rejected": -176.0,
      "loss": 0.6743,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0250244140625,
      "rewards/margins": 0.042236328125,
      "rewards/rejected": -0.0673828125,
      "step": 11
    },
    {
      "epoch": 0.043910806174957115,
      "grad_norm": 315.991943359375,
      "learning_rate": 1.912087912087912e-06,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.62890625,
      "logps/chosen": -137.0,
      "logps/rejected": -167.0,
      "loss": 0.6733,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.044677734375,
      "rewards/margins": 0.038330078125,
      "rewards/rejected": -0.0830078125,
      "step": 12
    },
    {
      "epoch": 0.047570040022870214,
      "grad_norm": 302.81024169921875,
      "learning_rate": 1.9047619047619045e-06,
      "logits/chosen": -0.62890625,
      "logits/rejected": -0.6171875,
      "logps/chosen": -162.0,
      "logps/rejected": -162.0,
      "loss": 0.7054,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": -0.078125,
      "rewards/margins": -0.01806640625,
      "rewards/rejected": -0.060302734375,
      "step": 13
    },
    {
      "epoch": 0.051229273870783305,
      "grad_norm": 312.2680969238281,
      "learning_rate": 1.8974358974358973e-06,
      "logits/chosen": -0.5859375,
      "logits/rejected": -0.5625,
      "logps/chosen": -141.0,
      "logps/rejected": -140.0,
      "loss": 0.6847,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.036865234375,
      "rewards/margins": 0.01953125,
      "rewards/rejected": -0.056396484375,
      "step": 14
    },
    {
      "epoch": 0.0548885077186964,
      "grad_norm": 450.512451171875,
      "learning_rate": 1.89010989010989e-06,
      "logits/chosen": -0.5703125,
      "logits/rejected": -0.5703125,
      "logps/chosen": -170.0,
      "logps/rejected": -186.0,
      "loss": 0.703,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.09912109375,
      "rewards/margins": -0.0194091796875,
      "rewards/rejected": -0.07958984375,
      "step": 15
    },
    {
      "epoch": 0.05854774156660949,
      "grad_norm": 319.3954772949219,
      "learning_rate": 1.8827838827838826e-06,
      "logits/chosen": -0.62890625,
      "logits/rejected": -0.62109375,
      "logps/chosen": -183.0,
      "logps/rejected": -161.0,
      "loss": 0.6943,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.06591796875,
      "rewards/margins": 0.004730224609375,
      "rewards/rejected": -0.0703125,
      "step": 16
    },
    {
      "epoch": 0.06220697541452259,
      "grad_norm": 328.090087890625,
      "learning_rate": 1.8754578754578754e-06,
      "logits/chosen": -0.58984375,
      "logits/rejected": -0.6328125,
      "logps/chosen": -158.0,
      "logps/rejected": -169.0,
      "loss": 0.6694,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.02587890625,
      "rewards/margins": 0.053955078125,
      "rewards/rejected": -0.07958984375,
      "step": 17
    },
    {
      "epoch": 0.06586620926243568,
      "grad_norm": 339.8540954589844,
      "learning_rate": 1.8681318681318681e-06,
      "logits/chosen": -0.6328125,
      "logits/rejected": -0.59375,
      "logps/chosen": -147.0,
      "logps/rejected": -165.0,
      "loss": 0.6788,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.046875,
      "rewards/margins": 0.035888671875,
      "rewards/rejected": -0.0830078125,
      "step": 18
    },
    {
      "epoch": 0.06952544311034876,
      "grad_norm": 316.8853759765625,
      "learning_rate": 1.8608058608058607e-06,
      "logits/chosen": -0.60546875,
      "logits/rejected": -0.609375,
      "logps/chosen": -150.0,
      "logps/rejected": -171.0,
      "loss": 0.682,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.050048828125,
      "rewards/margins": 0.0289306640625,
      "rewards/rejected": -0.0791015625,
      "step": 19
    },
    {
      "epoch": 0.07318467695826186,
      "grad_norm": 318.4516296386719,
      "learning_rate": 1.8534798534798535e-06,
      "logits/chosen": -0.6484375,
      "logits/rejected": -0.6484375,
      "logps/chosen": -138.0,
      "logps/rejected": -179.0,
      "loss": 0.6903,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.05322265625,
      "rewards/margins": 0.010986328125,
      "rewards/rejected": -0.06396484375,
      "step": 20
    },
    {
      "epoch": 0.07684391080617496,
      "grad_norm": 382.6263122558594,
      "learning_rate": 1.8461538461538462e-06,
      "logits/chosen": -0.58203125,
      "logits/rejected": -0.5546875,
      "logps/chosen": -143.0,
      "logps/rejected": -256.0,
      "loss": 0.6578,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.04541015625,
      "rewards/margins": 0.08154296875,
      "rewards/rejected": -0.126953125,
      "step": 21
    },
    {
      "epoch": 0.08050314465408805,
      "grad_norm": 351.90972900390625,
      "learning_rate": 1.8388278388278386e-06,
      "logits/chosen": -0.63671875,
      "logits/rejected": -0.578125,
      "logps/chosen": -152.0,
      "logps/rejected": -213.0,
      "loss": 0.6783,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.059326171875,
      "rewards/margins": 0.0296630859375,
      "rewards/rejected": -0.08935546875,
      "step": 22
    },
    {
      "epoch": 0.08416237850200115,
      "grad_norm": 306.3103332519531,
      "learning_rate": 1.8315018315018314e-06,
      "logits/chosen": -0.59765625,
      "logits/rejected": -0.578125,
      "logps/chosen": -140.0,
      "logps/rejected": -173.0,
      "loss": 0.6982,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.07177734375,
      "rewards/margins": 0.00628662109375,
      "rewards/rejected": -0.078125,
      "step": 23
    },
    {
      "epoch": 0.08782161234991423,
      "grad_norm": 322.9337158203125,
      "learning_rate": 1.8241758241758241e-06,
      "logits/chosen": -0.6015625,
      "logits/rejected": -0.57421875,
      "logps/chosen": -149.0,
      "logps/rejected": -160.0,
      "loss": 0.6774,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0595703125,
      "rewards/margins": 0.03759765625,
      "rewards/rejected": -0.09716796875,
      "step": 24
    },
    {
      "epoch": 0.09148084619782733,
      "grad_norm": 348.9863586425781,
      "learning_rate": 1.8168498168498167e-06,
      "logits/chosen": -0.59765625,
      "logits/rejected": -0.57421875,
      "logps/chosen": -183.0,
      "logps/rejected": -224.0,
      "loss": 0.6976,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.10498046875,
      "rewards/margins": 0.0078125,
      "rewards/rejected": -0.11279296875,
      "step": 25
    },
    {
      "epoch": 0.09514008004574043,
      "grad_norm": 312.5710144042969,
      "learning_rate": 1.8095238095238095e-06,
      "logits/chosen": -0.59375,
      "logits/rejected": -0.57421875,
      "logps/chosen": -160.0,
      "logps/rejected": -156.0,
      "loss": 0.6707,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.078125,
      "rewards/margins": 0.0361328125,
      "rewards/rejected": -0.1142578125,
      "step": 26
    },
    {
      "epoch": 0.09879931389365351,
      "grad_norm": 312.729736328125,
      "learning_rate": 1.8021978021978023e-06,
      "logits/chosen": -0.58984375,
      "logits/rejected": -0.56640625,
      "logps/chosen": -152.0,
      "logps/rejected": -177.0,
      "loss": 0.6608,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06884765625,
      "rewards/margins": 0.0703125,
      "rewards/rejected": -0.1396484375,
      "step": 27
    },
    {
      "epoch": 0.10245854774156661,
      "grad_norm": 335.3136291503906,
      "learning_rate": 1.7948717948717948e-06,
      "logits/chosen": -0.64453125,
      "logits/rejected": -0.6484375,
      "logps/chosen": -178.0,
      "logps/rejected": -159.0,
      "loss": 0.7061,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.142578125,
      "rewards/margins": -0.0186767578125,
      "rewards/rejected": -0.12353515625,
      "step": 28
    },
    {
      "epoch": 0.10611778158947971,
      "grad_norm": 308.06744384765625,
      "learning_rate": 1.7875457875457876e-06,
      "logits/chosen": -0.66796875,
      "logits/rejected": -0.62890625,
      "logps/chosen": -177.0,
      "logps/rejected": -165.0,
      "loss": 0.6785,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.10009765625,
      "rewards/margins": 0.035888671875,
      "rewards/rejected": -0.1357421875,
      "step": 29
    },
    {
      "epoch": 0.1097770154373928,
      "grad_norm": 324.392333984375,
      "learning_rate": 1.78021978021978e-06,
      "logits/chosen": -0.62890625,
      "logits/rejected": -0.6484375,
      "logps/chosen": -169.0,
      "logps/rejected": -166.0,
      "loss": 0.6895,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0908203125,
      "rewards/margins": 0.01409912109375,
      "rewards/rejected": -0.10498046875,
      "step": 30
    },
    {
      "epoch": 0.11343624928530589,
      "grad_norm": 348.1455993652344,
      "learning_rate": 1.7728937728937727e-06,
      "logits/chosen": -0.640625,
      "logits/rejected": -0.62890625,
      "logps/chosen": -150.0,
      "logps/rejected": -173.0,
      "loss": 0.6512,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.057861328125,
      "rewards/margins": 0.0859375,
      "rewards/rejected": -0.1435546875,
      "step": 31
    },
    {
      "epoch": 0.11709548313321898,
      "grad_norm": 310.3366394042969,
      "learning_rate": 1.7655677655677655e-06,
      "logits/chosen": -0.6171875,
      "logits/rejected": -0.62890625,
      "logps/chosen": -185.0,
      "logps/rejected": -176.0,
      "loss": 0.6638,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08154296875,
      "rewards/margins": 0.06494140625,
      "rewards/rejected": -0.146484375,
      "step": 32
    },
    {
      "epoch": 0.12075471698113208,
      "grad_norm": 367.6607971191406,
      "learning_rate": 1.758241758241758e-06,
      "logits/chosen": -0.609375,
      "logits/rejected": -0.640625,
      "logps/chosen": -213.0,
      "logps/rejected": -152.0,
      "loss": 0.7106,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.15625,
      "rewards/margins": -0.0234375,
      "rewards/rejected": -0.1328125,
      "step": 33
    },
    {
      "epoch": 0.12441395082904518,
      "grad_norm": 304.85638427734375,
      "learning_rate": 1.7509157509157508e-06,
      "logits/chosen": -0.55859375,
      "logits/rejected": -0.58203125,
      "logps/chosen": -155.0,
      "logps/rejected": -179.0,
      "loss": 0.676,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0859375,
      "rewards/margins": 0.0390625,
      "rewards/rejected": -0.125,
      "step": 34
    },
    {
      "epoch": 0.12807318467695827,
      "grad_norm": 344.142333984375,
      "learning_rate": 1.7435897435897436e-06,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.6171875,
      "logps/chosen": -148.0,
      "logps/rejected": -158.0,
      "loss": 0.7004,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": -0.10693359375,
      "rewards/margins": -0.0133056640625,
      "rewards/rejected": -0.09375,
      "step": 35
    },
    {
      "epoch": 0.13173241852487136,
      "grad_norm": 330.11029052734375,
      "learning_rate": 1.7362637362637362e-06,
      "logits/chosen": -0.5859375,
      "logits/rejected": -0.59765625,
      "logps/chosen": -138.0,
      "logps/rejected": -165.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.0859375,
      "rewards/margins": 0.010986328125,
      "rewards/rejected": -0.09716796875,
      "step": 36
    },
    {
      "epoch": 0.13539165237278444,
      "grad_norm": 367.31817626953125,
      "learning_rate": 1.728937728937729e-06,
      "logits/chosen": -0.61328125,
      "logits/rejected": -0.64453125,
      "logps/chosen": -150.0,
      "logps/rejected": -159.0,
      "loss": 0.6768,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08544921875,
      "rewards/margins": 0.046142578125,
      "rewards/rejected": -0.1318359375,
      "step": 37
    },
    {
      "epoch": 0.13905088622069753,
      "grad_norm": 332.1564025878906,
      "learning_rate": 1.7216117216117217e-06,
      "logits/chosen": -0.62109375,
      "logits/rejected": -0.59375,
      "logps/chosen": -158.0,
      "logps/rejected": -179.0,
      "loss": 0.6858,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.0859375,
      "rewards/margins": 0.02587890625,
      "rewards/rejected": -0.11181640625,
      "step": 38
    },
    {
      "epoch": 0.14271012006861064,
      "grad_norm": 358.6282653808594,
      "learning_rate": 1.714285714285714e-06,
      "logits/chosen": -0.67578125,
      "logits/rejected": -0.64453125,
      "logps/chosen": -151.0,
      "logps/rejected": -169.0,
      "loss": 0.6743,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0830078125,
      "rewards/margins": 0.0390625,
      "rewards/rejected": -0.1220703125,
      "step": 39
    },
    {
      "epoch": 0.14636935391652373,
      "grad_norm": 344.2862548828125,
      "learning_rate": 1.7069597069597068e-06,
      "logits/chosen": -0.5859375,
      "logits/rejected": -0.56640625,
      "logps/chosen": -144.0,
      "logps/rejected": -224.0,
      "loss": 0.653,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0625,
      "rewards/margins": 0.10791015625,
      "rewards/rejected": -0.1708984375,
      "step": 40
    },
    {
      "epoch": 0.1500285877644368,
      "grad_norm": 317.97698974609375,
      "learning_rate": 1.6996336996336996e-06,
      "logits/chosen": -0.65234375,
      "logits/rejected": -0.6328125,
      "logps/chosen": -157.0,
      "logps/rejected": -177.0,
      "loss": 0.6595,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.051513671875,
      "rewards/margins": 0.07763671875,
      "rewards/rejected": -0.12890625,
      "step": 41
    },
    {
      "epoch": 0.15368782161234992,
      "grad_norm": 336.11773681640625,
      "learning_rate": 1.6923076923076922e-06,
      "logits/chosen": -0.65234375,
      "logits/rejected": -0.6171875,
      "logps/chosen": -172.0,
      "logps/rejected": -168.0,
      "loss": 0.6805,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.09521484375,
      "rewards/margins": 0.03125,
      "rewards/rejected": -0.126953125,
      "step": 42
    },
    {
      "epoch": 0.157347055460263,
      "grad_norm": 306.1403503417969,
      "learning_rate": 1.684981684981685e-06,
      "logits/chosen": -0.6328125,
      "logits/rejected": -0.6015625,
      "logps/chosen": -146.0,
      "logps/rejected": -149.0,
      "loss": 0.6506,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06103515625,
      "rewards/margins": 0.0947265625,
      "rewards/rejected": -0.1552734375,
      "step": 43
    },
    {
      "epoch": 0.1610062893081761,
      "grad_norm": 330.2728271484375,
      "learning_rate": 1.6776556776556777e-06,
      "logits/chosen": -0.609375,
      "logits/rejected": -0.55859375,
      "logps/chosen": -141.0,
      "logps/rejected": -180.0,
      "loss": 0.6661,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0703125,
      "rewards/margins": 0.06103515625,
      "rewards/rejected": -0.1318359375,
      "step": 44
    },
    {
      "epoch": 0.1646655231560892,
      "grad_norm": 300.4102783203125,
      "learning_rate": 1.6703296703296703e-06,
      "logits/chosen": -0.63671875,
      "logits/rejected": -0.66015625,
      "logps/chosen": -148.0,
      "logps/rejected": -156.0,
      "loss": 0.6313,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.046142578125,
      "rewards/margins": 0.1357421875,
      "rewards/rejected": -0.181640625,
      "step": 45
    },
    {
      "epoch": 0.1683247570040023,
      "grad_norm": 314.258056640625,
      "learning_rate": 1.663003663003663e-06,
      "logits/chosen": -0.63671875,
      "logits/rejected": -0.6328125,
      "logps/chosen": -145.0,
      "logps/rejected": -164.0,
      "loss": 0.658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08935546875,
      "rewards/margins": 0.08056640625,
      "rewards/rejected": -0.169921875,
      "step": 46
    },
    {
      "epoch": 0.17198399085191537,
      "grad_norm": 300.92364501953125,
      "learning_rate": 1.6556776556776554e-06,
      "logits/chosen": -0.625,
      "logits/rejected": -0.625,
      "logps/chosen": -152.0,
      "logps/rejected": -189.0,
      "loss": 0.6508,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.04541015625,
      "rewards/margins": 0.09716796875,
      "rewards/rejected": -0.142578125,
      "step": 47
    },
    {
      "epoch": 0.17564322469982846,
      "grad_norm": 281.744384765625,
      "learning_rate": 1.6483516483516482e-06,
      "logits/chosen": -0.609375,
      "logits/rejected": -0.6328125,
      "logps/chosen": -131.0,
      "logps/rejected": -165.0,
      "loss": 0.655,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07373046875,
      "rewards/margins": 0.0888671875,
      "rewards/rejected": -0.1630859375,
      "step": 48
    },
    {
      "epoch": 0.17930245854774157,
      "grad_norm": 373.736328125,
      "learning_rate": 1.641025641025641e-06,
      "logits/chosen": -0.6484375,
      "logits/rejected": -0.62890625,
      "logps/chosen": -147.0,
      "logps/rejected": -176.0,
      "loss": 0.6635,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08447265625,
      "rewards/margins": 0.0751953125,
      "rewards/rejected": -0.1591796875,
      "step": 49
    },
    {
      "epoch": 0.18296169239565466,
      "grad_norm": 319.23675537109375,
      "learning_rate": 1.6336996336996335e-06,
      "logits/chosen": -0.62109375,
      "logits/rejected": -0.64453125,
      "logps/chosen": -168.0,
      "logps/rejected": -183.0,
      "loss": 0.6371,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0751953125,
      "rewards/margins": 0.134765625,
      "rewards/rejected": -0.2099609375,
      "step": 50
    },
    {
      "epoch": 0.18662092624356774,
      "grad_norm": 454.0714416503906,
      "learning_rate": 1.6263736263736263e-06,
      "logits/chosen": -0.62890625,
      "logits/rejected": -0.59375,
      "logps/chosen": -171.0,
      "logps/rejected": -196.0,
      "loss": 0.6519,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08740234375,
      "rewards/margins": 0.0908203125,
      "rewards/rejected": -0.1787109375,
      "step": 51
    },
    {
      "epoch": 0.19028016009148085,
      "grad_norm": 316.0301513671875,
      "learning_rate": 1.619047619047619e-06,
      "logits/chosen": -0.62109375,
      "logits/rejected": -0.6484375,
      "logps/chosen": -144.0,
      "logps/rejected": -183.0,
      "loss": 0.6611,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.107421875,
      "rewards/margins": 0.08544921875,
      "rewards/rejected": -0.1923828125,
      "step": 52
    },
    {
      "epoch": 0.19393939393939394,
      "grad_norm": 345.9140625,
      "learning_rate": 1.6117216117216116e-06,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.625,
      "logps/chosen": -169.0,
      "logps/rejected": -173.0,
      "loss": 0.658,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.09375,
      "rewards/margins": 0.08740234375,
      "rewards/rejected": -0.181640625,
      "step": 53
    },
    {
      "epoch": 0.19759862778730702,
      "grad_norm": 334.124755859375,
      "learning_rate": 1.6043956043956044e-06,
      "logits/chosen": -0.66015625,
      "logits/rejected": -0.6484375,
      "logps/chosen": -178.0,
      "logps/rejected": -202.0,
      "loss": 0.6633,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1630859375,
      "rewards/margins": 0.0751953125,
      "rewards/rejected": -0.2373046875,
      "step": 54
    },
    {
      "epoch": 0.20125786163522014,
      "grad_norm": 330.6783142089844,
      "learning_rate": 1.5970695970695972e-06,
      "logits/chosen": -0.6875,
      "logits/rejected": -0.69140625,
      "logps/chosen": -163.0,
      "logps/rejected": -140.0,
      "loss": 0.698,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.1533203125,
      "rewards/margins": -0.010986328125,
      "rewards/rejected": -0.142578125,
      "step": 55
    },
    {
      "epoch": 0.20491709548313322,
      "grad_norm": 370.80706787109375,
      "learning_rate": 1.5897435897435895e-06,
      "logits/chosen": -0.63671875,
      "logits/rejected": -0.66796875,
      "logps/chosen": -166.0,
      "logps/rejected": -181.0,
      "loss": 0.6512,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1689453125,
      "rewards/margins": 0.09375,
      "rewards/rejected": -0.263671875,
      "step": 56
    },
    {
      "epoch": 0.2085763293310463,
      "grad_norm": 323.4024353027344,
      "learning_rate": 1.5824175824175823e-06,
      "logits/chosen": -0.66015625,
      "logits/rejected": -0.62890625,
      "logps/chosen": -171.0,
      "logps/rejected": -157.0,
      "loss": 0.6428,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1318359375,
      "rewards/margins": 0.115234375,
      "rewards/rejected": -0.24609375,
      "step": 57
    },
    {
      "epoch": 0.21223556317895942,
      "grad_norm": 325.7928771972656,
      "learning_rate": 1.575091575091575e-06,
      "logits/chosen": -0.62109375,
      "logits/rejected": -0.59765625,
      "logps/chosen": -154.0,
      "logps/rejected": -167.0,
      "loss": 0.6663,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.12451171875,
      "rewards/margins": 0.07421875,
      "rewards/rejected": -0.1982421875,
      "step": 58
    },
    {
      "epoch": 0.2158947970268725,
      "grad_norm": 365.8585510253906,
      "learning_rate": 1.5677655677655676e-06,
      "logits/chosen": -0.62109375,
      "logits/rejected": -0.61328125,
      "logps/chosen": -192.0,
      "logps/rejected": -202.0,
      "loss": 0.6435,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1611328125,
      "rewards/margins": 0.125,
      "rewards/rejected": -0.287109375,
      "step": 59
    },
    {
      "epoch": 0.2195540308747856,
      "grad_norm": 339.77593994140625,
      "learning_rate": 1.5604395604395604e-06,
      "logits/chosen": -0.640625,
      "logits/rejected": -0.640625,
      "logps/chosen": -162.0,
      "logps/rejected": -207.0,
      "loss": 0.6537,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.15234375,
      "rewards/margins": 0.10595703125,
      "rewards/rejected": -0.2578125,
      "step": 60
    },
    {
      "epoch": 0.22321326472269867,
      "grad_norm": 316.12646484375,
      "learning_rate": 1.5531135531135532e-06,
      "logits/chosen": -0.6796875,
      "logits/rejected": -0.640625,
      "logps/chosen": -137.0,
      "logps/rejected": -166.0,
      "loss": 0.6564,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1494140625,
      "rewards/margins": 0.0908203125,
      "rewards/rejected": -0.240234375,
      "step": 61
    },
    {
      "epoch": 0.22687249857061179,
      "grad_norm": 315.9810485839844,
      "learning_rate": 1.5457875457875457e-06,
      "logits/chosen": -0.5546875,
      "logits/rejected": -0.5703125,
      "logps/chosen": -173.0,
      "logps/rejected": -170.0,
      "loss": 0.6639,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.166015625,
      "rewards/margins": 0.0703125,
      "rewards/rejected": -0.236328125,
      "step": 62
    },
    {
      "epoch": 0.23053173241852487,
      "grad_norm": 312.3990478515625,
      "learning_rate": 1.5384615384615385e-06,
      "logits/chosen": -0.58984375,
      "logits/rejected": -0.58984375,
      "logps/chosen": -160.0,
      "logps/rejected": -205.0,
      "loss": 0.6407,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1416015625,
      "rewards/margins": 0.1416015625,
      "rewards/rejected": -0.283203125,
      "step": 63
    },
    {
      "epoch": 0.23419096626643796,
      "grad_norm": 374.50360107421875,
      "learning_rate": 1.5311355311355309e-06,
      "logits/chosen": -0.6484375,
      "logits/rejected": -0.64453125,
      "logps/chosen": -229.0,
      "logps/rejected": -189.0,
      "loss": 0.6736,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1962890625,
      "rewards/margins": 0.056640625,
      "rewards/rejected": -0.251953125,
      "step": 64
    },
    {
      "epoch": 0.23785020011435107,
      "grad_norm": 322.1225280761719,
      "learning_rate": 1.5238095238095236e-06,
      "logits/chosen": -0.62109375,
      "logits/rejected": -0.640625,
      "logps/chosen": -168.0,
      "logps/rejected": -151.0,
      "loss": 0.6749,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.125,
      "rewards/margins": 0.0390625,
      "rewards/rejected": -0.1640625,
      "step": 65
    },
    {
      "epoch": 0.24150943396226415,
      "grad_norm": 322.7266845703125,
      "learning_rate": 1.5164835164835164e-06,
      "logits/chosen": -0.67578125,
      "logits/rejected": -0.67578125,
      "logps/chosen": -165.0,
      "logps/rejected": -178.0,
      "loss": 0.6255,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0673828125,
      "rewards/margins": 0.1455078125,
      "rewards/rejected": -0.212890625,
      "step": 66
    },
    {
      "epoch": 0.24516866781017724,
      "grad_norm": 317.9302978515625,
      "learning_rate": 1.509157509157509e-06,
      "logits/chosen": -0.69140625,
      "logits/rejected": -0.64453125,
      "logps/chosen": -163.0,
      "logps/rejected": -211.0,
      "loss": 0.6194,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.11279296875,
      "rewards/margins": 0.158203125,
      "rewards/rejected": -0.26953125,
      "step": 67
    },
    {
      "epoch": 0.24882790165809035,
      "grad_norm": 328.2178955078125,
      "learning_rate": 1.5018315018315017e-06,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.6171875,
      "logps/chosen": -162.0,
      "logps/rejected": -165.0,
      "loss": 0.6609,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1513671875,
      "rewards/margins": 0.0888671875,
      "rewards/rejected": -0.2412109375,
      "step": 68
    },
    {
      "epoch": 0.25248713550600344,
      "grad_norm": 350.5845642089844,
      "learning_rate": 1.4945054945054945e-06,
      "logits/chosen": -0.63671875,
      "logits/rejected": -0.6484375,
      "logps/chosen": -188.0,
      "logps/rejected": -182.0,
      "loss": 0.6567,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1591796875,
      "rewards/margins": 0.08056640625,
      "rewards/rejected": -0.2392578125,
      "step": 69
    },
    {
      "epoch": 0.25614636935391655,
      "grad_norm": 329.70184326171875,
      "learning_rate": 1.487179487179487e-06,
      "logits/chosen": -0.6484375,
      "logits/rejected": -0.6875,
      "logps/chosen": -162.0,
      "logps/rejected": -145.0,
      "loss": 0.6819,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.158203125,
      "rewards/margins": 0.0281982421875,
      "rewards/rejected": -0.1865234375,
      "step": 70
    },
    {
      "epoch": 0.2598056032018296,
      "grad_norm": 324.77587890625,
      "learning_rate": 1.4798534798534798e-06,
      "logits/chosen": -0.60546875,
      "logits/rejected": -0.63671875,
      "logps/chosen": -149.0,
      "logps/rejected": -177.0,
      "loss": 0.6274,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.119140625,
      "rewards/margins": 0.15625,
      "rewards/rejected": -0.275390625,
      "step": 71
    },
    {
      "epoch": 0.2634648370497427,
      "grad_norm": 348.8105773925781,
      "learning_rate": 1.4725274725274726e-06,
      "logits/chosen": -0.6484375,
      "logits/rejected": -0.62890625,
      "logps/chosen": -164.0,
      "logps/rejected": -217.0,
      "loss": 0.6568,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1455078125,
      "rewards/margins": 0.109375,
      "rewards/rejected": -0.255859375,
      "step": 72
    },
    {
      "epoch": 0.26712407089765583,
      "grad_norm": 293.3565368652344,
      "learning_rate": 1.465201465201465e-06,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.6328125,
      "logps/chosen": -150.0,
      "logps/rejected": -162.0,
      "loss": 0.6266,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15234375,
      "rewards/margins": 0.1552734375,
      "rewards/rejected": -0.30859375,
      "step": 73
    },
    {
      "epoch": 0.2707833047455689,
      "grad_norm": 322.1363220214844,
      "learning_rate": 1.4578754578754577e-06,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.65234375,
      "logps/chosen": -167.0,
      "logps/rejected": -164.0,
      "loss": 0.6566,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.150390625,
      "rewards/margins": 0.08984375,
      "rewards/rejected": -0.240234375,
      "step": 74
    },
    {
      "epoch": 0.274442538593482,
      "grad_norm": 337.89581298828125,
      "learning_rate": 1.4505494505494505e-06,
      "logits/chosen": -0.625,
      "logits/rejected": -0.62109375,
      "logps/chosen": -166.0,
      "logps/rejected": -174.0,
      "loss": 0.6544,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1689453125,
      "rewards/margins": 0.0869140625,
      "rewards/rejected": -0.255859375,
      "step": 75
    },
    {
      "epoch": 0.27810177244139506,
      "grad_norm": 309.1400146484375,
      "learning_rate": 1.443223443223443e-06,
      "logits/chosen": -0.64453125,
      "logits/rejected": -0.6171875,
      "logps/chosen": -143.0,
      "logps/rejected": -160.0,
      "loss": 0.6221,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10498046875,
      "rewards/margins": 0.171875,
      "rewards/rejected": -0.27734375,
      "step": 76
    },
    {
      "epoch": 0.28176100628930817,
      "grad_norm": 338.5057373046875,
      "learning_rate": 1.4358974358974359e-06,
      "logits/chosen": -0.625,
      "logits/rejected": -0.59765625,
      "logps/chosen": -139.0,
      "logps/rejected": -171.0,
      "loss": 0.6537,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1572265625,
      "rewards/margins": 0.0947265625,
      "rewards/rejected": -0.251953125,
      "step": 77
    },
    {
      "epoch": 0.2854202401372213,
      "grad_norm": 332.32794189453125,
      "learning_rate": 1.4285714285714286e-06,
      "logits/chosen": -0.609375,
      "logits/rejected": -0.6484375,
      "logps/chosen": -154.0,
      "logps/rejected": -185.0,
      "loss": 0.6379,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.142578125,
      "rewards/margins": 0.126953125,
      "rewards/rejected": -0.26953125,
      "step": 78
    },
    {
      "epoch": 0.28907947398513434,
      "grad_norm": 333.6225280761719,
      "learning_rate": 1.4212454212454212e-06,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.65234375,
      "logps/chosen": -151.0,
      "logps/rejected": -183.0,
      "loss": 0.6387,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1298828125,
      "rewards/margins": 0.1328125,
      "rewards/rejected": -0.263671875,
      "step": 79
    },
    {
      "epoch": 0.29273870783304745,
      "grad_norm": 302.85931396484375,
      "learning_rate": 1.413919413919414e-06,
      "logits/chosen": -0.6015625,
      "logits/rejected": -0.6015625,
      "logps/chosen": -153.0,
      "logps/rejected": -169.0,
      "loss": 0.6433,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1708984375,
      "rewards/margins": 0.111328125,
      "rewards/rejected": -0.28125,
      "step": 80
    },
    {
      "epoch": 0.29639794168096056,
      "grad_norm": 329.51885986328125,
      "learning_rate": 1.4065934065934067e-06,
      "logits/chosen": -0.63671875,
      "logits/rejected": -0.62890625,
      "logps/chosen": -158.0,
      "logps/rejected": -162.0,
      "loss": 0.6799,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1708984375,
      "rewards/margins": 0.034423828125,
      "rewards/rejected": -0.205078125,
      "step": 81
    },
    {
      "epoch": 0.3000571755288736,
      "grad_norm": 311.5542907714844,
      "learning_rate": 1.399267399267399e-06,
      "logits/chosen": -0.6875,
      "logits/rejected": -0.65234375,
      "logps/chosen": -131.0,
      "logps/rejected": -158.0,
      "loss": 0.6389,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1611328125,
      "rewards/margins": 0.140625,
      "rewards/rejected": -0.302734375,
      "step": 82
    },
    {
      "epoch": 0.30371640937678673,
      "grad_norm": 442.2575378417969,
      "learning_rate": 1.3919413919413919e-06,
      "logits/chosen": -0.6875,
      "logits/rejected": -0.625,
      "logps/chosen": -158.0,
      "logps/rejected": -195.0,
      "loss": 0.6479,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.173828125,
      "rewards/margins": 0.11279296875,
      "rewards/rejected": -0.287109375,
      "step": 83
    },
    {
      "epoch": 0.30737564322469985,
      "grad_norm": 341.3179016113281,
      "learning_rate": 1.3846153846153844e-06,
      "logits/chosen": -0.65234375,
      "logits/rejected": -0.6875,
      "logps/chosen": -144.0,
      "logps/rejected": -176.0,
      "loss": 0.6302,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.126953125,
      "rewards/margins": 0.1474609375,
      "rewards/rejected": -0.2734375,
      "step": 84
    },
    {
      "epoch": 0.3110348770726129,
      "grad_norm": 305.62109375,
      "learning_rate": 1.3772893772893772e-06,
      "logits/chosen": -0.63671875,
      "logits/rejected": -0.6328125,
      "logps/chosen": -140.0,
      "logps/rejected": -159.0,
      "loss": 0.6205,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0751953125,
      "rewards/margins": 0.158203125,
      "rewards/rejected": -0.2333984375,
      "step": 85
    },
    {
      "epoch": 0.314694110920526,
      "grad_norm": 300.09014892578125,
      "learning_rate": 1.36996336996337e-06,
      "logits/chosen": -0.59765625,
      "logits/rejected": -0.578125,
      "logps/chosen": -142.0,
      "logps/rejected": -157.0,
      "loss": 0.5966,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.038330078125,
      "rewards/margins": 0.2138671875,
      "rewards/rejected": -0.251953125,
      "step": 86
    },
    {
      "epoch": 0.31835334476843913,
      "grad_norm": 325.3841247558594,
      "learning_rate": 1.3626373626373625e-06,
      "logits/chosen": -0.66796875,
      "logits/rejected": -0.67578125,
      "logps/chosen": -164.0,
      "logps/rejected": -170.0,
      "loss": 0.6437,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.10546875,
      "rewards/margins": 0.12060546875,
      "rewards/rejected": -0.2255859375,
      "step": 87
    },
    {
      "epoch": 0.3220125786163522,
      "grad_norm": 347.8658447265625,
      "learning_rate": 1.3553113553113553e-06,
      "logits/chosen": -0.703125,
      "logits/rejected": -0.65234375,
      "logps/chosen": -154.0,
      "logps/rejected": -167.0,
      "loss": 0.6111,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.109375,
      "rewards/margins": 0.185546875,
      "rewards/rejected": -0.294921875,
      "step": 88
    },
    {
      "epoch": 0.3256718124642653,
      "grad_norm": 319.26513671875,
      "learning_rate": 1.347985347985348e-06,
      "logits/chosen": -0.6875,
      "logits/rejected": -0.66015625,
      "logps/chosen": -140.0,
      "logps/rejected": -184.0,
      "loss": 0.6299,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1279296875,
      "rewards/margins": 0.1611328125,
      "rewards/rejected": -0.2890625,
      "step": 89
    },
    {
      "epoch": 0.3293310463121784,
      "grad_norm": 326.7582702636719,
      "learning_rate": 1.3406593406593404e-06,
      "logits/chosen": -0.6640625,
      "logits/rejected": -0.67578125,
      "logps/chosen": -159.0,
      "logps/rejected": -178.0,
      "loss": 0.6315,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1376953125,
      "rewards/margins": 0.1640625,
      "rewards/rejected": -0.302734375,
      "step": 90
    },
    {
      "epoch": 0.33299028016009147,
      "grad_norm": 296.3334655761719,
      "learning_rate": 1.3333333333333332e-06,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.640625,
      "logps/chosen": -127.5,
      "logps/rejected": -174.0,
      "loss": 0.6214,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.158203125,
      "rewards/margins": 0.1767578125,
      "rewards/rejected": -0.333984375,
      "step": 91
    },
    {
      "epoch": 0.3366495140080046,
      "grad_norm": 352.0799865722656,
      "learning_rate": 1.326007326007326e-06,
      "logits/chosen": -0.640625,
      "logits/rejected": -0.62109375,
      "logps/chosen": -165.0,
      "logps/rejected": -151.0,
      "loss": 0.6456,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1826171875,
      "rewards/margins": 0.111328125,
      "rewards/rejected": -0.294921875,
      "step": 92
    },
    {
      "epoch": 0.3403087478559177,
      "grad_norm": 342.6123962402344,
      "learning_rate": 1.3186813186813185e-06,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.671875,
      "logps/chosen": -153.0,
      "logps/rejected": -198.0,
      "loss": 0.6411,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1748046875,
      "rewards/margins": 0.1318359375,
      "rewards/rejected": -0.306640625,
      "step": 93
    },
    {
      "epoch": 0.34396798170383075,
      "grad_norm": 400.05133056640625,
      "learning_rate": 1.3113553113553113e-06,
      "logits/chosen": -0.6796875,
      "logits/rejected": -0.70703125,
      "logps/chosen": -154.0,
      "logps/rejected": -185.0,
      "loss": 0.6475,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1455078125,
      "rewards/margins": 0.10595703125,
      "rewards/rejected": -0.251953125,
      "step": 94
    },
    {
      "epoch": 0.34762721555174386,
      "grad_norm": 314.9571838378906,
      "learning_rate": 1.304029304029304e-06,
      "logits/chosen": -0.6640625,
      "logits/rejected": -0.62890625,
      "logps/chosen": -160.0,
      "logps/rejected": -175.0,
      "loss": 0.6207,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.169921875,
      "rewards/margins": 0.1650390625,
      "rewards/rejected": -0.333984375,
      "step": 95
    },
    {
      "epoch": 0.3512864493996569,
      "grad_norm": 333.59881591796875,
      "learning_rate": 1.2967032967032966e-06,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.703125,
      "logps/chosen": -143.0,
      "logps/rejected": -218.0,
      "loss": 0.617,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.126953125,
      "rewards/margins": 0.1826171875,
      "rewards/rejected": -0.310546875,
      "step": 96
    },
    {
      "epoch": 0.35494568324757003,
      "grad_norm": 356.0664978027344,
      "learning_rate": 1.2893772893772894e-06,
      "logits/chosen": -0.6484375,
      "logits/rejected": -0.6875,
      "logps/chosen": -164.0,
      "logps/rejected": -185.0,
      "loss": 0.6273,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1669921875,
      "rewards/margins": 0.1865234375,
      "rewards/rejected": -0.353515625,
      "step": 97
    },
    {
      "epoch": 0.35860491709548314,
      "grad_norm": 354.0408630371094,
      "learning_rate": 1.2820512820512822e-06,
      "logits/chosen": -0.66015625,
      "logits/rejected": -0.70703125,
      "logps/chosen": -145.0,
      "logps/rejected": -165.0,
      "loss": 0.6429,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1494140625,
      "rewards/margins": 0.119140625,
      "rewards/rejected": -0.267578125,
      "step": 98
    },
    {
      "epoch": 0.3622641509433962,
      "grad_norm": 350.7596130371094,
      "learning_rate": 1.2747252747252745e-06,
      "logits/chosen": -0.6640625,
      "logits/rejected": -0.61328125,
      "logps/chosen": -186.0,
      "logps/rejected": -204.0,
      "loss": 0.5828,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.140625,
      "rewards/margins": 0.25,
      "rewards/rejected": -0.390625,
      "step": 99
    },
    {
      "epoch": 0.3659233847913093,
      "grad_norm": 314.3235778808594,
      "learning_rate": 1.2673992673992673e-06,
      "logits/chosen": -0.640625,
      "logits/rejected": -0.6171875,
      "logps/chosen": -159.0,
      "logps/rejected": -175.0,
      "loss": 0.6156,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1953125,
      "rewards/margins": 0.1845703125,
      "rewards/rejected": -0.380859375,
      "step": 100
    },
    {
      "epoch": 0.3695826186392224,
      "grad_norm": 308.3187561035156,
      "learning_rate": 1.2600732600732599e-06,
      "logits/chosen": -0.66015625,
      "logits/rejected": -0.64453125,
      "logps/chosen": -152.0,
      "logps/rejected": -206.0,
      "loss": 0.6235,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1728515625,
      "rewards/margins": 0.16015625,
      "rewards/rejected": -0.333984375,
      "step": 101
    },
    {
      "epoch": 0.3732418524871355,
      "grad_norm": 339.0077209472656,
      "learning_rate": 1.2527472527472527e-06,
      "logits/chosen": -0.671875,
      "logits/rejected": -0.6796875,
      "logps/chosen": -150.0,
      "logps/rejected": -198.0,
      "loss": 0.6145,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1669921875,
      "rewards/margins": 0.1796875,
      "rewards/rejected": -0.34765625,
      "step": 102
    },
    {
      "epoch": 0.3769010863350486,
      "grad_norm": 333.14642333984375,
      "learning_rate": 1.2454212454212454e-06,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.63671875,
      "logps/chosen": -160.0,
      "logps/rejected": -172.0,
      "loss": 0.5947,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.125,
      "rewards/margins": 0.2265625,
      "rewards/rejected": -0.3515625,
      "step": 103
    },
    {
      "epoch": 0.3805603201829617,
      "grad_norm": 327.3541564941406,
      "learning_rate": 1.238095238095238e-06,
      "logits/chosen": -0.64453125,
      "logits/rejected": -0.62890625,
      "logps/chosen": -173.0,
      "logps/rejected": -195.0,
      "loss": 0.6166,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1904296875,
      "rewards/margins": 0.1796875,
      "rewards/rejected": -0.37109375,
      "step": 104
    },
    {
      "epoch": 0.38421955403087477,
      "grad_norm": 353.1539611816406,
      "learning_rate": 1.2307692307692308e-06,
      "logits/chosen": -0.67578125,
      "logits/rejected": -0.66015625,
      "logps/chosen": -157.0,
      "logps/rejected": -188.0,
      "loss": 0.6166,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.2158203125,
      "rewards/margins": 0.189453125,
      "rewards/rejected": -0.40625,
      "step": 105
    },
    {
      "epoch": 0.3878787878787879,
      "grad_norm": 338.275390625,
      "learning_rate": 1.2234432234432235e-06,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.6875,
      "logps/chosen": -150.0,
      "logps/rejected": -162.0,
      "loss": 0.5969,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1337890625,
      "rewards/margins": 0.2177734375,
      "rewards/rejected": -0.3515625,
      "step": 106
    },
    {
      "epoch": 0.391538021726701,
      "grad_norm": 338.1829528808594,
      "learning_rate": 1.2161172161172159e-06,
      "logits/chosen": -0.65234375,
      "logits/rejected": -0.6171875,
      "logps/chosen": -157.0,
      "logps/rejected": -145.0,
      "loss": 0.645,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1943359375,
      "rewards/margins": 0.10546875,
      "rewards/rejected": -0.30078125,
      "step": 107
    },
    {
      "epoch": 0.39519725557461405,
      "grad_norm": 334.7667236328125,
      "learning_rate": 1.2087912087912087e-06,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.6640625,
      "logps/chosen": -132.0,
      "logps/rejected": -213.0,
      "loss": 0.5587,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.138671875,
      "rewards/margins": 0.314453125,
      "rewards/rejected": -0.453125,
      "step": 108
    },
    {
      "epoch": 0.39885648942252716,
      "grad_norm": 328.79302978515625,
      "learning_rate": 1.2014652014652014e-06,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.6796875,
      "logps/chosen": -180.0,
      "logps/rejected": -173.0,
      "loss": 0.6177,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.205078125,
      "rewards/margins": 0.189453125,
      "rewards/rejected": -0.39453125,
      "step": 109
    },
    {
      "epoch": 0.4025157232704403,
      "grad_norm": 397.0133056640625,
      "learning_rate": 1.194139194139194e-06,
      "logits/chosen": -0.6796875,
      "logits/rejected": -0.6640625,
      "logps/chosen": -157.0,
      "logps/rejected": -169.0,
      "loss": 0.6269,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.203125,
      "rewards/margins": 0.171875,
      "rewards/rejected": -0.375,
      "step": 110
    },
    {
      "epoch": 0.40617495711835333,
      "grad_norm": 333.8108825683594,
      "learning_rate": 1.1868131868131868e-06,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.671875,
      "logps/chosen": -176.0,
      "logps/rejected": -204.0,
      "loss": 0.6445,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.2177734375,
      "rewards/margins": 0.1220703125,
      "rewards/rejected": -0.33984375,
      "step": 111
    },
    {
      "epoch": 0.40983419096626644,
      "grad_norm": 340.60955810546875,
      "learning_rate": 1.1794871794871795e-06,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.6953125,
      "logps/chosen": -152.0,
      "logps/rejected": -188.0,
      "loss": 0.6043,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.1669921875,
      "rewards/margins": 0.212890625,
      "rewards/rejected": -0.380859375,
      "step": 112
    },
    {
      "epoch": 0.41349342481417956,
      "grad_norm": 338.4276123046875,
      "learning_rate": 1.172161172161172e-06,
      "logits/chosen": -0.69140625,
      "logits/rejected": -0.65625,
      "logps/chosen": -145.0,
      "logps/rejected": -161.0,
      "loss": 0.6066,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1865234375,
      "rewards/margins": 0.203125,
      "rewards/rejected": -0.388671875,
      "step": 113
    },
    {
      "epoch": 0.4171526586620926,
      "grad_norm": 357.9419250488281,
      "learning_rate": 1.1648351648351649e-06,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.6484375,
      "logps/chosen": -176.0,
      "logps/rejected": -196.0,
      "loss": 0.6153,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.291015625,
      "rewards/margins": 0.189453125,
      "rewards/rejected": -0.48046875,
      "step": 114
    },
    {
      "epoch": 0.4208118925100057,
      "grad_norm": 319.607177734375,
      "learning_rate": 1.1575091575091577e-06,
      "logits/chosen": -0.65234375,
      "logits/rejected": -0.625,
      "logps/chosen": -153.0,
      "logps/rejected": -165.0,
      "loss": 0.5956,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1865234375,
      "rewards/margins": 0.2109375,
      "rewards/rejected": -0.396484375,
      "step": 115
    },
    {
      "epoch": 0.42447112635791884,
      "grad_norm": 332.0836181640625,
      "learning_rate": 1.15018315018315e-06,
      "logits/chosen": -0.6953125,
      "logits/rejected": -0.69921875,
      "logps/chosen": -160.0,
      "logps/rejected": -176.0,
      "loss": 0.6059,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1767578125,
      "rewards/margins": 0.2099609375,
      "rewards/rejected": -0.38671875,
      "step": 116
    },
    {
      "epoch": 0.4281303602058319,
      "grad_norm": 327.6258544921875,
      "learning_rate": 1.1428571428571428e-06,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.734375,
      "logps/chosen": -152.0,
      "logps/rejected": -161.0,
      "loss": 0.639,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2177734375,
      "rewards/margins": 0.1357421875,
      "rewards/rejected": -0.353515625,
      "step": 117
    },
    {
      "epoch": 0.431789594053745,
      "grad_norm": 348.0113525390625,
      "learning_rate": 1.1355311355311353e-06,
      "logits/chosen": -0.62109375,
      "logits/rejected": -0.64453125,
      "logps/chosen": -180.0,
      "logps/rejected": -178.0,
      "loss": 0.6324,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.302734375,
      "rewards/margins": 0.1611328125,
      "rewards/rejected": -0.46484375,
      "step": 118
    },
    {
      "epoch": 0.43544882790165806,
      "grad_norm": 323.5909729003906,
      "learning_rate": 1.1282051282051281e-06,
      "logits/chosen": -0.65234375,
      "logits/rejected": -0.66796875,
      "logps/chosen": -171.0,
      "logps/rejected": -162.0,
      "loss": 0.6286,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.205078125,
      "rewards/margins": 0.1630859375,
      "rewards/rejected": -0.3671875,
      "step": 119
    },
    {
      "epoch": 0.4391080617495712,
      "grad_norm": 356.1521301269531,
      "learning_rate": 1.1208791208791209e-06,
      "logits/chosen": -0.671875,
      "logits/rejected": -0.69140625,
      "logps/chosen": -141.0,
      "logps/rejected": -170.0,
      "loss": 0.6358,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.224609375,
      "rewards/margins": 0.142578125,
      "rewards/rejected": -0.3671875,
      "step": 120
    },
    {
      "epoch": 0.4427672955974843,
      "grad_norm": 323.0431213378906,
      "learning_rate": 1.1135531135531134e-06,
      "logits/chosen": -0.58984375,
      "logits/rejected": -0.63671875,
      "logps/chosen": -149.0,
      "logps/rejected": -205.0,
      "loss": 0.5646,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.1533203125,
      "rewards/margins": 0.314453125,
      "rewards/rejected": -0.466796875,
      "step": 121
    },
    {
      "epoch": 0.44642652944539735,
      "grad_norm": 394.70880126953125,
      "learning_rate": 1.1062271062271062e-06,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.734375,
      "logps/chosen": -145.0,
      "logps/rejected": -171.0,
      "loss": 0.6443,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.2138671875,
      "rewards/margins": 0.1171875,
      "rewards/rejected": -0.33203125,
      "step": 122
    },
    {
      "epoch": 0.45008576329331046,
      "grad_norm": 361.07220458984375,
      "learning_rate": 1.098901098901099e-06,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.67578125,
      "logps/chosen": -174.0,
      "logps/rejected": -179.0,
      "loss": 0.6489,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.279296875,
      "rewards/margins": 0.1123046875,
      "rewards/rejected": -0.390625,
      "step": 123
    },
    {
      "epoch": 0.45374499714122357,
      "grad_norm": 378.9485778808594,
      "learning_rate": 1.0915750915750913e-06,
      "logits/chosen": -0.640625,
      "logits/rejected": -0.640625,
      "logps/chosen": -166.0,
      "logps/rejected": -181.0,
      "loss": 0.6277,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2314453125,
      "rewards/margins": 0.158203125,
      "rewards/rejected": -0.388671875,
      "step": 124
    },
    {
      "epoch": 0.45740423098913663,
      "grad_norm": 361.32470703125,
      "learning_rate": 1.0842490842490841e-06,
      "logits/chosen": -0.70703125,
      "logits/rejected": -0.65625,
      "logps/chosen": -162.0,
      "logps/rejected": -201.0,
      "loss": 0.6276,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2314453125,
      "rewards/margins": 0.1826171875,
      "rewards/rejected": -0.4140625,
      "step": 125
    },
    {
      "epoch": 0.46106346483704974,
      "grad_norm": 344.46905517578125,
      "learning_rate": 1.0769230769230769e-06,
      "logits/chosen": -0.63671875,
      "logits/rejected": -0.671875,
      "logps/chosen": -175.0,
      "logps/rejected": -169.0,
      "loss": 0.6512,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.255859375,
      "rewards/margins": 0.10400390625,
      "rewards/rejected": -0.361328125,
      "step": 126
    },
    {
      "epoch": 0.46472269868496285,
      "grad_norm": 350.5096130371094,
      "learning_rate": 1.0695970695970695e-06,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.72265625,
      "logps/chosen": -177.0,
      "logps/rejected": -169.0,
      "loss": 0.621,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2265625,
      "rewards/margins": 0.185546875,
      "rewards/rejected": -0.412109375,
      "step": 127
    },
    {
      "epoch": 0.4683819325328759,
      "grad_norm": 321.7780456542969,
      "learning_rate": 1.0622710622710622e-06,
      "logits/chosen": -0.6953125,
      "logits/rejected": -0.6640625,
      "logps/chosen": -144.0,
      "logps/rejected": -200.0,
      "loss": 0.5705,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1669921875,
      "rewards/margins": 0.3046875,
      "rewards/rejected": -0.47265625,
      "step": 128
    },
    {
      "epoch": 0.472041166380789,
      "grad_norm": 312.19964599609375,
      "learning_rate": 1.054945054945055e-06,
      "logits/chosen": -0.60546875,
      "logits/rejected": -0.58203125,
      "logps/chosen": -150.0,
      "logps/rejected": -210.0,
      "loss": 0.5723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1396484375,
      "rewards/margins": 0.3359375,
      "rewards/rejected": -0.474609375,
      "step": 129
    },
    {
      "epoch": 0.47570040022870214,
      "grad_norm": 354.0895080566406,
      "learning_rate": 1.0476190476190476e-06,
      "logits/chosen": -0.6640625,
      "logits/rejected": -0.66796875,
      "logps/chosen": -166.0,
      "logps/rejected": -170.0,
      "loss": 0.6128,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2890625,
      "rewards/margins": 0.1875,
      "rewards/rejected": -0.4765625,
      "step": 130
    },
    {
      "epoch": 0.4793596340766152,
      "grad_norm": 359.5901184082031,
      "learning_rate": 1.0402930402930403e-06,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.6953125,
      "logps/chosen": -155.0,
      "logps/rejected": -184.0,
      "loss": 0.6275,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1689453125,
      "rewards/margins": 0.1669921875,
      "rewards/rejected": -0.3359375,
      "step": 131
    },
    {
      "epoch": 0.4830188679245283,
      "grad_norm": 325.01580810546875,
      "learning_rate": 1.0329670329670331e-06,
      "logits/chosen": -0.6640625,
      "logits/rejected": -0.61328125,
      "logps/chosen": -155.0,
      "logps/rejected": -172.0,
      "loss": 0.6331,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2021484375,
      "rewards/margins": 0.169921875,
      "rewards/rejected": -0.37109375,
      "step": 132
    },
    {
      "epoch": 0.4866781017724414,
      "grad_norm": 306.410888671875,
      "learning_rate": 1.0256410256410255e-06,
      "logits/chosen": -0.66796875,
      "logits/rejected": -0.671875,
      "logps/chosen": -146.0,
      "logps/rejected": -181.0,
      "loss": 0.5833,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.1416015625,
      "rewards/margins": 0.248046875,
      "rewards/rejected": -0.388671875,
      "step": 133
    },
    {
      "epoch": 0.4903373356203545,
      "grad_norm": 330.1487121582031,
      "learning_rate": 1.0183150183150182e-06,
      "logits/chosen": -0.66015625,
      "logits/rejected": -0.68359375,
      "logps/chosen": -176.0,
      "logps/rejected": -228.0,
      "loss": 0.5701,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2099609375,
      "rewards/margins": 0.314453125,
      "rewards/rejected": -0.5234375,
      "step": 134
    },
    {
      "epoch": 0.4939965694682676,
      "grad_norm": 355.6802673339844,
      "learning_rate": 1.010989010989011e-06,
      "logits/chosen": -0.6875,
      "logits/rejected": -0.65234375,
      "logps/chosen": -166.0,
      "logps/rejected": -146.0,
      "loss": 0.6357,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.150390625,
      "rewards/margins": 0.150390625,
      "rewards/rejected": -0.30078125,
      "step": 135
    },
    {
      "epoch": 0.4976558033161807,
      "grad_norm": 325.06951904296875,
      "learning_rate": 1.0036630036630036e-06,
      "logits/chosen": -0.6796875,
      "logits/rejected": -0.609375,
      "logps/chosen": -151.0,
      "logps/rejected": -168.0,
      "loss": 0.62,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1455078125,
      "rewards/margins": 0.17578125,
      "rewards/rejected": -0.322265625,
      "step": 136
    },
    {
      "epoch": 0.5013150371640938,
      "grad_norm": 342.5461120605469,
      "learning_rate": 9.963369963369963e-07,
      "logits/chosen": -0.6328125,
      "logits/rejected": -0.625,
      "logps/chosen": -179.0,
      "logps/rejected": -178.0,
      "loss": 0.6149,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1533203125,
      "rewards/margins": 0.1826171875,
      "rewards/rejected": -0.3359375,
      "step": 137
    },
    {
      "epoch": 0.5049742710120069,
      "grad_norm": 294.18218994140625,
      "learning_rate": 9.89010989010989e-07,
      "logits/chosen": -0.671875,
      "logits/rejected": -0.640625,
      "logps/chosen": -158.0,
      "logps/rejected": -189.0,
      "loss": 0.5771,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.1455078125,
      "rewards/margins": 0.287109375,
      "rewards/rejected": -0.43359375,
      "step": 138
    },
    {
      "epoch": 0.50863350485992,
      "grad_norm": 349.0784606933594,
      "learning_rate": 9.816849816849817e-07,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.6171875,
      "logps/chosen": -141.0,
      "logps/rejected": -147.0,
      "loss": 0.6296,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08154296875,
      "rewards/margins": 0.15234375,
      "rewards/rejected": -0.2333984375,
      "step": 139
    },
    {
      "epoch": 0.5122927387078331,
      "grad_norm": 296.64013671875,
      "learning_rate": 9.743589743589742e-07,
      "logits/chosen": -0.69140625,
      "logits/rejected": -0.671875,
      "logps/chosen": -142.0,
      "logps/rejected": -157.0,
      "loss": 0.5943,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.10498046875,
      "rewards/margins": 0.224609375,
      "rewards/rejected": -0.330078125,
      "step": 140
    },
    {
      "epoch": 0.5159519725557461,
      "grad_norm": 366.21868896484375,
      "learning_rate": 9.67032967032967e-07,
      "logits/chosen": -0.66015625,
      "logits/rejected": -0.61328125,
      "logps/chosen": -158.0,
      "logps/rejected": -163.0,
      "loss": 0.6171,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1376953125,
      "rewards/margins": 0.1796875,
      "rewards/rejected": -0.318359375,
      "step": 141
    },
    {
      "epoch": 0.5196112064036592,
      "grad_norm": 356.66766357421875,
      "learning_rate": 9.597069597069598e-07,
      "logits/chosen": -0.625,
      "logits/rejected": -0.671875,
      "logps/chosen": -137.0,
      "logps/rejected": -187.0,
      "loss": 0.5703,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.16015625,
      "rewards/margins": 0.322265625,
      "rewards/rejected": -0.482421875,
      "step": 142
    },
    {
      "epoch": 0.5232704402515723,
      "grad_norm": 328.0158996582031,
      "learning_rate": 9.523809523809522e-07,
      "logits/chosen": -0.625,
      "logits/rejected": -0.64453125,
      "logps/chosen": -176.0,
      "logps/rejected": -209.0,
      "loss": 0.6207,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.244140625,
      "rewards/margins": 0.1884765625,
      "rewards/rejected": -0.43359375,
      "step": 143
    },
    {
      "epoch": 0.5269296740994854,
      "grad_norm": 343.86138916015625,
      "learning_rate": 9.45054945054945e-07,
      "logits/chosen": -0.6796875,
      "logits/rejected": -0.703125,
      "logps/chosen": -152.0,
      "logps/rejected": -203.0,
      "loss": 0.5776,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.04541015625,
      "rewards/margins": 0.291015625,
      "rewards/rejected": -0.3359375,
      "step": 144
    },
    {
      "epoch": 0.5305889079473985,
      "grad_norm": 339.378662109375,
      "learning_rate": 9.377289377289377e-07,
      "logits/chosen": -0.64453125,
      "logits/rejected": -0.61328125,
      "logps/chosen": -133.0,
      "logps/rejected": -168.0,
      "loss": 0.5869,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0908203125,
      "rewards/margins": 0.2451171875,
      "rewards/rejected": -0.3359375,
      "step": 145
    },
    {
      "epoch": 0.5342481417953117,
      "grad_norm": 357.1183166503906,
      "learning_rate": 9.304029304029304e-07,
      "logits/chosen": -0.72265625,
      "logits/rejected": -0.71484375,
      "logps/chosen": -152.0,
      "logps/rejected": -175.0,
      "loss": 0.6175,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.134765625,
      "rewards/margins": 0.197265625,
      "rewards/rejected": -0.33203125,
      "step": 146
    },
    {
      "epoch": 0.5379073756432247,
      "grad_norm": 315.3673095703125,
      "learning_rate": 9.230769230769231e-07,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.70703125,
      "logps/chosen": -151.0,
      "logps/rejected": -154.0,
      "loss": 0.5938,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.08837890625,
      "rewards/margins": 0.228515625,
      "rewards/rejected": -0.31640625,
      "step": 147
    },
    {
      "epoch": 0.5415666094911378,
      "grad_norm": 378.76580810546875,
      "learning_rate": 9.157509157509157e-07,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.66015625,
      "logps/chosen": -163.0,
      "logps/rejected": -184.0,
      "loss": 0.581,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.087890625,
      "rewards/margins": 0.263671875,
      "rewards/rejected": -0.3515625,
      "step": 148
    },
    {
      "epoch": 0.5452258433390509,
      "grad_norm": 315.32427978515625,
      "learning_rate": 9.084249084249084e-07,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.6875,
      "logps/chosen": -160.0,
      "logps/rejected": -180.0,
      "loss": 0.615,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.12060546875,
      "rewards/margins": 0.1826171875,
      "rewards/rejected": -0.302734375,
      "step": 149
    },
    {
      "epoch": 0.548885077186964,
      "grad_norm": 302.259521484375,
      "learning_rate": 9.010989010989011e-07,
      "logits/chosen": -0.62890625,
      "logits/rejected": -0.64453125,
      "logps/chosen": -147.0,
      "logps/rejected": -189.0,
      "loss": 0.5753,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0634765625,
      "rewards/margins": 0.291015625,
      "rewards/rejected": -0.35546875,
      "step": 150
    },
    {
      "epoch": 0.5525443110348771,
      "grad_norm": 327.5895690917969,
      "learning_rate": 8.937728937728938e-07,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.68359375,
      "logps/chosen": -168.0,
      "logps/rejected": -167.0,
      "loss": 0.5899,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 0.23828125,
      "rewards/rejected": -0.302734375,
      "step": 151
    },
    {
      "epoch": 0.5562035448827901,
      "grad_norm": 323.4364929199219,
      "learning_rate": 8.864468864468864e-07,
      "logits/chosen": -0.66796875,
      "logits/rejected": -0.6796875,
      "logps/chosen": -150.0,
      "logps/rejected": -155.0,
      "loss": 0.6307,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07958984375,
      "rewards/margins": 0.1435546875,
      "rewards/rejected": -0.2236328125,
      "step": 152
    },
    {
      "epoch": 0.5598627787307032,
      "grad_norm": 307.4286804199219,
      "learning_rate": 8.79120879120879e-07,
      "logits/chosen": -0.6796875,
      "logits/rejected": -0.65625,
      "logps/chosen": -136.0,
      "logps/rejected": -162.0,
      "loss": 0.5807,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09375,
      "rewards/margins": 0.26953125,
      "rewards/rejected": -0.36328125,
      "step": 153
    },
    {
      "epoch": 0.5635220125786163,
      "grad_norm": 307.41644287109375,
      "learning_rate": 8.717948717948718e-07,
      "logits/chosen": -0.71484375,
      "logits/rejected": -0.62890625,
      "logps/chosen": -136.0,
      "logps/rejected": -173.0,
      "loss": 0.5708,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0546875,
      "rewards/margins": 0.283203125,
      "rewards/rejected": -0.337890625,
      "step": 154
    },
    {
      "epoch": 0.5671812464265295,
      "grad_norm": 329.59661865234375,
      "learning_rate": 8.644688644688645e-07,
      "logits/chosen": -0.64453125,
      "logits/rejected": -0.66796875,
      "logps/chosen": -143.0,
      "logps/rejected": -196.0,
      "loss": 0.6014,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1669921875,
      "rewards/margins": 0.2255859375,
      "rewards/rejected": -0.392578125,
      "step": 155
    },
    {
      "epoch": 0.5708404802744426,
      "grad_norm": 330.7326354980469,
      "learning_rate": 8.57142857142857e-07,
      "logits/chosen": -0.5859375,
      "logits/rejected": -0.609375,
      "logps/chosen": -154.0,
      "logps/rejected": -199.0,
      "loss": 0.5914,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09130859375,
      "rewards/margins": 0.265625,
      "rewards/rejected": -0.357421875,
      "step": 156
    },
    {
      "epoch": 0.5744997141223557,
      "grad_norm": 335.531494140625,
      "learning_rate": 8.498168498168498e-07,
      "logits/chosen": -0.671875,
      "logits/rejected": -0.65234375,
      "logps/chosen": -145.0,
      "logps/rejected": -178.0,
      "loss": 0.59,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.018798828125,
      "rewards/margins": 0.259765625,
      "rewards/rejected": -0.27734375,
      "step": 157
    },
    {
      "epoch": 0.5781589479702687,
      "grad_norm": 324.5040588378906,
      "learning_rate": 8.424908424908425e-07,
      "logits/chosen": -0.71875,
      "logits/rejected": -0.68359375,
      "logps/chosen": -159.0,
      "logps/rejected": -193.0,
      "loss": 0.575,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.03125,
      "rewards/margins": 0.279296875,
      "rewards/rejected": -0.310546875,
      "step": 158
    },
    {
      "epoch": 0.5818181818181818,
      "grad_norm": 370.49615478515625,
      "learning_rate": 8.351648351648351e-07,
      "logits/chosen": -0.71484375,
      "logits/rejected": -0.69140625,
      "logps/chosen": -158.0,
      "logps/rejected": -142.0,
      "loss": 0.6056,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0830078125,
      "rewards/margins": 0.2099609375,
      "rewards/rejected": -0.29296875,
      "step": 159
    },
    {
      "epoch": 0.5854774156660949,
      "grad_norm": 320.1568603515625,
      "learning_rate": 8.278388278388277e-07,
      "logits/chosen": -0.7109375,
      "logits/rejected": -0.6953125,
      "logps/chosen": -161.0,
      "logps/rejected": -146.0,
      "loss": 0.6121,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.053955078125,
      "rewards/margins": 0.1904296875,
      "rewards/rejected": -0.244140625,
      "step": 160
    },
    {
      "epoch": 0.589136649514008,
      "grad_norm": 313.9174499511719,
      "learning_rate": 8.205128205128205e-07,
      "logits/chosen": -0.62109375,
      "logits/rejected": -0.64453125,
      "logps/chosen": -153.0,
      "logps/rejected": -178.0,
      "loss": 0.6049,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08984375,
      "rewards/margins": 0.2197265625,
      "rewards/rejected": -0.30859375,
      "step": 161
    },
    {
      "epoch": 0.5927958833619211,
      "grad_norm": 373.2075500488281,
      "learning_rate": 8.131868131868131e-07,
      "logits/chosen": -0.66015625,
      "logits/rejected": -0.6953125,
      "logps/chosen": -159.0,
      "logps/rejected": -190.0,
      "loss": 0.5909,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.03759765625,
      "rewards/margins": 0.2412109375,
      "rewards/rejected": -0.279296875,
      "step": 162
    },
    {
      "epoch": 0.5964551172098342,
      "grad_norm": 335.1282958984375,
      "learning_rate": 8.058608058608058e-07,
      "logits/chosen": -0.671875,
      "logits/rejected": -0.6328125,
      "logps/chosen": -164.0,
      "logps/rejected": -181.0,
      "loss": 0.5697,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.04541015625,
      "rewards/margins": 0.296875,
      "rewards/rejected": -0.341796875,
      "step": 163
    },
    {
      "epoch": 0.6001143510577472,
      "grad_norm": 335.8233642578125,
      "learning_rate": 7.985347985347986e-07,
      "logits/chosen": -0.7421875,
      "logits/rejected": -0.6796875,
      "logps/chosen": -146.0,
      "logps/rejected": -187.0,
      "loss": 0.5668,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.02197265625,
      "rewards/margins": 0.306640625,
      "rewards/rejected": -0.28515625,
      "step": 164
    },
    {
      "epoch": 0.6037735849056604,
      "grad_norm": 321.95721435546875,
      "learning_rate": 7.912087912087911e-07,
      "logits/chosen": -0.64453125,
      "logits/rejected": -0.6328125,
      "logps/chosen": -156.0,
      "logps/rejected": -180.0,
      "loss": 0.5832,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.08984375,
      "rewards/margins": 0.2890625,
      "rewards/rejected": -0.380859375,
      "step": 165
    },
    {
      "epoch": 0.6074328187535735,
      "grad_norm": 311.0093078613281,
      "learning_rate": 7.838827838827838e-07,
      "logits/chosen": -0.71875,
      "logits/rejected": -0.69140625,
      "logps/chosen": -143.0,
      "logps/rejected": -170.0,
      "loss": 0.5898,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07373046875,
      "rewards/margins": 0.244140625,
      "rewards/rejected": -0.318359375,
      "step": 166
    },
    {
      "epoch": 0.6110920526014866,
      "grad_norm": 370.56341552734375,
      "learning_rate": 7.765567765567766e-07,
      "logits/chosen": -0.66015625,
      "logits/rejected": -0.6640625,
      "logps/chosen": -162.0,
      "logps/rejected": -193.0,
      "loss": 0.6026,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.12060546875,
      "rewards/margins": 0.2216796875,
      "rewards/rejected": -0.341796875,
      "step": 167
    },
    {
      "epoch": 0.6147512864493997,
      "grad_norm": 338.77239990234375,
      "learning_rate": 7.692307692307693e-07,
      "logits/chosen": -0.7109375,
      "logits/rejected": -0.69921875,
      "logps/chosen": -149.0,
      "logps/rejected": -163.0,
      "loss": 0.5883,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.032958984375,
      "rewards/margins": 0.2412109375,
      "rewards/rejected": -0.2734375,
      "step": 168
    },
    {
      "epoch": 0.6184105202973128,
      "grad_norm": 408.2104187011719,
      "learning_rate": 7.619047619047618e-07,
      "logits/chosen": -0.66015625,
      "logits/rejected": -0.65234375,
      "logps/chosen": -187.0,
      "logps/rejected": -174.0,
      "loss": 0.6561,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1357421875,
      "rewards/margins": 0.0947265625,
      "rewards/rejected": -0.23046875,
      "step": 169
    },
    {
      "epoch": 0.6220697541452258,
      "grad_norm": 321.3171691894531,
      "learning_rate": 7.545787545787545e-07,
      "logits/chosen": -0.7421875,
      "logits/rejected": -0.70703125,
      "logps/chosen": -164.0,
      "logps/rejected": -182.0,
      "loss": 0.5742,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0281982421875,
      "rewards/margins": 0.296875,
      "rewards/rejected": -0.32421875,
      "step": 170
    },
    {
      "epoch": 0.6257289879931389,
      "grad_norm": 327.4810485839844,
      "learning_rate": 7.472527472527473e-07,
      "logits/chosen": -0.640625,
      "logits/rejected": -0.66796875,
      "logps/chosen": -151.0,
      "logps/rejected": -160.0,
      "loss": 0.6061,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07666015625,
      "rewards/margins": 0.2021484375,
      "rewards/rejected": -0.279296875,
      "step": 171
    },
    {
      "epoch": 0.629388221841052,
      "grad_norm": 357.8804016113281,
      "learning_rate": 7.399267399267399e-07,
      "logits/chosen": -0.734375,
      "logits/rejected": -0.703125,
      "logps/chosen": -154.0,
      "logps/rejected": -172.0,
      "loss": 0.5742,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.050048828125,
      "rewards/margins": 0.28515625,
      "rewards/rejected": -0.333984375,
      "step": 172
    },
    {
      "epoch": 0.6330474556889651,
      "grad_norm": 320.7285461425781,
      "learning_rate": 7.326007326007325e-07,
      "logits/chosen": -0.67578125,
      "logits/rejected": -0.6875,
      "logps/chosen": -161.0,
      "logps/rejected": -184.0,
      "loss": 0.564,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0361328125,
      "rewards/margins": 0.30859375,
      "rewards/rejected": -0.345703125,
      "step": 173
    },
    {
      "epoch": 0.6367066895368783,
      "grad_norm": 321.6654968261719,
      "learning_rate": 7.252747252747253e-07,
      "logits/chosen": -0.69140625,
      "logits/rejected": -0.69921875,
      "logps/chosen": -145.0,
      "logps/rejected": -153.0,
      "loss": 0.6049,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.038330078125,
      "rewards/margins": 0.22265625,
      "rewards/rejected": -0.26171875,
      "step": 174
    },
    {
      "epoch": 0.6403659233847913,
      "grad_norm": 354.5701599121094,
      "learning_rate": 7.179487179487179e-07,
      "logits/chosen": -0.6953125,
      "logits/rejected": -0.71875,
      "logps/chosen": -180.0,
      "logps/rejected": -174.0,
      "loss": 0.5979,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.09521484375,
      "rewards/margins": 0.2265625,
      "rewards/rejected": -0.322265625,
      "step": 175
    },
    {
      "epoch": 0.6440251572327044,
      "grad_norm": 333.7059020996094,
      "learning_rate": 7.106227106227106e-07,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.68359375,
      "logps/chosen": -157.0,
      "logps/rejected": -198.0,
      "loss": 0.5878,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.09228515625,
      "rewards/margins": 0.24609375,
      "rewards/rejected": -0.337890625,
      "step": 176
    },
    {
      "epoch": 0.6476843910806175,
      "grad_norm": 325.05035400390625,
      "learning_rate": 7.032967032967034e-07,
      "logits/chosen": -0.67578125,
      "logits/rejected": -0.671875,
      "logps/chosen": -146.0,
      "logps/rejected": -186.0,
      "loss": 0.5739,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0008392333984375,
      "rewards/margins": 0.3046875,
      "rewards/rejected": -0.306640625,
      "step": 177
    },
    {
      "epoch": 0.6513436249285306,
      "grad_norm": 352.8323974609375,
      "learning_rate": 6.959706959706959e-07,
      "logits/chosen": -0.6640625,
      "logits/rejected": -0.640625,
      "logps/chosen": -179.0,
      "logps/rejected": -218.0,
      "loss": 0.5719,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.10009765625,
      "rewards/margins": 0.314453125,
      "rewards/rejected": -0.4140625,
      "step": 178
    },
    {
      "epoch": 0.6550028587764437,
      "grad_norm": 307.6293029785156,
      "learning_rate": 6.886446886446886e-07,
      "logits/chosen": -0.6328125,
      "logits/rejected": -0.66796875,
      "logps/chosen": -146.0,
      "logps/rejected": -172.0,
      "loss": 0.5661,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.046142578125,
      "rewards/margins": 0.302734375,
      "rewards/rejected": -0.349609375,
      "step": 179
    },
    {
      "epoch": 0.6586620926243568,
      "grad_norm": 356.95587158203125,
      "learning_rate": 6.813186813186813e-07,
      "logits/chosen": -0.66015625,
      "logits/rejected": -0.67578125,
      "logps/chosen": -149.0,
      "logps/rejected": -164.0,
      "loss": 0.6138,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10009765625,
      "rewards/margins": 0.1865234375,
      "rewards/rejected": -0.287109375,
      "step": 180
    },
    {
      "epoch": 0.6623213264722698,
      "grad_norm": 290.56317138671875,
      "learning_rate": 6.73992673992674e-07,
      "logits/chosen": -0.70703125,
      "logits/rejected": -0.671875,
      "logps/chosen": -136.0,
      "logps/rejected": -171.0,
      "loss": 0.5764,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.031982421875,
      "rewards/margins": 0.29296875,
      "rewards/rejected": -0.32421875,
      "step": 181
    },
    {
      "epoch": 0.6659805603201829,
      "grad_norm": 313.93927001953125,
      "learning_rate": 6.666666666666666e-07,
      "logits/chosen": -0.66796875,
      "logits/rejected": -0.64453125,
      "logps/chosen": -145.0,
      "logps/rejected": -164.0,
      "loss": 0.5668,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0015411376953125,
      "rewards/margins": 0.287109375,
      "rewards/rejected": -0.287109375,
      "step": 182
    },
    {
      "epoch": 0.669639794168096,
      "grad_norm": 315.1253967285156,
      "learning_rate": 6.593406593406593e-07,
      "logits/chosen": -0.734375,
      "logits/rejected": -0.71875,
      "logps/chosen": -125.5,
      "logps/rejected": -232.0,
      "loss": 0.5599,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0186767578125,
      "rewards/margins": 0.3515625,
      "rewards/rejected": -0.333984375,
      "step": 183
    },
    {
      "epoch": 0.6732990280160092,
      "grad_norm": 326.492431640625,
      "learning_rate": 6.52014652014652e-07,
      "logits/chosen": -0.62890625,
      "logits/rejected": -0.61328125,
      "logps/chosen": -162.0,
      "logps/rejected": -222.0,
      "loss": 0.5437,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.030517578125,
      "rewards/margins": 0.36328125,
      "rewards/rejected": -0.392578125,
      "step": 184
    },
    {
      "epoch": 0.6769582618639223,
      "grad_norm": 348.7772521972656,
      "learning_rate": 6.446886446886447e-07,
      "logits/chosen": -0.6640625,
      "logits/rejected": -0.6484375,
      "logps/chosen": -167.0,
      "logps/rejected": -177.0,
      "loss": 0.5911,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0947265625,
      "rewards/margins": 0.2431640625,
      "rewards/rejected": -0.337890625,
      "step": 185
    },
    {
      "epoch": 0.6806174957118354,
      "grad_norm": 336.41827392578125,
      "learning_rate": 6.373626373626373e-07,
      "logits/chosen": -0.6875,
      "logits/rejected": -0.64453125,
      "logps/chosen": -150.0,
      "logps/rejected": -175.0,
      "loss": 0.5616,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.01165771484375,
      "rewards/margins": 0.298828125,
      "rewards/rejected": -0.310546875,
      "step": 186
    },
    {
      "epoch": 0.6842767295597484,
      "grad_norm": 328.49530029296875,
      "learning_rate": 6.300366300366299e-07,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.70703125,
      "logps/chosen": -146.0,
      "logps/rejected": -151.0,
      "loss": 0.5895,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.047607421875,
      "rewards/margins": 0.232421875,
      "rewards/rejected": -0.279296875,
      "step": 187
    },
    {
      "epoch": 0.6879359634076615,
      "grad_norm": 351.6430969238281,
      "learning_rate": 6.227106227106227e-07,
      "logits/chosen": -0.65234375,
      "logits/rejected": -0.671875,
      "logps/chosen": -148.0,
      "logps/rejected": -218.0,
      "loss": 0.5731,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1025390625,
      "rewards/margins": 0.30859375,
      "rewards/rejected": -0.41015625,
      "step": 188
    },
    {
      "epoch": 0.6915951972555746,
      "grad_norm": 318.9908752441406,
      "learning_rate": 6.153846153846154e-07,
      "logits/chosen": -0.73828125,
      "logits/rejected": -0.6640625,
      "logps/chosen": -145.0,
      "logps/rejected": -169.0,
      "loss": 0.5258,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.52587890625e-05,
      "rewards/margins": 0.390625,
      "rewards/rejected": -0.388671875,
      "step": 189
    },
    {
      "epoch": 0.6952544311034877,
      "grad_norm": 342.9971618652344,
      "learning_rate": 6.080586080586079e-07,
      "logits/chosen": -0.70703125,
      "logits/rejected": -0.69921875,
      "logps/chosen": -157.0,
      "logps/rejected": -189.0,
      "loss": 0.5966,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07666015625,
      "rewards/margins": 0.234375,
      "rewards/rejected": -0.310546875,
      "step": 190
    },
    {
      "epoch": 0.6989136649514008,
      "grad_norm": 309.0221862792969,
      "learning_rate": 6.007326007326007e-07,
      "logits/chosen": -0.6484375,
      "logits/rejected": -0.64453125,
      "logps/chosen": -146.0,
      "logps/rejected": -201.0,
      "loss": 0.5492,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.04931640625,
      "rewards/margins": 0.37109375,
      "rewards/rejected": -0.419921875,
      "step": 191
    },
    {
      "epoch": 0.7025728987993138,
      "grad_norm": 384.8427429199219,
      "learning_rate": 5.934065934065934e-07,
      "logits/chosen": -0.6875,
      "logits/rejected": -0.671875,
      "logps/chosen": -182.0,
      "logps/rejected": -179.0,
      "loss": 0.6272,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.12353515625,
      "rewards/margins": 0.1796875,
      "rewards/rejected": -0.302734375,
      "step": 192
    },
    {
      "epoch": 0.706232132647227,
      "grad_norm": 345.20538330078125,
      "learning_rate": 5.86080586080586e-07,
      "logits/chosen": -0.6875,
      "logits/rejected": -0.6875,
      "logps/chosen": -152.0,
      "logps/rejected": -150.0,
      "loss": 0.6308,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08447265625,
      "rewards/margins": 0.1513671875,
      "rewards/rejected": -0.236328125,
      "step": 193
    },
    {
      "epoch": 0.7098913664951401,
      "grad_norm": 373.9530334472656,
      "learning_rate": 5.787545787545788e-07,
      "logits/chosen": -0.71484375,
      "logits/rejected": -0.7265625,
      "logps/chosen": -174.0,
      "logps/rejected": -163.0,
      "loss": 0.619,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.09228515625,
      "rewards/margins": 0.1767578125,
      "rewards/rejected": -0.26953125,
      "step": 194
    },
    {
      "epoch": 0.7135506003430532,
      "grad_norm": 378.72406005859375,
      "learning_rate": 5.714285714285714e-07,
      "logits/chosen": -0.7109375,
      "logits/rejected": -0.7109375,
      "logps/chosen": -160.0,
      "logps/rejected": -201.0,
      "loss": 0.5354,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0311279296875,
      "rewards/margins": 0.380859375,
      "rewards/rejected": -0.412109375,
      "step": 195
    },
    {
      "epoch": 0.7172098341909663,
      "grad_norm": 326.7547912597656,
      "learning_rate": 5.641025641025641e-07,
      "logits/chosen": -0.67578125,
      "logits/rejected": -0.68359375,
      "logps/chosen": -152.0,
      "logps/rejected": -164.0,
      "loss": 0.594,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.04541015625,
      "rewards/margins": 0.2392578125,
      "rewards/rejected": -0.28515625,
      "step": 196
    },
    {
      "epoch": 0.7208690680388794,
      "grad_norm": 332.29193115234375,
      "learning_rate": 5.567765567765567e-07,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.71875,
      "logps/chosen": -152.0,
      "logps/rejected": -191.0,
      "loss": 0.5926,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0703125,
      "rewards/margins": 0.232421875,
      "rewards/rejected": -0.302734375,
      "step": 197
    },
    {
      "epoch": 0.7245283018867924,
      "grad_norm": 345.8672180175781,
      "learning_rate": 5.494505494505495e-07,
      "logits/chosen": -0.71484375,
      "logits/rejected": -0.671875,
      "logps/chosen": -153.0,
      "logps/rejected": -190.0,
      "loss": 0.5678,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07177734375,
      "rewards/margins": 0.310546875,
      "rewards/rejected": -0.3828125,
      "step": 198
    },
    {
      "epoch": 0.7281875357347055,
      "grad_norm": 418.1891174316406,
      "learning_rate": 5.421245421245421e-07,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.68359375,
      "logps/chosen": -167.0,
      "logps/rejected": -182.0,
      "loss": 0.5964,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10302734375,
      "rewards/margins": 0.2392578125,
      "rewards/rejected": -0.341796875,
      "step": 199
    },
    {
      "epoch": 0.7318467695826186,
      "grad_norm": 364.3031921386719,
      "learning_rate": 5.347985347985347e-07,
      "logits/chosen": -0.65234375,
      "logits/rejected": -0.671875,
      "logps/chosen": -186.0,
      "logps/rejected": -180.0,
      "loss": 0.5733,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.058837890625,
      "rewards/margins": 0.28515625,
      "rewards/rejected": -0.34375,
      "step": 200
    },
    {
      "epoch": 0.7355060034305317,
      "grad_norm": 344.48663330078125,
      "learning_rate": 5.274725274725275e-07,
      "logits/chosen": -0.76171875,
      "logits/rejected": -0.765625,
      "logps/chosen": -141.0,
      "logps/rejected": -156.0,
      "loss": 0.571,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.03125,
      "rewards/margins": 0.275390625,
      "rewards/rejected": -0.306640625,
      "step": 201
    },
    {
      "epoch": 0.7391652372784449,
      "grad_norm": 341.8359069824219,
      "learning_rate": 5.201465201465202e-07,
      "logits/chosen": -0.66796875,
      "logits/rejected": -0.69140625,
      "logps/chosen": -152.0,
      "logps/rejected": -169.0,
      "loss": 0.5828,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.109375,
      "rewards/margins": 0.27734375,
      "rewards/rejected": -0.38671875,
      "step": 202
    },
    {
      "epoch": 0.742824471126358,
      "grad_norm": 331.3113708496094,
      "learning_rate": 5.128205128205127e-07,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.64453125,
      "logps/chosen": -160.0,
      "logps/rejected": -174.0,
      "loss": 0.5509,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0211181640625,
      "rewards/margins": 0.328125,
      "rewards/rejected": -0.349609375,
      "step": 203
    },
    {
      "epoch": 0.746483704974271,
      "grad_norm": 331.06451416015625,
      "learning_rate": 5.054945054945055e-07,
      "logits/chosen": -0.640625,
      "logits/rejected": -0.64453125,
      "logps/chosen": -146.0,
      "logps/rejected": -145.0,
      "loss": 0.5928,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.050048828125,
      "rewards/margins": 0.2333984375,
      "rewards/rejected": -0.283203125,
      "step": 204
    },
    {
      "epoch": 0.7501429388221841,
      "grad_norm": 386.7975769042969,
      "learning_rate": 4.981684981684982e-07,
      "logits/chosen": -0.640625,
      "logits/rejected": -0.6328125,
      "logps/chosen": -131.0,
      "logps/rejected": -168.0,
      "loss": 0.5466,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.050048828125,
      "rewards/margins": 0.34765625,
      "rewards/rejected": -0.396484375,
      "step": 205
    },
    {
      "epoch": 0.7538021726700972,
      "grad_norm": 773.7296142578125,
      "learning_rate": 4.908424908424908e-07,
      "logits/chosen": -0.69140625,
      "logits/rejected": -0.71875,
      "logps/chosen": -143.0,
      "logps/rejected": -178.0,
      "loss": 0.6085,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1611328125,
      "rewards/margins": 0.2138671875,
      "rewards/rejected": -0.375,
      "step": 206
    },
    {
      "epoch": 0.7574614065180103,
      "grad_norm": 327.77874755859375,
      "learning_rate": 4.835164835164835e-07,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.66015625,
      "logps/chosen": -168.0,
      "logps/rejected": -157.0,
      "loss": 0.5521,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.040771484375,
      "rewards/margins": 0.318359375,
      "rewards/rejected": -0.359375,
      "step": 207
    },
    {
      "epoch": 0.7611206403659234,
      "grad_norm": 380.7069396972656,
      "learning_rate": 4.761904761904761e-07,
      "logits/chosen": -0.7265625,
      "logits/rejected": -0.6796875,
      "logps/chosen": -148.0,
      "logps/rejected": -183.0,
      "loss": 0.5838,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.04541015625,
      "rewards/margins": 0.2578125,
      "rewards/rejected": -0.302734375,
      "step": 208
    },
    {
      "epoch": 0.7647798742138365,
      "grad_norm": 362.50927734375,
      "learning_rate": 4.6886446886446884e-07,
      "logits/chosen": -0.73828125,
      "logits/rejected": -0.72265625,
      "logps/chosen": -177.0,
      "logps/rejected": -156.0,
      "loss": 0.5818,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0517578125,
      "rewards/margins": 0.2490234375,
      "rewards/rejected": -0.30078125,
      "step": 209
    },
    {
      "epoch": 0.7684391080617495,
      "grad_norm": 374.5245666503906,
      "learning_rate": 4.6153846153846156e-07,
      "logits/chosen": -0.66796875,
      "logits/rejected": -0.6953125,
      "logps/chosen": -140.0,
      "logps/rejected": -198.0,
      "loss": 0.6289,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.14453125,
      "rewards/margins": 0.193359375,
      "rewards/rejected": -0.337890625,
      "step": 210
    },
    {
      "epoch": 0.7720983419096626,
      "grad_norm": 341.1881103515625,
      "learning_rate": 4.542124542124542e-07,
      "logits/chosen": -0.70703125,
      "logits/rejected": -0.68359375,
      "logps/chosen": -143.0,
      "logps/rejected": -178.0,
      "loss": 0.576,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0203857421875,
      "rewards/margins": 0.279296875,
      "rewards/rejected": -0.30078125,
      "step": 211
    },
    {
      "epoch": 0.7757575757575758,
      "grad_norm": 337.7765808105469,
      "learning_rate": 4.468864468864469e-07,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.6875,
      "logps/chosen": -135.0,
      "logps/rejected": -186.0,
      "loss": 0.5106,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.005462646484375,
      "rewards/margins": 0.44140625,
      "rewards/rejected": -0.435546875,
      "step": 212
    },
    {
      "epoch": 0.7794168096054889,
      "grad_norm": 355.8410339355469,
      "learning_rate": 4.395604395604395e-07,
      "logits/chosen": -0.6796875,
      "logits/rejected": -0.68359375,
      "logps/chosen": -158.0,
      "logps/rejected": -233.0,
      "loss": 0.5145,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.04541015625,
      "rewards/margins": 0.44140625,
      "rewards/rejected": -0.48828125,
      "step": 213
    },
    {
      "epoch": 0.783076043453402,
      "grad_norm": 336.0704345703125,
      "learning_rate": 4.3223443223443223e-07,
      "logits/chosen": -0.71484375,
      "logits/rejected": -0.6953125,
      "logps/chosen": -152.0,
      "logps/rejected": -166.0,
      "loss": 0.5754,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0517578125,
      "rewards/margins": 0.2890625,
      "rewards/rejected": -0.341796875,
      "step": 214
    },
    {
      "epoch": 0.786735277301315,
      "grad_norm": 361.47052001953125,
      "learning_rate": 4.249084249084249e-07,
      "logits/chosen": -0.7265625,
      "logits/rejected": -0.73046875,
      "logps/chosen": -148.0,
      "logps/rejected": -169.0,
      "loss": 0.5704,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.043701171875,
      "rewards/margins": 0.298828125,
      "rewards/rejected": -0.341796875,
      "step": 215
    },
    {
      "epoch": 0.7903945111492281,
      "grad_norm": 323.4903259277344,
      "learning_rate": 4.1758241758241757e-07,
      "logits/chosen": -0.6953125,
      "logits/rejected": -0.73046875,
      "logps/chosen": -171.0,
      "logps/rejected": -186.0,
      "loss": 0.5572,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.095703125,
      "rewards/margins": 0.3359375,
      "rewards/rejected": -0.431640625,
      "step": 216
    },
    {
      "epoch": 0.7940537449971412,
      "grad_norm": 334.9060363769531,
      "learning_rate": 4.1025641025641024e-07,
      "logits/chosen": -0.63671875,
      "logits/rejected": -0.66796875,
      "logps/chosen": -174.0,
      "logps/rejected": -170.0,
      "loss": 0.59,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05078125,
      "rewards/margins": 0.2421875,
      "rewards/rejected": -0.29296875,
      "step": 217
    },
    {
      "epoch": 0.7977129788450543,
      "grad_norm": 318.1783142089844,
      "learning_rate": 4.029304029304029e-07,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.72265625,
      "logps/chosen": -146.0,
      "logps/rejected": -157.0,
      "loss": 0.5676,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08056640625,
      "rewards/margins": 0.2890625,
      "rewards/rejected": -0.369140625,
      "step": 218
    },
    {
      "epoch": 0.8013722126929674,
      "grad_norm": 361.3813171386719,
      "learning_rate": 3.9560439560439557e-07,
      "logits/chosen": -0.69140625,
      "logits/rejected": -0.69921875,
      "logps/chosen": -168.0,
      "logps/rejected": -171.0,
      "loss": 0.5864,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1064453125,
      "rewards/margins": 0.23828125,
      "rewards/rejected": -0.345703125,
      "step": 219
    },
    {
      "epoch": 0.8050314465408805,
      "grad_norm": 366.4966735839844,
      "learning_rate": 3.882783882783883e-07,
      "logits/chosen": -0.69140625,
      "logits/rejected": -0.69921875,
      "logps/chosen": -167.0,
      "logps/rejected": -196.0,
      "loss": 0.5481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12451171875,
      "rewards/margins": 0.416015625,
      "rewards/rejected": -0.5390625,
      "step": 220
    },
    {
      "epoch": 0.8086906803887935,
      "grad_norm": 337.0361022949219,
      "learning_rate": 3.809523809523809e-07,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.69140625,
      "logps/chosen": -155.0,
      "logps/rejected": -188.0,
      "loss": 0.5627,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.072265625,
      "rewards/margins": 0.322265625,
      "rewards/rejected": -0.39453125,
      "step": 221
    },
    {
      "epoch": 0.8123499142367067,
      "grad_norm": 323.75201416015625,
      "learning_rate": 3.7362637362637363e-07,
      "logits/chosen": -0.7421875,
      "logits/rejected": -0.6796875,
      "logps/chosen": -150.0,
      "logps/rejected": -199.0,
      "loss": 0.5634,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09521484375,
      "rewards/margins": 0.37109375,
      "rewards/rejected": -0.466796875,
      "step": 222
    },
    {
      "epoch": 0.8160091480846198,
      "grad_norm": 367.0987243652344,
      "learning_rate": 3.6630036630036624e-07,
      "logits/chosen": -0.67578125,
      "logits/rejected": -0.67578125,
      "logps/chosen": -168.0,
      "logps/rejected": -156.0,
      "loss": 0.5681,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04052734375,
      "rewards/margins": 0.294921875,
      "rewards/rejected": -0.3359375,
      "step": 223
    },
    {
      "epoch": 0.8196683819325329,
      "grad_norm": 362.4571838378906,
      "learning_rate": 3.5897435897435896e-07,
      "logits/chosen": -0.70703125,
      "logits/rejected": -0.73046875,
      "logps/chosen": -142.0,
      "logps/rejected": -204.0,
      "loss": 0.5521,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07373046875,
      "rewards/margins": 0.34765625,
      "rewards/rejected": -0.421875,
      "step": 224
    },
    {
      "epoch": 0.823327615780446,
      "grad_norm": 340.5346374511719,
      "learning_rate": 3.516483516483517e-07,
      "logits/chosen": -0.76953125,
      "logits/rejected": -0.6875,
      "logps/chosen": -173.0,
      "logps/rejected": -190.0,
      "loss": 0.5436,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0250244140625,
      "rewards/margins": 0.37890625,
      "rewards/rejected": -0.40234375,
      "step": 225
    },
    {
      "epoch": 0.8269868496283591,
      "grad_norm": 369.2457580566406,
      "learning_rate": 3.443223443223443e-07,
      "logits/chosen": -0.69140625,
      "logits/rejected": -0.70703125,
      "logps/chosen": -153.0,
      "logps/rejected": -188.0,
      "loss": 0.6034,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1328125,
      "rewards/margins": 0.2265625,
      "rewards/rejected": -0.359375,
      "step": 226
    },
    {
      "epoch": 0.8306460834762721,
      "grad_norm": 373.647216796875,
      "learning_rate": 3.36996336996337e-07,
      "logits/chosen": -0.734375,
      "logits/rejected": -0.75390625,
      "logps/chosen": -176.0,
      "logps/rejected": -161.0,
      "loss": 0.5767,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0751953125,
      "rewards/margins": 0.283203125,
      "rewards/rejected": -0.357421875,
      "step": 227
    },
    {
      "epoch": 0.8343053173241852,
      "grad_norm": 391.55511474609375,
      "learning_rate": 3.2967032967032963e-07,
      "logits/chosen": -0.7109375,
      "logits/rejected": -0.6640625,
      "logps/chosen": -167.0,
      "logps/rejected": -149.0,
      "loss": 0.5791,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0947265625,
      "rewards/margins": 0.265625,
      "rewards/rejected": -0.361328125,
      "step": 228
    },
    {
      "epoch": 0.8379645511720983,
      "grad_norm": 340.9716796875,
      "learning_rate": 3.2234432234432236e-07,
      "logits/chosen": -0.74609375,
      "logits/rejected": -0.70703125,
      "logps/chosen": -140.0,
      "logps/rejected": -163.0,
      "loss": 0.5575,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06103515625,
      "rewards/margins": 0.322265625,
      "rewards/rejected": -0.3828125,
      "step": 229
    },
    {
      "epoch": 0.8416237850200115,
      "grad_norm": 334.7230529785156,
      "learning_rate": 3.1501831501831497e-07,
      "logits/chosen": -0.7265625,
      "logits/rejected": -0.734375,
      "logps/chosen": -148.0,
      "logps/rejected": -168.0,
      "loss": 0.6023,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06640625,
      "rewards/margins": 0.22265625,
      "rewards/rejected": -0.2890625,
      "step": 230
    },
    {
      "epoch": 0.8452830188679246,
      "grad_norm": 351.0285339355469,
      "learning_rate": 3.076923076923077e-07,
      "logits/chosen": -0.7265625,
      "logits/rejected": -0.69140625,
      "logps/chosen": -151.0,
      "logps/rejected": -160.0,
      "loss": 0.5653,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07666015625,
      "rewards/margins": 0.3046875,
      "rewards/rejected": -0.380859375,
      "step": 231
    },
    {
      "epoch": 0.8489422527158377,
      "grad_norm": 340.0621337890625,
      "learning_rate": 3.0036630036630036e-07,
      "logits/chosen": -0.75390625,
      "logits/rejected": -0.7421875,
      "logps/chosen": -145.0,
      "logps/rejected": -157.0,
      "loss": 0.5784,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.015625,
      "rewards/margins": 0.26953125,
      "rewards/rejected": -0.28515625,
      "step": 232
    },
    {
      "epoch": 0.8526014865637507,
      "grad_norm": 373.56488037109375,
      "learning_rate": 2.93040293040293e-07,
      "logits/chosen": -0.71875,
      "logits/rejected": -0.703125,
      "logps/chosen": -156.0,
      "logps/rejected": -218.0,
      "loss": 0.5762,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.12451171875,
      "rewards/margins": 0.275390625,
      "rewards/rejected": -0.400390625,
      "step": 233
    },
    {
      "epoch": 0.8562607204116638,
      "grad_norm": 340.0142517089844,
      "learning_rate": 2.857142857142857e-07,
      "logits/chosen": -0.671875,
      "logits/rejected": -0.71875,
      "logps/chosen": -149.0,
      "logps/rejected": -139.0,
      "loss": 0.6108,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09375,
      "rewards/margins": 0.18359375,
      "rewards/rejected": -0.27734375,
      "step": 234
    },
    {
      "epoch": 0.8599199542595769,
      "grad_norm": 363.7709655761719,
      "learning_rate": 2.7838827838827836e-07,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.7109375,
      "logps/chosen": -168.0,
      "logps/rejected": -213.0,
      "loss": 0.551,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0859375,
      "rewards/margins": 0.353515625,
      "rewards/rejected": -0.439453125,
      "step": 235
    },
    {
      "epoch": 0.86357918810749,
      "grad_norm": 354.7901916503906,
      "learning_rate": 2.7106227106227103e-07,
      "logits/chosen": -0.66796875,
      "logits/rejected": -0.671875,
      "logps/chosen": -137.0,
      "logps/rejected": -155.0,
      "loss": 0.6004,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.126953125,
      "rewards/margins": 0.2265625,
      "rewards/rejected": -0.353515625,
      "step": 236
    },
    {
      "epoch": 0.8672384219554031,
      "grad_norm": 331.5390930175781,
      "learning_rate": 2.6373626373626375e-07,
      "logits/chosen": -0.69140625,
      "logits/rejected": -0.6875,
      "logps/chosen": -164.0,
      "logps/rejected": -212.0,
      "loss": 0.5416,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0791015625,
      "rewards/margins": 0.4140625,
      "rewards/rejected": -0.4921875,
      "step": 237
    },
    {
      "epoch": 0.8708976558033161,
      "grad_norm": 316.5717468261719,
      "learning_rate": 2.5641025641025636e-07,
      "logits/chosen": -0.71875,
      "logits/rejected": -0.74609375,
      "logps/chosen": -136.0,
      "logps/rejected": -169.0,
      "loss": 0.5723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.0517578125e-05,
      "rewards/margins": 0.294921875,
      "rewards/rejected": -0.294921875,
      "step": 238
    },
    {
      "epoch": 0.8745568896512292,
      "grad_norm": 317.3512268066406,
      "learning_rate": 2.490842490842491e-07,
      "logits/chosen": -0.69140625,
      "logits/rejected": -0.671875,
      "logps/chosen": -180.0,
      "logps/rejected": -165.0,
      "loss": 0.5302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.006256103515625,
      "rewards/margins": 0.388671875,
      "rewards/rejected": -0.39453125,
      "step": 239
    },
    {
      "epoch": 0.8782161234991424,
      "grad_norm": 365.77197265625,
      "learning_rate": 2.4175824175824175e-07,
      "logits/chosen": -0.71875,
      "logits/rejected": -0.67578125,
      "logps/chosen": -191.0,
      "logps/rejected": -222.0,
      "loss": 0.5382,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06884765625,
      "rewards/margins": 0.390625,
      "rewards/rejected": -0.458984375,
      "step": 240
    },
    {
      "epoch": 0.8818753573470555,
      "grad_norm": 344.7721252441406,
      "learning_rate": 2.3443223443223442e-07,
      "logits/chosen": -0.6875,
      "logits/rejected": -0.6640625,
      "logps/chosen": -142.0,
      "logps/rejected": -182.0,
      "loss": 0.5768,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.10400390625,
      "rewards/margins": 0.271484375,
      "rewards/rejected": -0.375,
      "step": 241
    },
    {
      "epoch": 0.8855345911949686,
      "grad_norm": 343.99896240234375,
      "learning_rate": 2.271062271062271e-07,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.69140625,
      "logps/chosen": -167.0,
      "logps/rejected": -187.0,
      "loss": 0.5477,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0908203125,
      "rewards/margins": 0.345703125,
      "rewards/rejected": -0.435546875,
      "step": 242
    },
    {
      "epoch": 0.8891938250428817,
      "grad_norm": 351.04913330078125,
      "learning_rate": 2.1978021978021976e-07,
      "logits/chosen": -0.69140625,
      "logits/rejected": -0.67578125,
      "logps/chosen": -154.0,
      "logps/rejected": -162.0,
      "loss": 0.5964,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.01806640625,
      "rewards/margins": 0.2431640625,
      "rewards/rejected": -0.26171875,
      "step": 243
    },
    {
      "epoch": 0.8928530588907947,
      "grad_norm": 331.8519287109375,
      "learning_rate": 2.1245421245421245e-07,
      "logits/chosen": -0.734375,
      "logits/rejected": -0.73046875,
      "logps/chosen": -137.0,
      "logps/rejected": -184.0,
      "loss": 0.5541,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0140380859375,
      "rewards/margins": 0.33984375,
      "rewards/rejected": -0.353515625,
      "step": 244
    },
    {
      "epoch": 0.8965122927387078,
      "grad_norm": 343.590087890625,
      "learning_rate": 2.0512820512820512e-07,
      "logits/chosen": -0.66796875,
      "logits/rejected": -0.65625,
      "logps/chosen": -147.0,
      "logps/rejected": -196.0,
      "loss": 0.5594,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.00628662109375,
      "rewards/margins": 0.314453125,
      "rewards/rejected": -0.30859375,
      "step": 245
    },
    {
      "epoch": 0.9001715265866209,
      "grad_norm": 400.9288635253906,
      "learning_rate": 1.9780219780219779e-07,
      "logits/chosen": -0.70703125,
      "logits/rejected": -0.6953125,
      "logps/chosen": -150.0,
      "logps/rejected": -184.0,
      "loss": 0.5757,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06103515625,
      "rewards/margins": 0.291015625,
      "rewards/rejected": -0.3515625,
      "step": 246
    },
    {
      "epoch": 0.903830760434534,
      "grad_norm": 346.36065673828125,
      "learning_rate": 1.9047619047619045e-07,
      "logits/chosen": -0.69140625,
      "logits/rejected": -0.67578125,
      "logps/chosen": -153.0,
      "logps/rejected": -198.0,
      "loss": 0.5306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.045166015625,
      "rewards/margins": 0.380859375,
      "rewards/rejected": -0.42578125,
      "step": 247
    },
    {
      "epoch": 0.9074899942824471,
      "grad_norm": 312.3040466308594,
      "learning_rate": 1.8315018315018312e-07,
      "logits/chosen": -0.6875,
      "logits/rejected": -0.67578125,
      "logps/chosen": -149.0,
      "logps/rejected": -218.0,
      "loss": 0.5116,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.048583984375,
      "rewards/margins": 0.48828125,
      "rewards/rejected": -0.53515625,
      "step": 248
    },
    {
      "epoch": 0.9111492281303603,
      "grad_norm": 320.7344665527344,
      "learning_rate": 1.7582417582417584e-07,
      "logits/chosen": -0.67578125,
      "logits/rejected": -0.6796875,
      "logps/chosen": -138.0,
      "logps/rejected": -164.0,
      "loss": 0.5797,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.035888671875,
      "rewards/margins": 0.267578125,
      "rewards/rejected": -0.302734375,
      "step": 249
    },
    {
      "epoch": 0.9148084619782733,
      "grad_norm": 331.97637939453125,
      "learning_rate": 1.684981684981685e-07,
      "logits/chosen": -0.74609375,
      "logits/rejected": -0.6953125,
      "logps/chosen": -158.0,
      "logps/rejected": -200.0,
      "loss": 0.5593,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.04541015625,
      "rewards/margins": 0.333984375,
      "rewards/rejected": -0.380859375,
      "step": 250
    },
    {
      "epoch": 0.9184676958261864,
      "grad_norm": 334.0833435058594,
      "learning_rate": 1.6117216117216118e-07,
      "logits/chosen": -0.73046875,
      "logits/rejected": -0.70703125,
      "logps/chosen": -153.0,
      "logps/rejected": -223.0,
      "loss": 0.5211,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0133056640625,
      "rewards/margins": 0.40625,
      "rewards/rejected": -0.419921875,
      "step": 251
    },
    {
      "epoch": 0.9221269296740995,
      "grad_norm": 315.7561950683594,
      "learning_rate": 1.5384615384615385e-07,
      "logits/chosen": -0.70703125,
      "logits/rejected": -0.69921875,
      "logps/chosen": -163.0,
      "logps/rejected": -171.0,
      "loss": 0.5767,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.01483154296875,
      "rewards/margins": 0.27734375,
      "rewards/rejected": -0.291015625,
      "step": 252
    },
    {
      "epoch": 0.9257861635220126,
      "grad_norm": 340.6386413574219,
      "learning_rate": 1.465201465201465e-07,
      "logits/chosen": -0.65234375,
      "logits/rejected": -0.69140625,
      "logps/chosen": -142.0,
      "logps/rejected": -180.0,
      "loss": 0.5369,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.051025390625,
      "rewards/margins": 0.380859375,
      "rewards/rejected": -0.431640625,
      "step": 253
    },
    {
      "epoch": 0.9294453973699257,
      "grad_norm": 325.98187255859375,
      "learning_rate": 1.3919413919413918e-07,
      "logits/chosen": -0.6875,
      "logits/rejected": -0.63671875,
      "logps/chosen": -150.0,
      "logps/rejected": -147.0,
      "loss": 0.5456,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0250244140625,
      "rewards/margins": 0.34375,
      "rewards/rejected": -0.369140625,
      "step": 254
    },
    {
      "epoch": 0.9331046312178388,
      "grad_norm": 355.2104187011719,
      "learning_rate": 1.3186813186813187e-07,
      "logits/chosen": -0.671875,
      "logits/rejected": -0.6953125,
      "logps/chosen": -154.0,
      "logps/rejected": -208.0,
      "loss": 0.5472,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05078125,
      "rewards/margins": 0.404296875,
      "rewards/rejected": -0.455078125,
      "step": 255
    },
    {
      "epoch": 0.9367638650657518,
      "grad_norm": 366.7920227050781,
      "learning_rate": 1.2454212454212454e-07,
      "logits/chosen": -0.703125,
      "logits/rejected": -0.7109375,
      "logps/chosen": -161.0,
      "logps/rejected": -159.0,
      "loss": 0.5634,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.043701171875,
      "rewards/margins": 0.318359375,
      "rewards/rejected": -0.361328125,
      "step": 256
    },
    {
      "epoch": 0.9404230989136649,
      "grad_norm": 345.1733093261719,
      "learning_rate": 1.1721611721611721e-07,
      "logits/chosen": -0.70703125,
      "logits/rejected": -0.703125,
      "logps/chosen": -182.0,
      "logps/rejected": -167.0,
      "loss": 0.5864,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10498046875,
      "rewards/margins": 0.255859375,
      "rewards/rejected": -0.361328125,
      "step": 257
    },
    {
      "epoch": 0.944082332761578,
      "grad_norm": 342.8348083496094,
      "learning_rate": 1.0989010989010988e-07,
      "logits/chosen": -0.69921875,
      "logits/rejected": -0.71875,
      "logps/chosen": -165.0,
      "logps/rejected": -150.0,
      "loss": 0.5539,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.00311279296875,
      "rewards/margins": 0.328125,
      "rewards/rejected": -0.330078125,
      "step": 258
    },
    {
      "epoch": 0.9477415666094912,
      "grad_norm": 317.3765563964844,
      "learning_rate": 1.0256410256410256e-07,
      "logits/chosen": -0.75,
      "logits/rejected": -0.71875,
      "logps/chosen": -133.0,
      "logps/rejected": -194.0,
      "loss": 0.5248,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.00946044921875,
      "rewards/margins": 0.41796875,
      "rewards/rejected": -0.427734375,
      "step": 259
    },
    {
      "epoch": 0.9514008004574043,
      "grad_norm": 334.01129150390625,
      "learning_rate": 9.523809523809523e-08,
      "logits/chosen": -0.7109375,
      "logits/rejected": -0.68359375,
      "logps/chosen": -142.0,
      "logps/rejected": -180.0,
      "loss": 0.5478,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.003936767578125,
      "rewards/margins": 0.353515625,
      "rewards/rejected": -0.349609375,
      "step": 260
    },
    {
      "epoch": 0.9550600343053173,
      "grad_norm": 384.39617919921875,
      "learning_rate": 8.791208791208792e-08,
      "logits/chosen": -0.671875,
      "logits/rejected": -0.62890625,
      "logps/chosen": -160.0,
      "logps/rejected": -151.0,
      "loss": 0.5869,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.078125,
      "rewards/margins": 0.2392578125,
      "rewards/rejected": -0.318359375,
      "step": 261
    },
    {
      "epoch": 0.9587192681532304,
      "grad_norm": 355.11517333984375,
      "learning_rate": 8.058608058608059e-08,
      "logits/chosen": -0.7421875,
      "logits/rejected": -0.7734375,
      "logps/chosen": -159.0,
      "logps/rejected": -186.0,
      "loss": 0.5905,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1220703125,
      "rewards/margins": 0.2314453125,
      "rewards/rejected": -0.353515625,
      "step": 262
    },
    {
      "epoch": 0.9623785020011435,
      "grad_norm": 361.0973815917969,
      "learning_rate": 7.326007326007326e-08,
      "logits/chosen": -0.70703125,
      "logits/rejected": -0.6875,
      "logps/chosen": -151.0,
      "logps/rejected": -163.0,
      "loss": 0.5654,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0859375,
      "rewards/margins": 0.30078125,
      "rewards/rejected": -0.38671875,
      "step": 263
    },
    {
      "epoch": 0.9660377358490566,
      "grad_norm": 341.8106994628906,
      "learning_rate": 6.593406593406594e-08,
      "logits/chosen": -0.6640625,
      "logits/rejected": -0.6796875,
      "logps/chosen": -155.0,
      "logps/rejected": -181.0,
      "loss": 0.567,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.078125,
      "rewards/margins": 0.291015625,
      "rewards/rejected": -0.369140625,
      "step": 264
    },
    {
      "epoch": 0.9696969696969697,
      "grad_norm": 306.0340576171875,
      "learning_rate": 5.8608058608058605e-08,
      "logits/chosen": -0.765625,
      "logits/rejected": -0.71484375,
      "logps/chosen": -132.0,
      "logps/rejected": -165.0,
      "loss": 0.5439,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.052490234375,
      "rewards/margins": 0.359375,
      "rewards/rejected": -0.41015625,
      "step": 265
    },
    {
      "epoch": 0.9733562035448828,
      "grad_norm": 353.2430114746094,
      "learning_rate": 5.128205128205128e-08,
      "logits/chosen": -0.7109375,
      "logits/rejected": -0.69140625,
      "logps/chosen": -181.0,
      "logps/rejected": -171.0,
      "loss": 0.6016,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.033447265625,
      "rewards/margins": 0.22265625,
      "rewards/rejected": -0.255859375,
      "step": 266
    },
    {
      "epoch": 0.9770154373927958,
      "grad_norm": 316.0684509277344,
      "learning_rate": 4.395604395604396e-08,
      "logits/chosen": -0.71484375,
      "logits/rejected": -0.71875,
      "logps/chosen": -137.0,
      "logps/rejected": -163.0,
      "loss": 0.551,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.031982421875,
      "rewards/margins": 0.330078125,
      "rewards/rejected": -0.36328125,
      "step": 267
    },
    {
      "epoch": 0.980674671240709,
      "grad_norm": 334.58050537109375,
      "learning_rate": 3.663003663003663e-08,
      "logits/chosen": -0.703125,
      "logits/rejected": -0.66015625,
      "logps/chosen": -165.0,
      "logps/rejected": -170.0,
      "loss": 0.6031,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.1416015625,
      "rewards/margins": 0.2060546875,
      "rewards/rejected": -0.34765625,
      "step": 268
    },
    {
      "epoch": 0.9843339050886221,
      "grad_norm": 328.41644287109375,
      "learning_rate": 2.9304029304029303e-08,
      "logits/chosen": -0.6640625,
      "logits/rejected": -0.66796875,
      "logps/chosen": -143.0,
      "logps/rejected": -156.0,
      "loss": 0.579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 0.26171875,
      "rewards/rejected": -0.330078125,
      "step": 269
    },
    {
      "epoch": 0.9879931389365352,
      "grad_norm": 307.48974609375,
      "learning_rate": 2.197802197802198e-08,
      "logits/chosen": -0.65625,
      "logits/rejected": -0.68359375,
      "logps/chosen": -145.0,
      "logps/rejected": -164.0,
      "loss": 0.5715,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0546875,
      "rewards/margins": 0.283203125,
      "rewards/rejected": -0.337890625,
      "step": 270
    },
    {
      "epoch": 0.9916523727844483,
      "grad_norm": 323.255126953125,
      "learning_rate": 1.4652014652014651e-08,
      "logits/chosen": -0.73828125,
      "logits/rejected": -0.69140625,
      "logps/chosen": -135.0,
      "logps/rejected": -147.0,
      "loss": 0.5377,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.01092529296875,
      "rewards/margins": 0.36328125,
      "rewards/rejected": -0.375,
      "step": 271
    },
    {
      "epoch": 0.9953116066323614,
      "grad_norm": 365.11749267578125,
      "learning_rate": 7.326007326007326e-09,
      "logits/chosen": -0.74609375,
      "logits/rejected": -0.734375,
      "logps/chosen": -157.0,
      "logps/rejected": -194.0,
      "loss": 0.5576,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07275390625,
      "rewards/margins": 0.3203125,
      "rewards/rejected": -0.39453125,
      "step": 272
    },
    {
      "epoch": 0.9989708404802744,
      "grad_norm": 335.74774169921875,
      "learning_rate": 0.0,
      "logits/chosen": -0.71875,
      "logits/rejected": -0.74609375,
      "logps/chosen": -155.0,
      "logps/rejected": -183.0,
      "loss": 0.5493,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0296630859375,
      "rewards/margins": 0.353515625,
      "rewards/rejected": -0.3828125,
      "step": 273
    }
  ],
  "logging_steps": 1,
  "max_steps": 273,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}