{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9985185185185185,
  "eval_steps": 500,
  "global_step": 337,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002962962962962963,
      "grad_norm": 314.5073547363281,
      "learning_rate": 1.9940652818991096e-06,
      "logits/chosen": -0.341796875,
      "logits/rejected": -0.35546875,
      "logps/chosen": -118.0,
      "logps/rejected": -159.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.005925925925925926,
      "grad_norm": 291.4620056152344,
      "learning_rate": 1.9881305637982194e-06,
      "logits/chosen": -0.388671875,
      "logits/rejected": -0.396484375,
      "logps/chosen": -130.0,
      "logps/rejected": -110.0,
      "loss": 0.6969,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.000782012939453125,
      "rewards/margins": -0.003143310546875,
      "rewards/rejected": 0.002349853515625,
      "step": 2
    },
    {
      "epoch": 0.008888888888888889,
      "grad_norm": 289.64862060546875,
      "learning_rate": 1.9821958456973295e-06,
      "logits/chosen": -0.361328125,
      "logits/rejected": -0.369140625,
      "logps/chosen": -116.0,
      "logps/rejected": -114.0,
      "loss": 0.6938,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.0133056640625,
      "rewards/margins": -0.001556396484375,
      "rewards/rejected": -0.01171875,
      "step": 3
    },
    {
      "epoch": 0.011851851851851851,
      "grad_norm": 295.88763427734375,
      "learning_rate": 1.9762611275964393e-06,
      "logits/chosen": -0.359375,
      "logits/rejected": -0.3671875,
      "logps/chosen": -108.5,
      "logps/rejected": -129.0,
      "loss": 0.6779,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.0172119140625,
      "rewards/margins": 0.03515625,
      "rewards/rejected": -0.0179443359375,
      "step": 4
    },
    {
      "epoch": 0.014814814814814815,
      "grad_norm": 268.0253601074219,
      "learning_rate": 1.970326409495549e-06,
      "logits/chosen": -0.330078125,
      "logits/rejected": -0.3671875,
      "logps/chosen": -108.0,
      "logps/rejected": -111.5,
      "loss": 0.6913,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006256103515625,
      "rewards/margins": 0.0054931640625,
      "rewards/rejected": 0.00078582763671875,
      "step": 5
    },
    {
      "epoch": 0.017777777777777778,
      "grad_norm": 262.7245178222656,
      "learning_rate": 1.9643916913946587e-06,
      "logits/chosen": -0.3671875,
      "logits/rejected": -0.384765625,
      "logps/chosen": -95.5,
      "logps/rejected": -104.5,
      "loss": 0.6852,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0133056640625,
      "rewards/margins": 0.016357421875,
      "rewards/rejected": -0.0031280517578125,
      "step": 6
    },
    {
      "epoch": 0.02074074074074074,
      "grad_norm": 292.5845642089844,
      "learning_rate": 1.9584569732937684e-06,
      "logits/chosen": -0.427734375,
      "logits/rejected": -0.427734375,
      "logps/chosen": -116.0,
      "logps/rejected": -127.0,
      "loss": 0.6946,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01019287109375,
      "rewards/margins": -0.0031280517578125,
      "rewards/rejected": -0.007049560546875,
      "step": 7
    },
    {
      "epoch": 0.023703703703703703,
      "grad_norm": 273.1530456542969,
      "learning_rate": 1.952522255192878e-06,
      "logits/chosen": -0.3828125,
      "logits/rejected": -0.3671875,
      "logps/chosen": -121.5,
      "logps/rejected": -116.0,
      "loss": 0.6863,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.01019287109375,
      "rewards/margins": 0.0179443359375,
      "rewards/rejected": -0.0078125,
      "step": 8
    },
    {
      "epoch": 0.02666666666666667,
      "grad_norm": 308.3035888671875,
      "learning_rate": 1.946587537091988e-06,
      "logits/chosen": -0.396484375,
      "logits/rejected": -0.384765625,
      "logps/chosen": -131.0,
      "logps/rejected": -114.5,
      "loss": 0.6702,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.042236328125,
      "rewards/margins": 0.04296875,
      "rewards/rejected": -0.000762939453125,
      "step": 9
    },
    {
      "epoch": 0.02962962962962963,
      "grad_norm": 295.3406982421875,
      "learning_rate": 1.9406528189910976e-06,
      "logits/chosen": -0.384765625,
      "logits/rejected": -0.390625,
      "logps/chosen": -120.5,
      "logps/rejected": -118.5,
      "loss": 0.6799,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0179443359375,
      "rewards/margins": 0.0296630859375,
      "rewards/rejected": -0.01171875,
      "step": 10
    },
    {
      "epoch": 0.03259259259259259,
      "grad_norm": 260.79620361328125,
      "learning_rate": 1.934718100890208e-06,
      "logits/chosen": -0.3671875,
      "logits/rejected": -0.3984375,
      "logps/chosen": -99.5,
      "logps/rejected": -83.5,
      "loss": 0.6849,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0172119140625,
      "rewards/margins": 0.0191650390625,
      "rewards/rejected": -0.001953125,
      "step": 11
    },
    {
      "epoch": 0.035555555555555556,
      "grad_norm": 284.2510986328125,
      "learning_rate": 1.9287833827893175e-06,
      "logits/chosen": -0.337890625,
      "logits/rejected": -0.353515625,
      "logps/chosen": -114.5,
      "logps/rejected": -105.0,
      "loss": 0.6869,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.030517578125,
      "rewards/margins": 0.0164794921875,
      "rewards/rejected": 0.0140380859375,
      "step": 12
    },
    {
      "epoch": 0.03851851851851852,
      "grad_norm": 240.7752685546875,
      "learning_rate": 1.9228486646884273e-06,
      "logits/chosen": -0.357421875,
      "logits/rejected": -0.404296875,
      "logps/chosen": -91.0,
      "logps/rejected": -98.0,
      "loss": 0.687,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.0179443359375,
      "rewards/margins": 0.01446533203125,
      "rewards/rejected": 0.003509521484375,
      "step": 13
    },
    {
      "epoch": 0.04148148148148148,
      "grad_norm": 349.65118408203125,
      "learning_rate": 1.916913946587537e-06,
      "logits/chosen": -0.35546875,
      "logits/rejected": -0.337890625,
      "logps/chosen": -105.0,
      "logps/rejected": -158.0,
      "loss": 0.7042,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.01409912109375,
      "rewards/margins": -0.01806640625,
      "rewards/rejected": 0.031982421875,
      "step": 14
    },
    {
      "epoch": 0.044444444444444446,
      "grad_norm": 274.92315673828125,
      "learning_rate": 1.9109792284866467e-06,
      "logits/chosen": -0.37890625,
      "logits/rejected": -0.36328125,
      "logps/chosen": -114.0,
      "logps/rejected": -107.5,
      "loss": 0.6887,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0234375,
      "rewards/margins": 0.01409912109375,
      "rewards/rejected": 0.0093994140625,
      "step": 15
    },
    {
      "epoch": 0.047407407407407405,
      "grad_norm": 284.9162292480469,
      "learning_rate": 1.9050445103857567e-06,
      "logits/chosen": -0.369140625,
      "logits/rejected": -0.419921875,
      "logps/chosen": -101.5,
      "logps/rejected": -113.0,
      "loss": 0.6893,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.039794921875,
      "rewards/margins": 0.00860595703125,
      "rewards/rejected": 0.03125,
      "step": 16
    },
    {
      "epoch": 0.05037037037037037,
      "grad_norm": 295.5323486328125,
      "learning_rate": 1.8991097922848664e-06,
      "logits/chosen": -0.41796875,
      "logits/rejected": -0.408203125,
      "logps/chosen": -124.0,
      "logps/rejected": -167.0,
      "loss": 0.6849,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0242919921875,
      "rewards/margins": 0.0257568359375,
      "rewards/rejected": -0.001556396484375,
      "step": 17
    },
    {
      "epoch": 0.05333333333333334,
      "grad_norm": 357.27410888671875,
      "learning_rate": 1.8931750741839763e-06,
      "logits/chosen": -0.40625,
      "logits/rejected": -0.400390625,
      "logps/chosen": -136.0,
      "logps/rejected": -220.0,
      "loss": 0.6925,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0289306640625,
      "rewards/margins": 0.0031280517578125,
      "rewards/rejected": 0.0257568359375,
      "step": 18
    },
    {
      "epoch": 0.056296296296296296,
      "grad_norm": 286.5829772949219,
      "learning_rate": 1.887240356083086e-06,
      "logits/chosen": -0.43359375,
      "logits/rejected": -0.439453125,
      "logps/chosen": -110.5,
      "logps/rejected": -138.0,
      "loss": 0.6929,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": 0.043701171875,
      "rewards/margins": 0.0078125,
      "rewards/rejected": 0.035888671875,
      "step": 19
    },
    {
      "epoch": 0.05925925925925926,
      "grad_norm": 502.36614990234375,
      "learning_rate": 1.8813056379821956e-06,
      "logits/chosen": -0.392578125,
      "logits/rejected": -0.361328125,
      "logps/chosen": -105.5,
      "logps/rejected": -97.5,
      "loss": 0.6826,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0172119140625,
      "rewards/margins": 0.0289306640625,
      "rewards/rejected": -0.01165771484375,
      "step": 20
    },
    {
      "epoch": 0.06222222222222222,
      "grad_norm": 258.4698486328125,
      "learning_rate": 1.8753709198813055e-06,
      "logits/chosen": -0.33203125,
      "logits/rejected": -0.36328125,
      "logps/chosen": -94.0,
      "logps/rejected": -98.5,
      "loss": 0.6742,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.05859375,
      "rewards/margins": 0.038330078125,
      "rewards/rejected": 0.0203857421875,
      "step": 21
    },
    {
      "epoch": 0.06518518518518518,
      "grad_norm": 282.2925109863281,
      "learning_rate": 1.8694362017804153e-06,
      "logits/chosen": -0.390625,
      "logits/rejected": -0.37109375,
      "logps/chosen": -125.0,
      "logps/rejected": -108.5,
      "loss": 0.6752,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.04541015625,
      "rewards/margins": 0.038330078125,
      "rewards/rejected": 0.00701904296875,
      "step": 22
    },
    {
      "epoch": 0.06814814814814815,
      "grad_norm": 291.8224182128906,
      "learning_rate": 1.863501483679525e-06,
      "logits/chosen": -0.3828125,
      "logits/rejected": -0.388671875,
      "logps/chosen": -111.0,
      "logps/rejected": -108.5,
      "loss": 0.6808,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.06396484375,
      "rewards/margins": 0.0281982421875,
      "rewards/rejected": 0.035888671875,
      "step": 23
    },
    {
      "epoch": 0.07111111111111111,
      "grad_norm": 258.92022705078125,
      "learning_rate": 1.857566765578635e-06,
      "logits/chosen": -0.369140625,
      "logits/rejected": -0.38671875,
      "logps/chosen": -93.5,
      "logps/rejected": -97.5,
      "loss": 0.6665,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.076171875,
      "rewards/margins": 0.058349609375,
      "rewards/rejected": 0.0179443359375,
      "step": 24
    },
    {
      "epoch": 0.07407407407407407,
      "grad_norm": 281.2393798828125,
      "learning_rate": 1.8516320474777447e-06,
      "logits/chosen": -0.37109375,
      "logits/rejected": -0.369140625,
      "logps/chosen": -106.5,
      "logps/rejected": -115.0,
      "loss": 0.6932,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.03125,
      "rewards/margins": 0.00628662109375,
      "rewards/rejected": 0.0250244140625,
      "step": 25
    },
    {
      "epoch": 0.07703703703703704,
      "grad_norm": 279.8067626953125,
      "learning_rate": 1.8456973293768546e-06,
      "logits/chosen": -0.369140625,
      "logits/rejected": -0.36328125,
      "logps/chosen": -120.5,
      "logps/rejected": -113.5,
      "loss": 0.6689,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.061767578125,
      "rewards/margins": 0.049072265625,
      "rewards/rejected": 0.01287841796875,
      "step": 26
    },
    {
      "epoch": 0.08,
      "grad_norm": 293.7424621582031,
      "learning_rate": 1.8397626112759643e-06,
      "logits/chosen": -0.396484375,
      "logits/rejected": -0.416015625,
      "logps/chosen": -131.0,
      "logps/rejected": -135.0,
      "loss": 0.6686,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07763671875,
      "rewards/margins": 0.052490234375,
      "rewards/rejected": 0.0250244140625,
      "step": 27
    },
    {
      "epoch": 0.08296296296296296,
      "grad_norm": 287.2871398925781,
      "learning_rate": 1.833827893175074e-06,
      "logits/chosen": -0.375,
      "logits/rejected": -0.408203125,
      "logps/chosen": -113.0,
      "logps/rejected": -150.0,
      "loss": 0.6848,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.053955078125,
      "rewards/margins": 0.02978515625,
      "rewards/rejected": 0.0242919921875,
      "step": 28
    },
    {
      "epoch": 0.08592592592592592,
      "grad_norm": 275.70904541015625,
      "learning_rate": 1.827893175074184e-06,
      "logits/chosen": -0.3671875,
      "logits/rejected": -0.36328125,
      "logps/chosen": -108.0,
      "logps/rejected": -106.5,
      "loss": 0.6874,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.047607421875,
      "rewards/margins": 0.014892578125,
      "rewards/rejected": 0.032958984375,
      "step": 29
    },
    {
      "epoch": 0.08888888888888889,
      "grad_norm": 275.59912109375,
      "learning_rate": 1.8219584569732937e-06,
      "logits/chosen": -0.341796875,
      "logits/rejected": -0.326171875,
      "logps/chosen": -102.0,
      "logps/rejected": -109.0,
      "loss": 0.6935,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.03759765625,
      "rewards/margins": -0.0022735595703125,
      "rewards/rejected": 0.039794921875,
      "step": 30
    },
    {
      "epoch": 0.09185185185185185,
      "grad_norm": 288.207275390625,
      "learning_rate": 1.8160237388724035e-06,
      "logits/chosen": -0.357421875,
      "logits/rejected": -0.328125,
      "logps/chosen": -129.0,
      "logps/rejected": -109.0,
      "loss": 0.6677,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0791015625,
      "rewards/margins": 0.05712890625,
      "rewards/rejected": 0.0218505859375,
      "step": 31
    },
    {
      "epoch": 0.09481481481481481,
      "grad_norm": 290.9600524902344,
      "learning_rate": 1.8100890207715134e-06,
      "logits/chosen": -0.37109375,
      "logits/rejected": -0.380859375,
      "logps/chosen": -124.0,
      "logps/rejected": -112.5,
      "loss": 0.6808,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.035888671875,
      "rewards/margins": 0.034423828125,
      "rewards/rejected": 0.001556396484375,
      "step": 32
    },
    {
      "epoch": 0.09777777777777778,
      "grad_norm": 274.994384765625,
      "learning_rate": 1.804154302670623e-06,
      "logits/chosen": -0.384765625,
      "logits/rejected": -0.419921875,
      "logps/chosen": -108.0,
      "logps/rejected": -108.0,
      "loss": 0.6702,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.046875,
      "rewards/margins": 0.04736328125,
      "rewards/rejected": -0.0003910064697265625,
      "step": 33
    },
    {
      "epoch": 0.10074074074074074,
      "grad_norm": 302.385009765625,
      "learning_rate": 1.7982195845697327e-06,
      "logits/chosen": -0.361328125,
      "logits/rejected": -0.3828125,
      "logps/chosen": -111.5,
      "logps/rejected": -133.0,
      "loss": 0.666,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.06640625,
      "rewards/margins": 0.060302734375,
      "rewards/rejected": 0.006256103515625,
      "step": 34
    },
    {
      "epoch": 0.1037037037037037,
      "grad_norm": 276.3713684082031,
      "learning_rate": 1.7922848664688426e-06,
      "logits/chosen": -0.400390625,
      "logits/rejected": -0.41015625,
      "logps/chosen": -114.0,
      "logps/rejected": -128.0,
      "loss": 0.6787,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.032958984375,
      "rewards/margins": 0.036376953125,
      "rewards/rejected": -0.003509521484375,
      "step": 35
    },
    {
      "epoch": 0.10666666666666667,
      "grad_norm": 305.0614929199219,
      "learning_rate": 1.7863501483679523e-06,
      "logits/chosen": -0.369140625,
      "logits/rejected": -0.400390625,
      "logps/chosen": -110.5,
      "logps/rejected": -165.0,
      "loss": 0.6464,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0791015625,
      "rewards/margins": 0.1064453125,
      "rewards/rejected": -0.02734375,
      "step": 36
    },
    {
      "epoch": 0.10962962962962963,
      "grad_norm": 333.9048767089844,
      "learning_rate": 1.7804154302670623e-06,
      "logits/chosen": -0.341796875,
      "logits/rejected": -0.373046875,
      "logps/chosen": -112.0,
      "logps/rejected": -118.5,
      "loss": 0.6801,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.06103515625,
      "rewards/margins": 0.03271484375,
      "rewards/rejected": 0.0281982421875,
      "step": 37
    },
    {
      "epoch": 0.11259259259259259,
      "grad_norm": 282.57940673828125,
      "learning_rate": 1.774480712166172e-06,
      "logits/chosen": -0.39453125,
      "logits/rejected": -0.455078125,
      "logps/chosen": -112.0,
      "logps/rejected": -122.0,
      "loss": 0.6456,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.08349609375,
      "rewards/margins": 0.10546875,
      "rewards/rejected": -0.02197265625,
      "step": 38
    },
    {
      "epoch": 0.11555555555555555,
      "grad_norm": 296.46282958984375,
      "learning_rate": 1.7685459940652817e-06,
      "logits/chosen": -0.37109375,
      "logits/rejected": -0.404296875,
      "logps/chosen": -100.0,
      "logps/rejected": -130.0,
      "loss": 0.6799,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0289306640625,
      "rewards/margins": 0.039794921875,
      "rewards/rejected": -0.01092529296875,
      "step": 39
    },
    {
      "epoch": 0.11851851851851852,
      "grad_norm": 283.2777404785156,
      "learning_rate": 1.7626112759643917e-06,
      "logits/chosen": -0.31640625,
      "logits/rejected": -0.33984375,
      "logps/chosen": -105.5,
      "logps/rejected": -106.5,
      "loss": 0.6694,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.04150390625,
      "rewards/margins": 0.0517578125,
      "rewards/rejected": -0.0101318359375,
      "step": 40
    },
    {
      "epoch": 0.12148148148148148,
      "grad_norm": 273.7195129394531,
      "learning_rate": 1.7566765578635014e-06,
      "logits/chosen": -0.40625,
      "logits/rejected": -0.392578125,
      "logps/chosen": -107.0,
      "logps/rejected": -103.0,
      "loss": 0.6556,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.03466796875,
      "rewards/margins": 0.0810546875,
      "rewards/rejected": -0.046142578125,
      "step": 41
    },
    {
      "epoch": 0.12444444444444444,
      "grad_norm": 294.5924072265625,
      "learning_rate": 1.7507418397626112e-06,
      "logits/chosen": -0.33984375,
      "logits/rejected": -0.36328125,
      "logps/chosen": -105.0,
      "logps/rejected": -156.0,
      "loss": 0.6646,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.046142578125,
      "rewards/margins": 0.07568359375,
      "rewards/rejected": -0.029296875,
      "step": 42
    },
    {
      "epoch": 0.1274074074074074,
      "grad_norm": 307.97613525390625,
      "learning_rate": 1.744807121661721e-06,
      "logits/chosen": -0.396484375,
      "logits/rejected": -0.43359375,
      "logps/chosen": -118.0,
      "logps/rejected": -133.0,
      "loss": 0.6605,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.047607421875,
      "rewards/margins": 0.07421875,
      "rewards/rejected": -0.026611328125,
      "step": 43
    },
    {
      "epoch": 0.13037037037037036,
      "grad_norm": 281.07147216796875,
      "learning_rate": 1.7388724035608308e-06,
      "logits/chosen": -0.40234375,
      "logits/rejected": -0.37109375,
      "logps/chosen": -119.5,
      "logps/rejected": -104.5,
      "loss": 0.6449,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06884765625,
      "rewards/margins": 0.10107421875,
      "rewards/rejected": -0.031982421875,
      "step": 44
    },
    {
      "epoch": 0.13333333333333333,
      "grad_norm": 305.4576110839844,
      "learning_rate": 1.7329376854599408e-06,
      "logits/chosen": -0.345703125,
      "logits/rejected": -0.33203125,
      "logps/chosen": -116.0,
      "logps/rejected": -128.0,
      "loss": 0.6584,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.05078125,
      "rewards/margins": 0.09228515625,
      "rewards/rejected": -0.04150390625,
      "step": 45
    },
    {
      "epoch": 0.1362962962962963,
      "grad_norm": 296.2742614746094,
      "learning_rate": 1.7270029673590503e-06,
      "logits/chosen": -0.396484375,
      "logits/rejected": -0.375,
      "logps/chosen": -124.0,
      "logps/rejected": -105.0,
      "loss": 0.6591,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.08837890625,
      "rewards/margins": 0.078125,
      "rewards/rejected": 0.0101318359375,
      "step": 46
    },
    {
      "epoch": 0.13925925925925925,
      "grad_norm": 271.832275390625,
      "learning_rate": 1.72106824925816e-06,
      "logits/chosen": -0.359375,
      "logits/rejected": -0.384765625,
      "logps/chosen": -109.5,
      "logps/rejected": -119.5,
      "loss": 0.6566,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07373046875,
      "rewards/margins": 0.08935546875,
      "rewards/rejected": -0.015625,
      "step": 47
    },
    {
      "epoch": 0.14222222222222222,
      "grad_norm": 300.9525146484375,
      "learning_rate": 1.71513353115727e-06,
      "logits/chosen": -0.38671875,
      "logits/rejected": -0.390625,
      "logps/chosen": -116.0,
      "logps/rejected": -165.0,
      "loss": 0.6597,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016357421875,
      "rewards/margins": 0.091796875,
      "rewards/rejected": -0.0751953125,
      "step": 48
    },
    {
      "epoch": 0.1451851851851852,
      "grad_norm": 284.7471008300781,
      "learning_rate": 1.7091988130563797e-06,
      "logits/chosen": -0.361328125,
      "logits/rejected": -0.369140625,
      "logps/chosen": -116.0,
      "logps/rejected": -107.0,
      "loss": 0.6768,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.0164794921875,
      "rewards/margins": 0.039794921875,
      "rewards/rejected": -0.0234375,
      "step": 49
    },
    {
      "epoch": 0.14814814814814814,
      "grad_norm": 244.4700469970703,
      "learning_rate": 1.7032640949554894e-06,
      "logits/chosen": -0.357421875,
      "logits/rejected": -0.373046875,
      "logps/chosen": -87.0,
      "logps/rejected": -86.0,
      "loss": 0.6724,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.005462646484375,
      "rewards/margins": 0.055419921875,
      "rewards/rejected": -0.050048828125,
      "step": 50
    },
    {
      "epoch": 0.1511111111111111,
      "grad_norm": 336.86126708984375,
      "learning_rate": 1.6973293768545994e-06,
      "logits/chosen": -0.373046875,
      "logits/rejected": -0.3828125,
      "logps/chosen": -101.5,
      "logps/rejected": -122.0,
      "loss": 0.6431,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.039794921875,
      "rewards/margins": 0.11669921875,
      "rewards/rejected": -0.07666015625,
      "step": 51
    },
    {
      "epoch": 0.15407407407407409,
      "grad_norm": 302.233642578125,
      "learning_rate": 1.691394658753709e-06,
      "logits/chosen": -0.36328125,
      "logits/rejected": -0.416015625,
      "logps/chosen": -108.5,
      "logps/rejected": -130.0,
      "loss": 0.6747,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0101318359375,
      "rewards/margins": 0.04541015625,
      "rewards/rejected": -0.03515625,
      "step": 52
    },
    {
      "epoch": 0.15703703703703703,
      "grad_norm": 298.18658447265625,
      "learning_rate": 1.685459940652819e-06,
      "logits/chosen": -0.359375,
      "logits/rejected": -0.408203125,
      "logps/chosen": -110.0,
      "logps/rejected": -153.0,
      "loss": 0.6451,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03125,
      "rewards/margins": 0.13671875,
      "rewards/rejected": -0.10546875,
      "step": 53
    },
    {
      "epoch": 0.16,
      "grad_norm": 262.4490051269531,
      "learning_rate": 1.6795252225519288e-06,
      "logits/chosen": -0.353515625,
      "logits/rejected": -0.361328125,
      "logps/chosen": -103.5,
      "logps/rejected": -105.5,
      "loss": 0.6473,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06884765625,
      "rewards/margins": 0.10107421875,
      "rewards/rejected": -0.031982421875,
      "step": 54
    },
    {
      "epoch": 0.16296296296296298,
      "grad_norm": 307.3597412109375,
      "learning_rate": 1.6735905044510385e-06,
      "logits/chosen": -0.392578125,
      "logits/rejected": -0.458984375,
      "logps/chosen": -114.5,
      "logps/rejected": -179.0,
      "loss": 0.6312,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.05712890625,
      "rewards/margins": 0.166015625,
      "rewards/rejected": -0.10888671875,
      "step": 55
    },
    {
      "epoch": 0.16592592592592592,
      "grad_norm": 319.4726257324219,
      "learning_rate": 1.6676557863501484e-06,
      "logits/chosen": -0.3671875,
      "logits/rejected": -0.3515625,
      "logps/chosen": -100.5,
      "logps/rejected": -96.5,
      "loss": 0.6974,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.007781982421875,
      "rewards/margins": 0.0015106201171875,
      "rewards/rejected": -0.00933837890625,
      "step": 56
    },
    {
      "epoch": 0.1688888888888889,
      "grad_norm": 315.0474548339844,
      "learning_rate": 1.6617210682492582e-06,
      "logits/chosen": -0.412109375,
      "logits/rejected": -0.42578125,
      "logps/chosen": -137.0,
      "logps/rejected": -184.0,
      "loss": 0.6286,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00775146484375,
      "rewards/margins": 0.1484375,
      "rewards/rejected": -0.140625,
      "step": 57
    },
    {
      "epoch": 0.17185185185185184,
      "grad_norm": 302.9765625,
      "learning_rate": 1.655786350148368e-06,
      "logits/chosen": -0.359375,
      "logits/rejected": -0.3828125,
      "logps/chosen": -103.5,
      "logps/rejected": -110.5,
      "loss": 0.6796,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04296875,
      "rewards/margins": 0.038330078125,
      "rewards/rejected": 0.00469970703125,
      "step": 58
    },
    {
      "epoch": 0.1748148148148148,
      "grad_norm": 272.1220703125,
      "learning_rate": 1.6498516320474776e-06,
      "logits/chosen": -0.388671875,
      "logits/rejected": -0.390625,
      "logps/chosen": -110.5,
      "logps/rejected": -96.5,
      "loss": 0.6531,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0546875,
      "rewards/margins": 0.09619140625,
      "rewards/rejected": -0.04150390625,
      "step": 59
    },
    {
      "epoch": 0.17777777777777778,
      "grad_norm": 307.90301513671875,
      "learning_rate": 1.6439169139465874e-06,
      "logits/chosen": -0.369140625,
      "logits/rejected": -0.390625,
      "logps/chosen": -112.0,
      "logps/rejected": -124.0,
      "loss": 0.6341,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0625,
      "rewards/margins": 0.126953125,
      "rewards/rejected": -0.06494140625,
      "step": 60
    },
    {
      "epoch": 0.18074074074074073,
      "grad_norm": 312.32916259765625,
      "learning_rate": 1.637982195845697e-06,
      "logits/chosen": -0.408203125,
      "logits/rejected": -0.376953125,
      "logps/chosen": -132.0,
      "logps/rejected": -113.0,
      "loss": 0.647,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04150390625,
      "rewards/margins": 0.10888671875,
      "rewards/rejected": -0.0673828125,
      "step": 61
    },
    {
      "epoch": 0.1837037037037037,
      "grad_norm": 327.3997802734375,
      "learning_rate": 1.632047477744807e-06,
      "logits/chosen": -0.404296875,
      "logits/rejected": -0.3828125,
      "logps/chosen": -115.0,
      "logps/rejected": -153.0,
      "loss": 0.6488,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0179443359375,
      "rewards/margins": 0.11083984375,
      "rewards/rejected": -0.09326171875,
      "step": 62
    },
    {
      "epoch": 0.18666666666666668,
      "grad_norm": 290.80908203125,
      "learning_rate": 1.6261127596439168e-06,
      "logits/chosen": -0.337890625,
      "logits/rejected": -0.39453125,
      "logps/chosen": -114.0,
      "logps/rejected": -120.0,
      "loss": 0.6129,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.057861328125,
      "rewards/margins": 0.1826171875,
      "rewards/rejected": -0.12451171875,
      "step": 63
    },
    {
      "epoch": 0.18962962962962962,
      "grad_norm": 266.57843017578125,
      "learning_rate": 1.6201780415430267e-06,
      "logits/chosen": -0.3515625,
      "logits/rejected": -0.373046875,
      "logps/chosen": -105.0,
      "logps/rejected": -88.5,
      "loss": 0.6377,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05322265625,
      "rewards/margins": 0.1181640625,
      "rewards/rejected": -0.06494140625,
      "step": 64
    },
    {
      "epoch": 0.1925925925925926,
      "grad_norm": 314.43389892578125,
      "learning_rate": 1.6142433234421364e-06,
      "logits/chosen": -0.4140625,
      "logits/rejected": -0.427734375,
      "logps/chosen": -113.5,
      "logps/rejected": -193.0,
      "loss": 0.6267,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.061767578125,
      "rewards/margins": 0.208984375,
      "rewards/rejected": -0.1474609375,
      "step": 65
    },
    {
      "epoch": 0.19555555555555557,
      "grad_norm": 315.067626953125,
      "learning_rate": 1.6083086053412462e-06,
      "logits/chosen": -0.376953125,
      "logits/rejected": -0.400390625,
      "logps/chosen": -116.0,
      "logps/rejected": -117.5,
      "loss": 0.6625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0257568359375,
      "rewards/margins": 0.07958984375,
      "rewards/rejected": -0.053955078125,
      "step": 66
    },
    {
      "epoch": 0.1985185185185185,
      "grad_norm": 309.8024597167969,
      "learning_rate": 1.6023738872403561e-06,
      "logits/chosen": -0.4296875,
      "logits/rejected": -0.466796875,
      "logps/chosen": -122.5,
      "logps/rejected": -139.0,
      "loss": 0.6585,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.03759765625,
      "rewards/margins": 0.09130859375,
      "rewards/rejected": -0.053955078125,
      "step": 67
    },
    {
      "epoch": 0.20148148148148148,
      "grad_norm": 303.5541687011719,
      "learning_rate": 1.5964391691394659e-06,
      "logits/chosen": -0.376953125,
      "logits/rejected": -0.4375,
      "logps/chosen": -124.5,
      "logps/rejected": -142.0,
      "loss": 0.6592,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0289306640625,
      "rewards/margins": 0.08056640625,
      "rewards/rejected": -0.109375,
      "step": 68
    },
    {
      "epoch": 0.20444444444444446,
      "grad_norm": 336.009033203125,
      "learning_rate": 1.5905044510385756e-06,
      "logits/chosen": -0.3515625,
      "logits/rejected": -0.369140625,
      "logps/chosen": -118.0,
      "logps/rejected": -132.0,
      "loss": 0.6651,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.00225830078125,
      "rewards/margins": 0.07373046875,
      "rewards/rejected": -0.07568359375,
      "step": 69
    },
    {
      "epoch": 0.2074074074074074,
      "grad_norm": 295.90008544921875,
      "learning_rate": 1.5845697329376855e-06,
      "logits/chosen": -0.41015625,
      "logits/rejected": -0.43359375,
      "logps/chosen": -123.5,
      "logps/rejected": -129.0,
      "loss": 0.6312,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.04345703125,
      "rewards/margins": 0.1455078125,
      "rewards/rejected": -0.1025390625,
      "step": 70
    },
    {
      "epoch": 0.21037037037037037,
      "grad_norm": 308.5846862792969,
      "learning_rate": 1.5786350148367953e-06,
      "logits/chosen": -0.400390625,
      "logits/rejected": -0.3984375,
      "logps/chosen": -110.0,
      "logps/rejected": -160.0,
      "loss": 0.6731,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.01806640625,
      "rewards/margins": 0.11767578125,
      "rewards/rejected": -0.09912109375,
      "step": 71
    },
    {
      "epoch": 0.21333333333333335,
      "grad_norm": 311.0722961425781,
      "learning_rate": 1.5727002967359048e-06,
      "logits/chosen": -0.44140625,
      "logits/rejected": -0.4453125,
      "logps/chosen": -137.0,
      "logps/rejected": -214.0,
      "loss": 0.593,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.064453125,
      "rewards/margins": 0.294921875,
      "rewards/rejected": -0.2294921875,
      "step": 72
    },
    {
      "epoch": 0.2162962962962963,
      "grad_norm": 304.479248046875,
      "learning_rate": 1.5667655786350147e-06,
      "logits/chosen": -0.416015625,
      "logits/rejected": -0.447265625,
      "logps/chosen": -124.5,
      "logps/rejected": -148.0,
      "loss": 0.6428,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.022705078125,
      "rewards/margins": 0.125,
      "rewards/rejected": -0.1025390625,
      "step": 73
    },
    {
      "epoch": 0.21925925925925926,
      "grad_norm": 280.2344665527344,
      "learning_rate": 1.5608308605341244e-06,
      "logits/chosen": -0.392578125,
      "logits/rejected": -0.373046875,
      "logps/chosen": -107.5,
      "logps/rejected": -119.0,
      "loss": 0.5997,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0908203125,
      "rewards/margins": 0.236328125,
      "rewards/rejected": -0.1455078125,
      "step": 74
    },
    {
      "epoch": 0.2222222222222222,
      "grad_norm": 275.5752258300781,
      "learning_rate": 1.5548961424332344e-06,
      "logits/chosen": -0.40234375,
      "logits/rejected": -0.44140625,
      "logps/chosen": -116.0,
      "logps/rejected": -110.5,
      "loss": 0.6214,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.057861328125,
      "rewards/margins": 0.154296875,
      "rewards/rejected": -0.09619140625,
      "step": 75
    },
    {
      "epoch": 0.22518518518518518,
      "grad_norm": 262.7906188964844,
      "learning_rate": 1.5489614243323441e-06,
      "logits/chosen": -0.408203125,
      "logits/rejected": -0.419921875,
      "logps/chosen": -100.5,
      "logps/rejected": -137.0,
      "loss": 0.5972,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.050048828125,
      "rewards/margins": 0.2392578125,
      "rewards/rejected": -0.189453125,
      "step": 76
    },
    {
      "epoch": 0.22814814814814816,
      "grad_norm": 308.56695556640625,
      "learning_rate": 1.5430267062314539e-06,
      "logits/chosen": -0.40234375,
      "logits/rejected": -0.4140625,
      "logps/chosen": -114.0,
      "logps/rejected": -199.0,
      "loss": 0.6227,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0078125,
      "rewards/margins": 0.2177734375,
      "rewards/rejected": -0.2255859375,
      "step": 77
    },
    {
      "epoch": 0.2311111111111111,
      "grad_norm": 320.17828369140625,
      "learning_rate": 1.5370919881305638e-06,
      "logits/chosen": -0.404296875,
      "logits/rejected": -0.43359375,
      "logps/chosen": -137.0,
      "logps/rejected": -103.0,
      "loss": 0.6665,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.0257568359375,
      "rewards/margins": 0.07177734375,
      "rewards/rejected": -0.046142578125,
      "step": 78
    },
    {
      "epoch": 0.23407407407407407,
      "grad_norm": 325.34100341796875,
      "learning_rate": 1.5311572700296735e-06,
      "logits/chosen": -0.416015625,
      "logits/rejected": -0.447265625,
      "logps/chosen": -138.0,
      "logps/rejected": -125.5,
      "loss": 0.6537,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.031982421875,
      "rewards/margins": 0.11669921875,
      "rewards/rejected": -0.08447265625,
      "step": 79
    },
    {
      "epoch": 0.23703703703703705,
      "grad_norm": 341.1995544433594,
      "learning_rate": 1.5252225519287833e-06,
      "logits/chosen": -0.380859375,
      "logits/rejected": -0.447265625,
      "logps/chosen": -112.0,
      "logps/rejected": -179.0,
      "loss": 0.6116,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.02734375,
      "rewards/margins": 0.2109375,
      "rewards/rejected": -0.18359375,
      "step": 80
    },
    {
      "epoch": 0.24,
      "grad_norm": 284.32147216796875,
      "learning_rate": 1.5192878338278932e-06,
      "logits/chosen": -0.404296875,
      "logits/rejected": -0.390625,
      "logps/chosen": -132.0,
      "logps/rejected": -123.0,
      "loss": 0.6152,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.109375,
      "rewards/margins": 0.1845703125,
      "rewards/rejected": -0.0751953125,
      "step": 81
    },
    {
      "epoch": 0.24296296296296296,
      "grad_norm": 280.8007507324219,
      "learning_rate": 1.513353115727003e-06,
      "logits/chosen": -0.3515625,
      "logits/rejected": -0.37890625,
      "logps/chosen": -97.5,
      "logps/rejected": -107.0,
      "loss": 0.6219,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.06591796875,
      "rewards/margins": 0.1650390625,
      "rewards/rejected": -0.0986328125,
      "step": 82
    },
    {
      "epoch": 0.24592592592592594,
      "grad_norm": 287.39727783203125,
      "learning_rate": 1.5074183976261129e-06,
      "logits/chosen": -0.4140625,
      "logits/rejected": -0.4453125,
      "logps/chosen": -121.0,
      "logps/rejected": -130.0,
      "loss": 0.6078,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0693359375,
      "rewards/margins": 0.212890625,
      "rewards/rejected": -0.1435546875,
      "step": 83
    },
    {
      "epoch": 0.24888888888888888,
      "grad_norm": 293.0041809082031,
      "learning_rate": 1.5014836795252226e-06,
      "logits/chosen": -0.41796875,
      "logits/rejected": -0.412109375,
      "logps/chosen": -95.5,
      "logps/rejected": -104.5,
      "loss": 0.655,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.032470703125,
      "rewards/margins": 0.0966796875,
      "rewards/rejected": -0.06396484375,
      "step": 84
    },
    {
      "epoch": 0.2518518518518518,
      "grad_norm": 365.5411071777344,
      "learning_rate": 1.4955489614243321e-06,
      "logits/chosen": -0.439453125,
      "logits/rejected": -0.41015625,
      "logps/chosen": -159.0,
      "logps/rejected": -165.0,
      "loss": 0.6666,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0546875,
      "rewards/margins": 0.1064453125,
      "rewards/rejected": -0.1611328125,
      "step": 85
    },
    {
      "epoch": 0.2548148148148148,
      "grad_norm": 327.99237060546875,
      "learning_rate": 1.489614243323442e-06,
      "logits/chosen": -0.4140625,
      "logits/rejected": -0.396484375,
      "logps/chosen": -126.0,
      "logps/rejected": -171.0,
      "loss": 0.6431,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02978515625,
      "rewards/margins": 0.1904296875,
      "rewards/rejected": -0.1611328125,
      "step": 86
    },
    {
      "epoch": 0.2577777777777778,
      "grad_norm": 299.70306396484375,
      "learning_rate": 1.4836795252225518e-06,
      "logits/chosen": -0.43359375,
      "logits/rejected": -0.490234375,
      "logps/chosen": -124.5,
      "logps/rejected": -192.0,
      "loss": 0.5707,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1396484375,
      "rewards/margins": 0.373046875,
      "rewards/rejected": -0.2333984375,
      "step": 87
    },
    {
      "epoch": 0.2607407407407407,
      "grad_norm": 293.9344787597656,
      "learning_rate": 1.4777448071216615e-06,
      "logits/chosen": -0.390625,
      "logits/rejected": -0.4140625,
      "logps/chosen": -107.0,
      "logps/rejected": -125.0,
      "loss": 0.6241,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08349609375,
      "rewards/margins": 0.1875,
      "rewards/rejected": -0.10400390625,
      "step": 88
    },
    {
      "epoch": 0.2637037037037037,
      "grad_norm": 295.9560241699219,
      "learning_rate": 1.4718100890207715e-06,
      "logits/chosen": -0.408203125,
      "logits/rejected": -0.400390625,
      "logps/chosen": -136.0,
      "logps/rejected": -129.0,
      "loss": 0.5942,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.08837890625,
      "rewards/margins": 0.2353515625,
      "rewards/rejected": -0.146484375,
      "step": 89
    },
    {
      "epoch": 0.26666666666666666,
      "grad_norm": 341.2995910644531,
      "learning_rate": 1.4658753709198812e-06,
      "logits/chosen": -0.40625,
      "logits/rejected": -0.41796875,
      "logps/chosen": -112.5,
      "logps/rejected": -119.5,
      "loss": 0.638,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09228515625,
      "rewards/margins": 0.1435546875,
      "rewards/rejected": -0.051513671875,
      "step": 90
    },
    {
      "epoch": 0.2696296296296296,
      "grad_norm": 295.9945373535156,
      "learning_rate": 1.459940652818991e-06,
      "logits/chosen": -0.390625,
      "logits/rejected": -0.40234375,
      "logps/chosen": -97.5,
      "logps/rejected": -122.5,
      "loss": 0.6188,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.043701171875,
      "rewards/margins": 0.181640625,
      "rewards/rejected": -0.1376953125,
      "step": 91
    },
    {
      "epoch": 0.2725925925925926,
      "grad_norm": 335.8014831542969,
      "learning_rate": 1.4540059347181009e-06,
      "logits/chosen": -0.423828125,
      "logits/rejected": -0.404296875,
      "logps/chosen": -141.0,
      "logps/rejected": -124.5,
      "loss": 0.6823,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.015625,
      "rewards/margins": 0.10546875,
      "rewards/rejected": -0.12109375,
      "step": 92
    },
    {
      "epoch": 0.27555555555555555,
      "grad_norm": 316.69366455078125,
      "learning_rate": 1.4480712166172106e-06,
      "logits/chosen": -0.4140625,
      "logits/rejected": -0.447265625,
      "logps/chosen": -132.0,
      "logps/rejected": -121.0,
      "loss": 0.5893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09326171875,
      "rewards/margins": 0.251953125,
      "rewards/rejected": -0.158203125,
      "step": 93
    },
    {
      "epoch": 0.2785185185185185,
      "grad_norm": 287.4927062988281,
      "learning_rate": 1.4421364985163206e-06,
      "logits/chosen": -0.375,
      "logits/rejected": -0.388671875,
      "logps/chosen": -98.0,
      "logps/rejected": -114.5,
      "loss": 0.6371,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.08203125,
      "rewards/margins": 0.142578125,
      "rewards/rejected": -0.060302734375,
      "step": 94
    },
    {
      "epoch": 0.2814814814814815,
      "grad_norm": 279.02618408203125,
      "learning_rate": 1.4362017804154303e-06,
      "logits/chosen": -0.3828125,
      "logits/rejected": -0.416015625,
      "logps/chosen": -104.5,
      "logps/rejected": -145.0,
      "loss": 0.5956,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.07568359375,
      "rewards/margins": 0.2890625,
      "rewards/rejected": -0.2138671875,
      "step": 95
    },
    {
      "epoch": 0.28444444444444444,
      "grad_norm": 387.8583068847656,
      "learning_rate": 1.43026706231454e-06,
      "logits/chosen": -0.404296875,
      "logits/rejected": -0.396484375,
      "logps/chosen": -108.5,
      "logps/rejected": -106.0,
      "loss": 0.6595,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0133056640625,
      "rewards/margins": 0.09130859375,
      "rewards/rejected": -0.078125,
      "step": 96
    },
    {
      "epoch": 0.2874074074074074,
      "grad_norm": 362.9851379394531,
      "learning_rate": 1.42433234421365e-06,
      "logits/chosen": -0.43359375,
      "logits/rejected": -0.4375,
      "logps/chosen": -139.0,
      "logps/rejected": -120.0,
      "loss": 0.673,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.020263671875,
      "rewards/margins": 0.0751953125,
      "rewards/rejected": -0.0546875,
      "step": 97
    },
    {
      "epoch": 0.2903703703703704,
      "grad_norm": 308.3312072753906,
      "learning_rate": 1.4183976261127595e-06,
      "logits/chosen": -0.390625,
      "logits/rejected": -0.439453125,
      "logps/chosen": -113.0,
      "logps/rejected": -119.5,
      "loss": 0.6538,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.013916015625,
      "rewards/margins": 0.126953125,
      "rewards/rejected": -0.11328125,
      "step": 98
    },
    {
      "epoch": 0.29333333333333333,
      "grad_norm": 360.1666259765625,
      "learning_rate": 1.4124629080118692e-06,
      "logits/chosen": -0.419921875,
      "logits/rejected": -0.4609375,
      "logps/chosen": -131.0,
      "logps/rejected": -124.0,
      "loss": 0.6746,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.06982421875,
      "rewards/margins": 0.053466796875,
      "rewards/rejected": -0.12353515625,
      "step": 99
    },
    {
      "epoch": 0.2962962962962963,
      "grad_norm": 285.48065185546875,
      "learning_rate": 1.4065281899109791e-06,
      "logits/chosen": -0.392578125,
      "logits/rejected": -0.4140625,
      "logps/chosen": -101.5,
      "logps/rejected": -109.0,
      "loss": 0.5988,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.039794921875,
      "rewards/margins": 0.228515625,
      "rewards/rejected": -0.189453125,
      "step": 100
    },
    {
      "epoch": 0.2992592592592593,
      "grad_norm": 272.54583740234375,
      "learning_rate": 1.4005934718100889e-06,
      "logits/chosen": -0.3828125,
      "logits/rejected": -0.37109375,
      "logps/chosen": -94.0,
      "logps/rejected": -100.5,
      "loss": 0.6157,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06884765625,
      "rewards/margins": 0.1865234375,
      "rewards/rejected": -0.1181640625,
      "step": 101
    },
    {
      "epoch": 0.3022222222222222,
      "grad_norm": 261.4800109863281,
      "learning_rate": 1.3946587537091988e-06,
      "logits/chosen": -0.3671875,
      "logits/rejected": -0.396484375,
      "logps/chosen": -90.0,
      "logps/rejected": -101.0,
      "loss": 0.6438,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.018798828125,
      "rewards/margins": 0.12353515625,
      "rewards/rejected": -0.10498046875,
      "step": 102
    },
    {
      "epoch": 0.30518518518518517,
      "grad_norm": 299.9373779296875,
      "learning_rate": 1.3887240356083086e-06,
      "logits/chosen": -0.443359375,
      "logits/rejected": -0.435546875,
      "logps/chosen": -109.0,
      "logps/rejected": -102.0,
      "loss": 0.6418,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0361328125,
      "rewards/margins": 0.1298828125,
      "rewards/rejected": -0.09375,
      "step": 103
    },
    {
      "epoch": 0.30814814814814817,
      "grad_norm": 315.3984069824219,
      "learning_rate": 1.3827893175074183e-06,
      "logits/chosen": -0.421875,
      "logits/rejected": -0.439453125,
      "logps/chosen": -112.0,
      "logps/rejected": -118.0,
      "loss": 0.6638,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.024169921875,
      "rewards/margins": 0.09716796875,
      "rewards/rejected": -0.12109375,
      "step": 104
    },
    {
      "epoch": 0.3111111111111111,
      "grad_norm": 309.1874084472656,
      "learning_rate": 1.3768545994065282e-06,
      "logits/chosen": -0.41015625,
      "logits/rejected": -0.412109375,
      "logps/chosen": -104.0,
      "logps/rejected": -118.5,
      "loss": 0.6292,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.038330078125,
      "rewards/margins": 0.15234375,
      "rewards/rejected": -0.1142578125,
      "step": 105
    },
    {
      "epoch": 0.31407407407407406,
      "grad_norm": 306.6035461425781,
      "learning_rate": 1.370919881305638e-06,
      "logits/chosen": -0.39453125,
      "logits/rejected": -0.423828125,
      "logps/chosen": -121.5,
      "logps/rejected": -111.5,
      "loss": 0.636,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02294921875,
      "rewards/margins": 0.1611328125,
      "rewards/rejected": -0.1845703125,
      "step": 106
    },
    {
      "epoch": 0.31703703703703706,
      "grad_norm": 267.6772766113281,
      "learning_rate": 1.3649851632047477e-06,
      "logits/chosen": -0.384765625,
      "logits/rejected": -0.412109375,
      "logps/chosen": -91.0,
      "logps/rejected": -116.5,
      "loss": 0.5857,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0849609375,
      "rewards/margins": 0.2734375,
      "rewards/rejected": -0.1884765625,
      "step": 107
    },
    {
      "epoch": 0.32,
      "grad_norm": 323.27239990234375,
      "learning_rate": 1.3590504451038576e-06,
      "logits/chosen": -0.419921875,
      "logits/rejected": -0.458984375,
      "logps/chosen": -98.0,
      "logps/rejected": -118.5,
      "loss": 0.6342,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.08056640625,
      "rewards/margins": 0.1572265625,
      "rewards/rejected": -0.0771484375,
      "step": 108
    },
    {
      "epoch": 0.32296296296296295,
      "grad_norm": 303.9071044921875,
      "learning_rate": 1.3531157270029674e-06,
      "logits/chosen": -0.4609375,
      "logits/rejected": -0.466796875,
      "logps/chosen": -114.0,
      "logps/rejected": -108.5,
      "loss": 0.6272,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.028076171875,
      "rewards/margins": 0.1650390625,
      "rewards/rejected": -0.13671875,
      "step": 109
    },
    {
      "epoch": 0.32592592592592595,
      "grad_norm": 309.7470397949219,
      "learning_rate": 1.347181008902077e-06,
      "logits/chosen": -0.439453125,
      "logits/rejected": -0.427734375,
      "logps/chosen": -132.0,
      "logps/rejected": -168.0,
      "loss": 0.5938,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.036865234375,
      "rewards/margins": 0.30859375,
      "rewards/rejected": -0.2734375,
      "step": 110
    },
    {
      "epoch": 0.3288888888888889,
      "grad_norm": 276.505615234375,
      "learning_rate": 1.3412462908011868e-06,
      "logits/chosen": -0.39453125,
      "logits/rejected": -0.40234375,
      "logps/chosen": -97.5,
      "logps/rejected": -106.0,
      "loss": 0.6151,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1220703125,
      "rewards/margins": 0.1875,
      "rewards/rejected": -0.06591796875,
      "step": 111
    },
    {
      "epoch": 0.33185185185185184,
      "grad_norm": 317.2270202636719,
      "learning_rate": 1.3353115727002966e-06,
      "logits/chosen": -0.41796875,
      "logits/rejected": -0.439453125,
      "logps/chosen": -117.0,
      "logps/rejected": -106.5,
      "loss": 0.6424,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.046142578125,
      "rewards/margins": 0.1298828125,
      "rewards/rejected": -0.08349609375,
      "step": 112
    },
    {
      "epoch": 0.3348148148148148,
      "grad_norm": 283.0464172363281,
      "learning_rate": 1.3293768545994065e-06,
      "logits/chosen": -0.419921875,
      "logits/rejected": -0.421875,
      "logps/chosen": -88.5,
      "logps/rejected": -97.5,
      "loss": 0.6356,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0947265625,
      "rewards/margins": 0.14453125,
      "rewards/rejected": -0.050048828125,
      "step": 113
    },
    {
      "epoch": 0.3377777777777778,
      "grad_norm": 280.1183166503906,
      "learning_rate": 1.3234421364985162e-06,
      "logits/chosen": -0.44140625,
      "logits/rejected": -0.466796875,
      "logps/chosen": -99.0,
      "logps/rejected": -152.0,
      "loss": 0.5299,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.12353515625,
      "rewards/margins": 0.404296875,
      "rewards/rejected": -0.28125,
      "step": 114
    },
    {
      "epoch": 0.34074074074074073,
      "grad_norm": 280.7152404785156,
      "learning_rate": 1.317507418397626e-06,
      "logits/chosen": -0.39453125,
      "logits/rejected": -0.412109375,
      "logps/chosen": -102.5,
      "logps/rejected": -117.5,
      "loss": 0.5846,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.10546875,
      "rewards/margins": 0.27734375,
      "rewards/rejected": -0.1728515625,
      "step": 115
    },
    {
      "epoch": 0.3437037037037037,
      "grad_norm": 311.7525329589844,
      "learning_rate": 1.311572700296736e-06,
      "logits/chosen": -0.384765625,
      "logits/rejected": -0.40625,
      "logps/chosen": -98.5,
      "logps/rejected": -100.5,
      "loss": 0.6609,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.038330078125,
      "rewards/margins": 0.08837890625,
      "rewards/rejected": -0.126953125,
      "step": 116
    },
    {
      "epoch": 0.3466666666666667,
      "grad_norm": 299.9917297363281,
      "learning_rate": 1.3056379821958456e-06,
      "logits/chosen": -0.4140625,
      "logits/rejected": -0.42578125,
      "logps/chosen": -97.5,
      "logps/rejected": -112.0,
      "loss": 0.6263,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.056884765625,
      "rewards/margins": 0.166015625,
      "rewards/rejected": -0.109375,
      "step": 117
    },
    {
      "epoch": 0.3496296296296296,
      "grad_norm": 309.1261901855469,
      "learning_rate": 1.2997032640949554e-06,
      "logits/chosen": -0.42578125,
      "logits/rejected": -0.416015625,
      "logps/chosen": -113.5,
      "logps/rejected": -113.5,
      "loss": 0.621,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.052490234375,
      "rewards/margins": 0.1826171875,
      "rewards/rejected": -0.130859375,
      "step": 118
    },
    {
      "epoch": 0.35259259259259257,
      "grad_norm": 272.764404296875,
      "learning_rate": 1.2937685459940653e-06,
      "logits/chosen": -0.384765625,
      "logits/rejected": -0.431640625,
      "logps/chosen": -105.0,
      "logps/rejected": -113.5,
      "loss": 0.6042,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.060302734375,
      "rewards/margins": 0.2265625,
      "rewards/rejected": -0.166015625,
      "step": 119
    },
    {
      "epoch": 0.35555555555555557,
      "grad_norm": 268.743408203125,
      "learning_rate": 1.287833827893175e-06,
      "logits/chosen": -0.421875,
      "logits/rejected": -0.435546875,
      "logps/chosen": -109.5,
      "logps/rejected": -209.0,
      "loss": 0.5652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0908203125,
      "rewards/margins": 0.39453125,
      "rewards/rejected": -0.3046875,
      "step": 120
    },
    {
      "epoch": 0.3585185185185185,
      "grad_norm": 328.0078125,
      "learning_rate": 1.281899109792285e-06,
      "logits/chosen": -0.40234375,
      "logits/rejected": -0.412109375,
      "logps/chosen": -110.0,
      "logps/rejected": -120.5,
      "loss": 0.6072,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.031494140625,
      "rewards/margins": 0.2177734375,
      "rewards/rejected": -0.1865234375,
      "step": 121
    },
    {
      "epoch": 0.36148148148148146,
      "grad_norm": 291.20794677734375,
      "learning_rate": 1.2759643916913947e-06,
      "logits/chosen": -0.357421875,
      "logits/rejected": -0.36328125,
      "logps/chosen": -96.0,
      "logps/rejected": -93.5,
      "loss": 0.6382,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.047607421875,
      "rewards/margins": 0.1435546875,
      "rewards/rejected": -0.09619140625,
      "step": 122
    },
    {
      "epoch": 0.36444444444444446,
      "grad_norm": 311.23870849609375,
      "learning_rate": 1.2700296735905044e-06,
      "logits/chosen": -0.37890625,
      "logits/rejected": -0.380859375,
      "logps/chosen": -101.5,
      "logps/rejected": -191.0,
      "loss": 0.6198,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.03271484375,
      "rewards/margins": 0.349609375,
      "rewards/rejected": -0.31640625,
      "step": 123
    },
    {
      "epoch": 0.3674074074074074,
      "grad_norm": 327.720703125,
      "learning_rate": 1.2640949554896142e-06,
      "logits/chosen": -0.3984375,
      "logits/rejected": -0.443359375,
      "logps/chosen": -111.0,
      "logps/rejected": -133.0,
      "loss": 0.6528,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01953125,
      "rewards/margins": 0.1513671875,
      "rewards/rejected": -0.1318359375,
      "step": 124
    },
    {
      "epoch": 0.37037037037037035,
      "grad_norm": 301.95654296875,
      "learning_rate": 1.258160237388724e-06,
      "logits/chosen": -0.4296875,
      "logits/rejected": -0.435546875,
      "logps/chosen": -121.0,
      "logps/rejected": -118.5,
      "loss": 0.5869,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0732421875,
      "rewards/margins": 0.28125,
      "rewards/rejected": -0.20703125,
      "step": 125
    },
    {
      "epoch": 0.37333333333333335,
      "grad_norm": 349.1401672363281,
      "learning_rate": 1.2522255192878336e-06,
      "logits/chosen": -0.412109375,
      "logits/rejected": -0.41015625,
      "logps/chosen": -120.5,
      "logps/rejected": -104.5,
      "loss": 0.6876,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.083984375,
      "rewards/margins": 0.052978515625,
      "rewards/rejected": -0.13671875,
      "step": 126
    },
    {
      "epoch": 0.3762962962962963,
      "grad_norm": 319.4087219238281,
      "learning_rate": 1.2462908011869436e-06,
      "logits/chosen": -0.453125,
      "logits/rejected": -0.49609375,
      "logps/chosen": -115.5,
      "logps/rejected": -129.0,
      "loss": 0.6176,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.043701171875,
      "rewards/margins": 0.2080078125,
      "rewards/rejected": -0.1640625,
      "step": 127
    },
    {
      "epoch": 0.37925925925925924,
      "grad_norm": 325.74456787109375,
      "learning_rate": 1.2403560830860533e-06,
      "logits/chosen": -0.412109375,
      "logits/rejected": -0.4140625,
      "logps/chosen": -109.5,
      "logps/rejected": -177.0,
      "loss": 0.6262,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0322265625,
      "rewards/margins": 0.2578125,
      "rewards/rejected": -0.2255859375,
      "step": 128
    },
    {
      "epoch": 0.38222222222222224,
      "grad_norm": 288.1813659667969,
      "learning_rate": 1.234421364985163e-06,
      "logits/chosen": -0.439453125,
      "logits/rejected": -0.458984375,
      "logps/chosen": -106.0,
      "logps/rejected": -96.5,
      "loss": 0.6172,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0108642578125,
      "rewards/margins": 0.208984375,
      "rewards/rejected": -0.197265625,
      "step": 129
    },
    {
      "epoch": 0.3851851851851852,
      "grad_norm": 291.5497131347656,
      "learning_rate": 1.228486646884273e-06,
      "logits/chosen": -0.43359375,
      "logits/rejected": -0.4609375,
      "logps/chosen": -110.5,
      "logps/rejected": -158.0,
      "loss": 0.5666,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.03662109375,
      "rewards/margins": 0.36328125,
      "rewards/rejected": -0.326171875,
      "step": 130
    },
    {
      "epoch": 0.38814814814814813,
      "grad_norm": 298.1041564941406,
      "learning_rate": 1.2225519287833827e-06,
      "logits/chosen": -0.404296875,
      "logits/rejected": -0.419921875,
      "logps/chosen": -115.0,
      "logps/rejected": -133.0,
      "loss": 0.6319,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.053955078125,
      "rewards/margins": 0.162109375,
      "rewards/rejected": -0.10791015625,
      "step": 131
    },
    {
      "epoch": 0.39111111111111113,
      "grad_norm": 291.80596923828125,
      "learning_rate": 1.2166172106824927e-06,
      "logits/chosen": -0.400390625,
      "logits/rejected": -0.3984375,
      "logps/chosen": -100.5,
      "logps/rejected": -109.0,
      "loss": 0.6202,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0390625,
      "rewards/margins": 0.171875,
      "rewards/rejected": -0.1328125,
      "step": 132
    },
    {
      "epoch": 0.3940740740740741,
      "grad_norm": 291.0092468261719,
      "learning_rate": 1.2106824925816024e-06,
      "logits/chosen": -0.3828125,
      "logits/rejected": -0.421875,
      "logps/chosen": -97.5,
      "logps/rejected": -105.5,
      "loss": 0.5925,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.03662109375,
      "rewards/margins": 0.2578125,
      "rewards/rejected": -0.2216796875,
      "step": 133
    },
    {
      "epoch": 0.397037037037037,
      "grad_norm": 341.5547180175781,
      "learning_rate": 1.2047477744807121e-06,
      "logits/chosen": -0.392578125,
      "logits/rejected": -0.416015625,
      "logps/chosen": -109.5,
      "logps/rejected": -112.0,
      "loss": 0.6822,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.025146484375,
      "rewards/margins": 0.059326171875,
      "rewards/rejected": -0.08447265625,
      "step": 134
    },
    {
      "epoch": 0.4,
      "grad_norm": 392.70574951171875,
      "learning_rate": 1.198813056379822e-06,
      "logits/chosen": -0.431640625,
      "logits/rejected": -0.439453125,
      "logps/chosen": -127.0,
      "logps/rejected": -137.0,
      "loss": 0.6592,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.046875,
      "rewards/margins": 0.15625,
      "rewards/rejected": -0.203125,
      "step": 135
    },
    {
      "epoch": 0.40296296296296297,
      "grad_norm": 306.712646484375,
      "learning_rate": 1.1928783382789318e-06,
      "logits/chosen": -0.408203125,
      "logits/rejected": -0.4765625,
      "logps/chosen": -107.0,
      "logps/rejected": -115.5,
      "loss": 0.6097,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.016357421875,
      "rewards/margins": 0.2001953125,
      "rewards/rejected": -0.2158203125,
      "step": 136
    },
    {
      "epoch": 0.4059259259259259,
      "grad_norm": 303.9560546875,
      "learning_rate": 1.1869436201780413e-06,
      "logits/chosen": -0.37890625,
      "logits/rejected": -0.423828125,
      "logps/chosen": -101.5,
      "logps/rejected": -131.0,
      "loss": 0.6343,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.030517578125,
      "rewards/margins": 0.15234375,
      "rewards/rejected": -0.1826171875,
      "step": 137
    },
    {
      "epoch": 0.4088888888888889,
      "grad_norm": 360.12445068359375,
      "learning_rate": 1.1810089020771513e-06,
      "logits/chosen": -0.423828125,
      "logits/rejected": -0.412109375,
      "logps/chosen": -122.0,
      "logps/rejected": -104.5,
      "loss": 0.6484,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0281982421875,
      "rewards/margins": 0.1591796875,
      "rewards/rejected": -0.1865234375,
      "step": 138
    },
    {
      "epoch": 0.41185185185185186,
      "grad_norm": 281.4104919433594,
      "learning_rate": 1.175074183976261e-06,
      "logits/chosen": -0.392578125,
      "logits/rejected": -0.40625,
      "logps/chosen": -109.5,
      "logps/rejected": -115.5,
      "loss": 0.6265,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.06103515625,
      "rewards/margins": 0.1845703125,
      "rewards/rejected": -0.12353515625,
      "step": 139
    },
    {
      "epoch": 0.4148148148148148,
      "grad_norm": 319.68548583984375,
      "learning_rate": 1.169139465875371e-06,
      "logits/chosen": -0.40234375,
      "logits/rejected": -0.427734375,
      "logps/chosen": -118.0,
      "logps/rejected": -120.0,
      "loss": 0.604,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02490234375,
      "rewards/margins": 0.2294921875,
      "rewards/rejected": -0.205078125,
      "step": 140
    },
    {
      "epoch": 0.4177777777777778,
      "grad_norm": 285.38104248046875,
      "learning_rate": 1.1632047477744807e-06,
      "logits/chosen": -0.416015625,
      "logits/rejected": -0.4609375,
      "logps/chosen": -102.0,
      "logps/rejected": -134.0,
      "loss": 0.5652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05078125,
      "rewards/margins": 0.34765625,
      "rewards/rejected": -0.296875,
      "step": 141
    },
    {
      "epoch": 0.42074074074074075,
      "grad_norm": 295.56475830078125,
      "learning_rate": 1.1572700296735904e-06,
      "logits/chosen": -0.4296875,
      "logits/rejected": -0.40625,
      "logps/chosen": -100.5,
      "logps/rejected": -97.5,
      "loss": 0.6649,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02197265625,
      "rewards/margins": 0.083984375,
      "rewards/rejected": -0.06201171875,
      "step": 142
    },
    {
      "epoch": 0.4237037037037037,
      "grad_norm": 323.8175354003906,
      "learning_rate": 1.1513353115727003e-06,
      "logits/chosen": -0.453125,
      "logits/rejected": -0.427734375,
      "logps/chosen": -136.0,
      "logps/rejected": -125.0,
      "loss": 0.6222,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.061767578125,
      "rewards/margins": 0.2041015625,
      "rewards/rejected": -0.142578125,
      "step": 143
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 314.3506164550781,
      "learning_rate": 1.14540059347181e-06,
      "logits/chosen": -0.4140625,
      "logits/rejected": -0.42578125,
      "logps/chosen": -115.5,
      "logps/rejected": -104.0,
      "loss": 0.6947,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.044677734375,
      "rewards/margins": 0.047607421875,
      "rewards/rejected": -0.09228515625,
      "step": 144
    },
    {
      "epoch": 0.42962962962962964,
      "grad_norm": 324.5817565917969,
      "learning_rate": 1.1394658753709198e-06,
      "logits/chosen": -0.419921875,
      "logits/rejected": -0.345703125,
      "logps/chosen": -121.0,
      "logps/rejected": -150.0,
      "loss": 0.6394,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.039794921875,
      "rewards/margins": 0.1982421875,
      "rewards/rejected": -0.158203125,
      "step": 145
    },
    {
      "epoch": 0.4325925925925926,
      "grad_norm": 314.7122497558594,
      "learning_rate": 1.1335311572700297e-06,
      "logits/chosen": -0.41796875,
      "logits/rejected": -0.41015625,
      "logps/chosen": -105.5,
      "logps/rejected": -96.0,
      "loss": 0.6624,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.034423828125,
      "rewards/margins": 0.09765625,
      "rewards/rejected": -0.06298828125,
      "step": 146
    },
    {
      "epoch": 0.43555555555555553,
      "grad_norm": 275.6307678222656,
      "learning_rate": 1.1275964391691395e-06,
      "logits/chosen": -0.423828125,
      "logits/rejected": -0.39453125,
      "logps/chosen": -93.0,
      "logps/rejected": -100.5,
      "loss": 0.5839,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.11083984375,
      "rewards/margins": 0.271484375,
      "rewards/rejected": -0.16015625,
      "step": 147
    },
    {
      "epoch": 0.43851851851851853,
      "grad_norm": 279.8615417480469,
      "learning_rate": 1.1216617210682492e-06,
      "logits/chosen": -0.41796875,
      "logits/rejected": -0.349609375,
      "logps/chosen": -97.5,
      "logps/rejected": -237.0,
      "loss": 0.523,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1181640625,
      "rewards/margins": 0.609375,
      "rewards/rejected": -0.490234375,
      "step": 148
    },
    {
      "epoch": 0.4414814814814815,
      "grad_norm": 311.0171813964844,
      "learning_rate": 1.1157270029673591e-06,
      "logits/chosen": -0.427734375,
      "logits/rejected": -0.400390625,
      "logps/chosen": -114.0,
      "logps/rejected": -134.0,
      "loss": 0.6284,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.04443359375,
      "rewards/margins": 0.1884765625,
      "rewards/rejected": -0.14453125,
      "step": 149
    },
    {
      "epoch": 0.4444444444444444,
      "grad_norm": 301.62060546875,
      "learning_rate": 1.1097922848664687e-06,
      "logits/chosen": -0.392578125,
      "logits/rejected": -0.392578125,
      "logps/chosen": -110.5,
      "logps/rejected": -112.0,
      "loss": 0.6028,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.12109375,
      "rewards/margins": 0.2314453125,
      "rewards/rejected": -0.1103515625,
      "step": 150
    },
    {
      "epoch": 0.4474074074074074,
      "grad_norm": 291.1073303222656,
      "learning_rate": 1.1038575667655786e-06,
      "logits/chosen": -0.400390625,
      "logits/rejected": -0.373046875,
      "logps/chosen": -91.5,
      "logps/rejected": -101.5,
      "loss": 0.6195,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11865234375,
      "rewards/margins": 0.18359375,
      "rewards/rejected": -0.06396484375,
      "step": 151
    },
    {
      "epoch": 0.45037037037037037,
      "grad_norm": 299.3499450683594,
      "learning_rate": 1.0979228486646883e-06,
      "logits/chosen": -0.408203125,
      "logits/rejected": -0.453125,
      "logps/chosen": -84.5,
      "logps/rejected": -128.0,
      "loss": 0.5974,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.0703125,
      "rewards/margins": 0.2451171875,
      "rewards/rejected": -0.1748046875,
      "step": 152
    },
    {
      "epoch": 0.4533333333333333,
      "grad_norm": 306.00897216796875,
      "learning_rate": 1.091988130563798e-06,
      "logits/chosen": -0.4140625,
      "logits/rejected": -0.451171875,
      "logps/chosen": -109.5,
      "logps/rejected": -103.0,
      "loss": 0.6441,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05859375,
      "rewards/margins": 0.115234375,
      "rewards/rejected": -0.056640625,
      "step": 153
    },
    {
      "epoch": 0.4562962962962963,
      "grad_norm": 285.1234436035156,
      "learning_rate": 1.086053412462908e-06,
      "logits/chosen": -0.421875,
      "logits/rejected": -0.3828125,
      "logps/chosen": -96.5,
      "logps/rejected": -147.0,
      "loss": 0.5937,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13671875,
      "rewards/margins": 0.33984375,
      "rewards/rejected": -0.2041015625,
      "step": 154
    },
    {
      "epoch": 0.45925925925925926,
      "grad_norm": 333.5130920410156,
      "learning_rate": 1.0801186943620177e-06,
      "logits/chosen": -0.427734375,
      "logits/rejected": -0.41796875,
      "logps/chosen": -115.5,
      "logps/rejected": -108.5,
      "loss": 0.6072,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0771484375,
      "rewards/margins": 0.21875,
      "rewards/rejected": -0.1416015625,
      "step": 155
    },
    {
      "epoch": 0.4622222222222222,
      "grad_norm": 327.6874084472656,
      "learning_rate": 1.0741839762611275e-06,
      "logits/chosen": -0.416015625,
      "logits/rejected": -0.4609375,
      "logps/chosen": -117.0,
      "logps/rejected": -145.0,
      "loss": 0.5776,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.0849609375,
      "rewards/margins": 0.310546875,
      "rewards/rejected": -0.2255859375,
      "step": 156
    },
    {
      "epoch": 0.4651851851851852,
      "grad_norm": 274.0838928222656,
      "learning_rate": 1.0682492581602374e-06,
      "logits/chosen": -0.447265625,
      "logits/rejected": -0.4609375,
      "logps/chosen": -94.5,
      "logps/rejected": -164.0,
      "loss": 0.5773,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.1064453125,
      "rewards/margins": 0.38671875,
      "rewards/rejected": -0.279296875,
      "step": 157
    },
    {
      "epoch": 0.46814814814814815,
      "grad_norm": 309.1082458496094,
      "learning_rate": 1.0623145400593471e-06,
      "logits/chosen": -0.46484375,
      "logits/rejected": -0.51953125,
      "logps/chosen": -107.0,
      "logps/rejected": -236.0,
      "loss": 0.5369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0849609375,
      "rewards/margins": 0.53125,
      "rewards/rejected": -0.443359375,
      "step": 158
    },
    {
      "epoch": 0.4711111111111111,
      "grad_norm": 328.0971374511719,
      "learning_rate": 1.056379821958457e-06,
      "logits/chosen": -0.392578125,
      "logits/rejected": -0.392578125,
      "logps/chosen": -122.0,
      "logps/rejected": -205.0,
      "loss": 0.5574,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1328125,
      "rewards/margins": 0.380859375,
      "rewards/rejected": -0.248046875,
      "step": 159
    },
    {
      "epoch": 0.4740740740740741,
      "grad_norm": 314.67706298828125,
      "learning_rate": 1.0504451038575668e-06,
      "logits/chosen": -0.421875,
      "logits/rejected": -0.40625,
      "logps/chosen": -131.0,
      "logps/rejected": -110.0,
      "loss": 0.6024,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.07470703125,
      "rewards/margins": 0.23828125,
      "rewards/rejected": -0.1630859375,
      "step": 160
    },
    {
      "epoch": 0.47703703703703704,
      "grad_norm": 365.208984375,
      "learning_rate": 1.0445103857566765e-06,
      "logits/chosen": -0.4609375,
      "logits/rejected": -0.4140625,
      "logps/chosen": -142.0,
      "logps/rejected": -162.0,
      "loss": 0.5747,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.053955078125,
      "rewards/margins": 0.310546875,
      "rewards/rejected": -0.2578125,
      "step": 161
    },
    {
      "epoch": 0.48,
      "grad_norm": 314.8697814941406,
      "learning_rate": 1.0385756676557865e-06,
      "logits/chosen": -0.388671875,
      "logits/rejected": -0.404296875,
      "logps/chosen": -96.0,
      "logps/rejected": -108.0,
      "loss": 0.6699,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.04931640625,
      "rewards/margins": 0.0869140625,
      "rewards/rejected": -0.03759765625,
      "step": 162
    },
    {
      "epoch": 0.482962962962963,
      "grad_norm": 297.8361511230469,
      "learning_rate": 1.032640949554896e-06,
      "logits/chosen": -0.435546875,
      "logits/rejected": -0.423828125,
      "logps/chosen": -114.0,
      "logps/rejected": -138.0,
      "loss": 0.6028,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.053955078125,
      "rewards/margins": 0.25390625,
      "rewards/rejected": -0.19921875,
      "step": 163
    },
    {
      "epoch": 0.48592592592592593,
      "grad_norm": 316.10369873046875,
      "learning_rate": 1.0267062314540057e-06,
      "logits/chosen": -0.44140625,
      "logits/rejected": -0.46875,
      "logps/chosen": -100.0,
      "logps/rejected": -119.0,
      "loss": 0.6105,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.0830078125,
      "rewards/margins": 0.2353515625,
      "rewards/rejected": -0.1533203125,
      "step": 164
    },
    {
      "epoch": 0.4888888888888889,
      "grad_norm": 282.8133239746094,
      "learning_rate": 1.0207715133531157e-06,
      "logits/chosen": -0.416015625,
      "logits/rejected": -0.447265625,
      "logps/chosen": -97.0,
      "logps/rejected": -152.0,
      "loss": 0.5733,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.072265625,
      "rewards/margins": 0.421875,
      "rewards/rejected": -0.349609375,
      "step": 165
    },
    {
      "epoch": 0.4918518518518519,
      "grad_norm": 299.11846923828125,
      "learning_rate": 1.0148367952522254e-06,
      "logits/chosen": -0.40625,
      "logits/rejected": -0.4453125,
      "logps/chosen": -104.5,
      "logps/rejected": -185.0,
      "loss": 0.6322,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00164794921875,
      "rewards/margins": 0.248046875,
      "rewards/rejected": -0.2470703125,
      "step": 166
    },
    {
      "epoch": 0.4948148148148148,
      "grad_norm": 327.81646728515625,
      "learning_rate": 1.0089020771513351e-06,
      "logits/chosen": -0.45703125,
      "logits/rejected": -0.4453125,
      "logps/chosen": -111.5,
      "logps/rejected": -143.0,
      "loss": 0.5774,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.052490234375,
      "rewards/margins": 0.30859375,
      "rewards/rejected": -0.255859375,
      "step": 167
    },
    {
      "epoch": 0.49777777777777776,
      "grad_norm": 342.8192443847656,
      "learning_rate": 1.002967359050445e-06,
      "logits/chosen": -0.3671875,
      "logits/rejected": -0.39453125,
      "logps/chosen": -102.5,
      "logps/rejected": -111.5,
      "loss": 0.5883,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.057861328125,
      "rewards/margins": 0.2451171875,
      "rewards/rejected": -0.1875,
      "step": 168
    },
    {
      "epoch": 0.5007407407407407,
      "grad_norm": 334.7403259277344,
      "learning_rate": 9.970326409495548e-07,
      "logits/chosen": -0.408203125,
      "logits/rejected": -0.427734375,
      "logps/chosen": -124.5,
      "logps/rejected": -124.5,
      "loss": 0.5725,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.0218505859375,
      "rewards/margins": 0.3125,
      "rewards/rejected": -0.2890625,
      "step": 169
    },
    {
      "epoch": 0.5037037037037037,
      "grad_norm": 285.1996154785156,
      "learning_rate": 9.910979228486648e-07,
      "logits/chosen": -0.421875,
      "logits/rejected": -0.458984375,
      "logps/chosen": -107.0,
      "logps/rejected": -109.5,
      "loss": 0.6043,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07275390625,
      "rewards/margins": 0.21875,
      "rewards/rejected": -0.1455078125,
      "step": 170
    },
    {
      "epoch": 0.5066666666666667,
      "grad_norm": 326.8195495605469,
      "learning_rate": 9.851632047477745e-07,
      "logits/chosen": -0.408203125,
      "logits/rejected": -0.43359375,
      "logps/chosen": -105.0,
      "logps/rejected": -107.5,
      "loss": 0.6083,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0888671875,
      "rewards/margins": 0.2080078125,
      "rewards/rejected": -0.11865234375,
      "step": 171
    },
    {
      "epoch": 0.5096296296296297,
      "grad_norm": 390.0325012207031,
      "learning_rate": 9.792284866468842e-07,
      "logits/chosen": -0.41015625,
      "logits/rejected": -0.3984375,
      "logps/chosen": -133.0,
      "logps/rejected": -104.5,
      "loss": 0.6678,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.00787353515625,
      "rewards/margins": 0.07666015625,
      "rewards/rejected": -0.08447265625,
      "step": 172
    },
    {
      "epoch": 0.5125925925925926,
      "grad_norm": 285.420166015625,
      "learning_rate": 9.73293768545994e-07,
      "logits/chosen": -0.4609375,
      "logits/rejected": -0.455078125,
      "logps/chosen": -101.5,
      "logps/rejected": -104.5,
      "loss": 0.5923,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1416015625,
      "rewards/margins": 0.2470703125,
      "rewards/rejected": -0.1044921875,
      "step": 173
    },
    {
      "epoch": 0.5155555555555555,
      "grad_norm": 377.0559387207031,
      "learning_rate": 9.67359050445104e-07,
      "logits/chosen": -0.412109375,
      "logits/rejected": -0.388671875,
      "logps/chosen": -104.5,
      "logps/rejected": -105.0,
      "loss": 0.6486,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.1455078125,
      "rewards/margins": 0.12060546875,
      "rewards/rejected": 0.0250244140625,
      "step": 174
    },
    {
      "epoch": 0.5185185185185185,
      "grad_norm": 348.386962890625,
      "learning_rate": 9.614243323442136e-07,
      "logits/chosen": -0.427734375,
      "logits/rejected": -0.4765625,
      "logps/chosen": -116.0,
      "logps/rejected": -126.5,
      "loss": 0.6291,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.05859375,
      "rewards/margins": 0.19140625,
      "rewards/rejected": -0.1328125,
      "step": 175
    },
    {
      "epoch": 0.5214814814814814,
      "grad_norm": 308.8468017578125,
      "learning_rate": 9.554896142433234e-07,
      "logits/chosen": -0.435546875,
      "logits/rejected": -0.419921875,
      "logps/chosen": -110.5,
      "logps/rejected": -105.0,
      "loss": 0.5789,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1279296875,
      "rewards/margins": 0.275390625,
      "rewards/rejected": -0.1474609375,
      "step": 176
    },
    {
      "epoch": 0.5244444444444445,
      "grad_norm": 309.856201171875,
      "learning_rate": 9.495548961424332e-07,
      "logits/chosen": -0.380859375,
      "logits/rejected": -0.44140625,
      "logps/chosen": -98.5,
      "logps/rejected": -133.0,
      "loss": 0.6226,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0986328125,
      "rewards/margins": 0.2265625,
      "rewards/rejected": -0.1279296875,
      "step": 177
    },
    {
      "epoch": 0.5274074074074074,
      "grad_norm": 296.9087219238281,
      "learning_rate": 9.43620178041543e-07,
      "logits/chosen": -0.3984375,
      "logits/rejected": -0.458984375,
      "logps/chosen": -103.5,
      "logps/rejected": -111.0,
      "loss": 0.5776,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.123046875,
      "rewards/margins": 0.29296875,
      "rewards/rejected": -0.169921875,
      "step": 178
    },
    {
      "epoch": 0.5303703703703704,
      "grad_norm": 284.62908935546875,
      "learning_rate": 9.376854599406528e-07,
      "logits/chosen": -0.453125,
      "logits/rejected": -0.470703125,
      "logps/chosen": -105.5,
      "logps/rejected": -131.0,
      "loss": 0.5454,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0771484375,
      "rewards/margins": 0.41015625,
      "rewards/rejected": -0.33203125,
      "step": 179
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 303.474853515625,
      "learning_rate": 9.317507418397625e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": -0.474609375,
      "logps/chosen": -106.5,
      "logps/rejected": -112.5,
      "loss": 0.5734,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.134765625,
      "rewards/margins": 0.3046875,
      "rewards/rejected": -0.169921875,
      "step": 180
    },
    {
      "epoch": 0.5362962962962963,
      "grad_norm": 296.1676025390625,
      "learning_rate": 9.258160237388723e-07,
      "logits/chosen": -0.423828125,
      "logits/rejected": -0.431640625,
      "logps/chosen": -110.5,
      "logps/rejected": -107.0,
      "loss": 0.5694,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.1181640625,
      "rewards/margins": 0.3359375,
      "rewards/rejected": -0.21875,
      "step": 181
    },
    {
      "epoch": 0.5392592592592592,
      "grad_norm": 275.0120849609375,
      "learning_rate": 9.198813056379822e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": -0.41796875,
      "logps/chosen": -94.0,
      "logps/rejected": -100.0,
      "loss": 0.6093,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1279296875,
      "rewards/margins": 0.2080078125,
      "rewards/rejected": -0.07958984375,
      "step": 182
    },
    {
      "epoch": 0.5422222222222223,
      "grad_norm": 323.9147033691406,
      "learning_rate": 9.13946587537092e-07,
      "logits/chosen": -0.45703125,
      "logits/rejected": -0.455078125,
      "logps/chosen": -125.5,
      "logps/rejected": -101.5,
      "loss": 0.5729,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1220703125,
      "rewards/margins": 0.28515625,
      "rewards/rejected": -0.1630859375,
      "step": 183
    },
    {
      "epoch": 0.5451851851851852,
      "grad_norm": 307.3267517089844,
      "learning_rate": 9.080118694362017e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": -0.5,
      "logps/chosen": -98.0,
      "logps/rejected": -149.0,
      "loss": 0.5919,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.08837890625,
      "rewards/margins": 0.318359375,
      "rewards/rejected": -0.2294921875,
      "step": 184
    },
    {
      "epoch": 0.5481481481481482,
      "grad_norm": 321.1439514160156,
      "learning_rate": 9.020771513353115e-07,
      "logits/chosen": -0.384765625,
      "logits/rejected": -0.431640625,
      "logps/chosen": -95.5,
      "logps/rejected": -104.5,
      "loss": 0.6293,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.11572265625,
      "rewards/margins": 0.1962890625,
      "rewards/rejected": -0.08056640625,
      "step": 185
    },
    {
      "epoch": 0.5511111111111111,
      "grad_norm": 328.7718200683594,
      "learning_rate": 8.961424332344213e-07,
      "logits/chosen": -0.41015625,
      "logits/rejected": -0.404296875,
      "logps/chosen": -124.0,
      "logps/rejected": -109.0,
      "loss": 0.6006,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.125,
      "rewards/margins": 0.248046875,
      "rewards/rejected": -0.12353515625,
      "step": 186
    },
    {
      "epoch": 0.554074074074074,
      "grad_norm": 342.34442138671875,
      "learning_rate": 8.902077151335311e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": -0.40234375,
      "logps/chosen": -116.0,
      "logps/rejected": -117.5,
      "loss": 0.6102,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.033203125,
      "rewards/margins": 0.2275390625,
      "rewards/rejected": -0.1943359375,
      "step": 187
    },
    {
      "epoch": 0.557037037037037,
      "grad_norm": 340.3980712890625,
      "learning_rate": 8.842729970326409e-07,
      "logits/chosen": -0.396484375,
      "logits/rejected": -0.38671875,
      "logps/chosen": -122.5,
      "logps/rejected": -111.0,
      "loss": 0.5869,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0576171875,
      "rewards/margins": 0.287109375,
      "rewards/rejected": -0.2294921875,
      "step": 188
    },
    {
      "epoch": 0.56,
      "grad_norm": 355.1933288574219,
      "learning_rate": 8.783382789317507e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": -0.421875,
      "logps/chosen": -118.5,
      "logps/rejected": -102.0,
      "loss": 0.6305,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0732421875,
      "rewards/margins": 0.169921875,
      "rewards/rejected": -0.09619140625,
      "step": 189
    },
    {
      "epoch": 0.562962962962963,
      "grad_norm": 286.2369689941406,
      "learning_rate": 8.724035608308605e-07,
      "logits/chosen": -0.4453125,
      "logits/rejected": -0.494140625,
      "logps/chosen": -111.0,
      "logps/rejected": -137.0,
      "loss": 0.5476,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.173828125,
      "rewards/margins": 0.37890625,
      "rewards/rejected": -0.2041015625,
      "step": 190
    },
    {
      "epoch": 0.5659259259259259,
      "grad_norm": 362.5728759765625,
      "learning_rate": 8.664688427299704e-07,
      "logits/chosen": -0.373046875,
      "logits/rejected": -0.44140625,
      "logps/chosen": -101.0,
      "logps/rejected": -130.0,
      "loss": 0.5947,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.06103515625,
      "rewards/margins": 0.259765625,
      "rewards/rejected": -0.19921875,
      "step": 191
    },
    {
      "epoch": 0.5688888888888889,
      "grad_norm": 316.38043212890625,
      "learning_rate": 8.6053412462908e-07,
      "logits/chosen": -0.388671875,
      "logits/rejected": -0.41796875,
      "logps/chosen": -93.5,
      "logps/rejected": -99.5,
      "loss": 0.6141,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0869140625,
      "rewards/margins": 0.2119140625,
      "rewards/rejected": -0.125,
      "step": 192
    },
    {
      "epoch": 0.5718518518518518,
      "grad_norm": 298.670166015625,
      "learning_rate": 8.545994065281898e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": -0.447265625,
      "logps/chosen": -101.0,
      "logps/rejected": -122.0,
      "loss": 0.5581,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.197265625,
      "rewards/margins": 0.365234375,
      "rewards/rejected": -0.166015625,
      "step": 193
    },
    {
      "epoch": 0.5748148148148148,
      "grad_norm": 331.94580078125,
      "learning_rate": 8.486646884272997e-07,
      "logits/chosen": -0.439453125,
      "logits/rejected": -0.44921875,
      "logps/chosen": -107.0,
      "logps/rejected": -160.0,
      "loss": 0.5891,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.08203125,
      "rewards/margins": 0.34765625,
      "rewards/rejected": -0.265625,
      "step": 194
    },
    {
      "epoch": 0.5777777777777777,
      "grad_norm": 312.4453125,
      "learning_rate": 8.427299703264095e-07,
      "logits/chosen": -0.41796875,
      "logits/rejected": -0.458984375,
      "logps/chosen": -96.5,
      "logps/rejected": -117.5,
      "loss": 0.5737,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1201171875,
      "rewards/margins": 0.306640625,
      "rewards/rejected": -0.1865234375,
      "step": 195
    },
    {
      "epoch": 0.5807407407407408,
      "grad_norm": 300.3534240722656,
      "learning_rate": 8.367952522255193e-07,
      "logits/chosen": -0.435546875,
      "logits/rejected": -0.4609375,
      "logps/chosen": -110.5,
      "logps/rejected": -131.0,
      "loss": 0.5817,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.038330078125,
      "rewards/margins": 0.333984375,
      "rewards/rejected": -0.296875,
      "step": 196
    },
    {
      "epoch": 0.5837037037037037,
      "grad_norm": 319.2505798339844,
      "learning_rate": 8.308605341246291e-07,
      "logits/chosen": -0.416015625,
      "logits/rejected": -0.46875,
      "logps/chosen": -96.0,
      "logps/rejected": -107.0,
      "loss": 0.5994,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1171875,
      "rewards/margins": 0.25390625,
      "rewards/rejected": -0.13671875,
      "step": 197
    },
    {
      "epoch": 0.5866666666666667,
      "grad_norm": 315.4450988769531,
      "learning_rate": 8.249258160237388e-07,
      "logits/chosen": -0.44140625,
      "logits/rejected": -0.4765625,
      "logps/chosen": -115.0,
      "logps/rejected": -114.5,
      "loss": 0.5748,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0986328125,
      "rewards/margins": 0.310546875,
      "rewards/rejected": -0.212890625,
      "step": 198
    },
    {
      "epoch": 0.5896296296296296,
      "grad_norm": 303.500732421875,
      "learning_rate": 8.189910979228485e-07,
      "logits/chosen": -0.453125,
      "logits/rejected": -0.48828125,
      "logps/chosen": -108.0,
      "logps/rejected": -122.5,
      "loss": 0.5804,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.140625,
      "rewards/margins": 0.29296875,
      "rewards/rejected": -0.15234375,
      "step": 199
    },
    {
      "epoch": 0.5925925925925926,
      "grad_norm": 319.0402526855469,
      "learning_rate": 8.130563798219584e-07,
      "logits/chosen": -0.48046875,
      "logits/rejected": -0.474609375,
      "logps/chosen": -113.5,
      "logps/rejected": -111.0,
      "loss": 0.6036,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.11865234375,
      "rewards/margins": 0.2421875,
      "rewards/rejected": -0.12353515625,
      "step": 200
    },
    {
      "epoch": 0.5955555555555555,
      "grad_norm": 320.802490234375,
      "learning_rate": 8.071216617210682e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": -0.396484375,
      "logps/chosen": -97.5,
      "logps/rejected": -107.5,
      "loss": 0.5582,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.134765625,
      "rewards/margins": 0.365234375,
      "rewards/rejected": -0.23046875,
      "step": 201
    },
    {
      "epoch": 0.5985185185185186,
      "grad_norm": 341.16033935546875,
      "learning_rate": 8.011869436201781e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": -0.453125,
      "logps/chosen": -110.0,
      "logps/rejected": -103.0,
      "loss": 0.6088,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0289306640625,
      "rewards/margins": 0.2021484375,
      "rewards/rejected": -0.173828125,
      "step": 202
    },
    {
      "epoch": 0.6014814814814815,
      "grad_norm": 339.4845275878906,
      "learning_rate": 7.952522255192878e-07,
      "logits/chosen": -0.453125,
      "logits/rejected": -0.447265625,
      "logps/chosen": -124.5,
      "logps/rejected": -141.0,
      "loss": 0.5481,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.146484375,
      "rewards/margins": 0.3671875,
      "rewards/rejected": -0.2216796875,
      "step": 203
    },
    {
      "epoch": 0.6044444444444445,
      "grad_norm": 288.8636169433594,
      "learning_rate": 7.893175074183976e-07,
      "logits/chosen": -0.44921875,
      "logits/rejected": -0.494140625,
      "logps/chosen": -99.0,
      "logps/rejected": -138.0,
      "loss": 0.5414,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1494140625,
      "rewards/margins": 0.44140625,
      "rewards/rejected": -0.291015625,
      "step": 204
    },
    {
      "epoch": 0.6074074074074074,
      "grad_norm": 297.95355224609375,
      "learning_rate": 7.833827893175074e-07,
      "logits/chosen": -0.447265625,
      "logits/rejected": -0.4453125,
      "logps/chosen": -105.5,
      "logps/rejected": -147.0,
      "loss": 0.5584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.150390625,
      "rewards/margins": 0.43359375,
      "rewards/rejected": -0.283203125,
      "step": 205
    },
    {
      "epoch": 0.6103703703703703,
      "grad_norm": 259.13623046875,
      "learning_rate": 7.774480712166172e-07,
      "logits/chosen": -0.4921875,
      "logits/rejected": -0.470703125,
      "logps/chosen": -107.0,
      "logps/rejected": -213.0,
      "loss": 0.4939,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1640625,
      "rewards/margins": 0.6796875,
      "rewards/rejected": -0.515625,
      "step": 206
    },
    {
      "epoch": 0.6133333333333333,
      "grad_norm": 352.1375732421875,
      "learning_rate": 7.715133531157269e-07,
      "logits/chosen": -0.416015625,
      "logits/rejected": -0.42578125,
      "logps/chosen": -106.0,
      "logps/rejected": -113.5,
      "loss": 0.6303,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.036865234375,
      "rewards/margins": 0.162109375,
      "rewards/rejected": -0.1259765625,
      "step": 207
    },
    {
      "epoch": 0.6162962962962963,
      "grad_norm": 309.4278869628906,
      "learning_rate": 7.655786350148368e-07,
      "logits/chosen": -0.455078125,
      "logits/rejected": -0.48046875,
      "logps/chosen": -121.5,
      "logps/rejected": -150.0,
      "loss": 0.5443,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08740234375,
      "rewards/margins": 0.3984375,
      "rewards/rejected": -0.3125,
      "step": 208
    },
    {
      "epoch": 0.6192592592592593,
      "grad_norm": 302.8489990234375,
      "learning_rate": 7.596439169139466e-07,
      "logits/chosen": -0.416015625,
      "logits/rejected": -0.423828125,
      "logps/chosen": -94.5,
      "logps/rejected": -106.0,
      "loss": 0.5962,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1259765625,
      "rewards/margins": 0.28125,
      "rewards/rejected": -0.1552734375,
      "step": 209
    },
    {
      "epoch": 0.6222222222222222,
      "grad_norm": 315.8456115722656,
      "learning_rate": 7.537091988130564e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": -0.443359375,
      "logps/chosen": -108.5,
      "logps/rejected": -136.0,
      "loss": 0.5422,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1064453125,
      "rewards/margins": 0.42578125,
      "rewards/rejected": -0.3203125,
      "step": 210
    },
    {
      "epoch": 0.6251851851851852,
      "grad_norm": 327.0802307128906,
      "learning_rate": 7.477744807121661e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": -0.447265625,
      "logps/chosen": -97.5,
      "logps/rejected": -109.0,
      "loss": 0.5616,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.08056640625,
      "rewards/margins": 0.3359375,
      "rewards/rejected": -0.255859375,
      "step": 211
    },
    {
      "epoch": 0.6281481481481481,
      "grad_norm": 317.4538879394531,
      "learning_rate": 7.418397626112759e-07,
      "logits/chosen": -0.46484375,
      "logits/rejected": -0.486328125,
      "logps/chosen": -106.5,
      "logps/rejected": -114.0,
      "loss": 0.5953,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.17578125,
      "rewards/margins": 0.26953125,
      "rewards/rejected": -0.09375,
      "step": 212
    },
    {
      "epoch": 0.6311111111111111,
      "grad_norm": 334.5902099609375,
      "learning_rate": 7.359050445103857e-07,
      "logits/chosen": -0.435546875,
      "logits/rejected": -0.4921875,
      "logps/chosen": -110.5,
      "logps/rejected": -127.0,
      "loss": 0.55,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.1904296875,
      "rewards/margins": 0.423828125,
      "rewards/rejected": -0.2333984375,
      "step": 213
    },
    {
      "epoch": 0.6340740740740741,
      "grad_norm": 275.5820007324219,
      "learning_rate": 7.299703264094955e-07,
      "logits/chosen": -0.4140625,
      "logits/rejected": -0.45703125,
      "logps/chosen": -85.5,
      "logps/rejected": -114.5,
      "loss": 0.5648,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1376953125,
      "rewards/margins": 0.416015625,
      "rewards/rejected": -0.27734375,
      "step": 214
    },
    {
      "epoch": 0.6370370370370371,
      "grad_norm": 326.2808532714844,
      "learning_rate": 7.240356083086053e-07,
      "logits/chosen": -0.423828125,
      "logits/rejected": -0.4609375,
      "logps/chosen": -101.5,
      "logps/rejected": -113.0,
      "loss": 0.6071,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.0830078125,
      "rewards/margins": 0.2333984375,
      "rewards/rejected": -0.150390625,
      "step": 215
    },
    {
      "epoch": 0.64,
      "grad_norm": 345.86163330078125,
      "learning_rate": 7.181008902077151e-07,
      "logits/chosen": -0.455078125,
      "logits/rejected": -0.494140625,
      "logps/chosen": -117.0,
      "logps/rejected": -119.0,
      "loss": 0.5734,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.024169921875,
      "rewards/margins": 0.314453125,
      "rewards/rejected": -0.291015625,
      "step": 216
    },
    {
      "epoch": 0.642962962962963,
      "grad_norm": 302.5231018066406,
      "learning_rate": 7.12166172106825e-07,
      "logits/chosen": -0.44140625,
      "logits/rejected": -0.4453125,
      "logps/chosen": -113.5,
      "logps/rejected": -122.0,
      "loss": 0.55,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1123046875,
      "rewards/margins": 0.38671875,
      "rewards/rejected": -0.2734375,
      "step": 217
    },
    {
      "epoch": 0.6459259259259259,
      "grad_norm": 321.6429443359375,
      "learning_rate": 7.062314540059346e-07,
      "logits/chosen": -0.416015625,
      "logits/rejected": -0.431640625,
      "logps/chosen": -114.0,
      "logps/rejected": -110.5,
      "loss": 0.6326,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0810546875,
      "rewards/margins": 0.185546875,
      "rewards/rejected": -0.1044921875,
      "step": 218
    },
    {
      "epoch": 0.6488888888888888,
      "grad_norm": 321.2485656738281,
      "learning_rate": 7.002967359050444e-07,
      "logits/chosen": -0.40234375,
      "logits/rejected": -0.4609375,
      "logps/chosen": -104.0,
      "logps/rejected": -146.0,
      "loss": 0.5269,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.09228515625,
      "rewards/margins": 0.453125,
      "rewards/rejected": -0.361328125,
      "step": 219
    },
    {
      "epoch": 0.6518518518518519,
      "grad_norm": 351.099609375,
      "learning_rate": 6.943620178041543e-07,
      "logits/chosen": -0.484375,
      "logits/rejected": -0.4609375,
      "logps/chosen": -125.5,
      "logps/rejected": -114.0,
      "loss": 0.585,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11962890625,
      "rewards/margins": 0.333984375,
      "rewards/rejected": -0.21484375,
      "step": 220
    },
    {
      "epoch": 0.6548148148148148,
      "grad_norm": 319.5416259765625,
      "learning_rate": 6.884272997032641e-07,
      "logits/chosen": -0.41796875,
      "logits/rejected": -0.423828125,
      "logps/chosen": -94.0,
      "logps/rejected": -102.5,
      "loss": 0.5757,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.173828125,
      "rewards/margins": 0.3125,
      "rewards/rejected": -0.138671875,
      "step": 221
    },
    {
      "epoch": 0.6577777777777778,
      "grad_norm": 315.0635681152344,
      "learning_rate": 6.824925816023738e-07,
      "logits/chosen": -0.455078125,
      "logits/rejected": -0.486328125,
      "logps/chosen": -99.5,
      "logps/rejected": -109.5,
      "loss": 0.5692,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1181640625,
      "rewards/margins": 0.328125,
      "rewards/rejected": -0.2099609375,
      "step": 222
    },
    {
      "epoch": 0.6607407407407407,
      "grad_norm": 403.2724304199219,
      "learning_rate": 6.765578635014837e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": -0.490234375,
      "logps/chosen": -134.0,
      "logps/rejected": -116.5,
      "loss": 0.6002,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0771484375,
      "rewards/margins": 0.30859375,
      "rewards/rejected": -0.232421875,
      "step": 223
    },
    {
      "epoch": 0.6637037037037037,
      "grad_norm": 296.415771484375,
      "learning_rate": 6.706231454005934e-07,
      "logits/chosen": -0.46875,
      "logits/rejected": -0.5,
      "logps/chosen": -110.5,
      "logps/rejected": -144.0,
      "loss": 0.5433,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.11279296875,
      "rewards/margins": 0.48828125,
      "rewards/rejected": -0.376953125,
      "step": 224
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 289.037841796875,
      "learning_rate": 6.646884272997032e-07,
      "logits/chosen": -0.486328125,
      "logits/rejected": -0.498046875,
      "logps/chosen": -125.0,
      "logps/rejected": -127.0,
      "loss": 0.5747,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.1298828125,
      "rewards/margins": 0.306640625,
      "rewards/rejected": -0.17578125,
      "step": 225
    },
    {
      "epoch": 0.6696296296296296,
      "grad_norm": 308.6339111328125,
      "learning_rate": 6.58753709198813e-07,
      "logits/chosen": -0.45703125,
      "logits/rejected": -0.470703125,
      "logps/chosen": -89.0,
      "logps/rejected": -125.0,
      "loss": 0.5129,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.302734375,
      "rewards/margins": 0.470703125,
      "rewards/rejected": -0.1689453125,
      "step": 226
    },
    {
      "epoch": 0.6725925925925926,
      "grad_norm": 345.19512939453125,
      "learning_rate": 6.528189910979228e-07,
      "logits/chosen": -0.498046875,
      "logits/rejected": -0.50390625,
      "logps/chosen": -122.0,
      "logps/rejected": -111.0,
      "loss": 0.6402,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0673828125,
      "rewards/margins": 0.16796875,
      "rewards/rejected": -0.10107421875,
      "step": 227
    },
    {
      "epoch": 0.6755555555555556,
      "grad_norm": 309.3340148925781,
      "learning_rate": 6.468842729970327e-07,
      "logits/chosen": -0.478515625,
      "logits/rejected": -0.46484375,
      "logps/chosen": -101.0,
      "logps/rejected": -105.5,
      "loss": 0.5455,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.19921875,
      "rewards/margins": 0.369140625,
      "rewards/rejected": -0.1689453125,
      "step": 228
    },
    {
      "epoch": 0.6785185185185185,
      "grad_norm": 342.4759216308594,
      "learning_rate": 6.409495548961425e-07,
      "logits/chosen": -0.484375,
      "logits/rejected": -0.4765625,
      "logps/chosen": -124.5,
      "logps/rejected": -125.5,
      "loss": 0.5702,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.1689453125,
      "rewards/margins": 0.318359375,
      "rewards/rejected": -0.150390625,
      "step": 229
    },
    {
      "epoch": 0.6814814814814815,
      "grad_norm": 359.9955749511719,
      "learning_rate": 6.350148367952522e-07,
      "logits/chosen": -0.46484375,
      "logits/rejected": -0.50390625,
      "logps/chosen": -116.5,
      "logps/rejected": -132.0,
      "loss": 0.5958,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.1533203125,
      "rewards/margins": 0.298828125,
      "rewards/rejected": -0.1455078125,
      "step": 230
    },
    {
      "epoch": 0.6844444444444444,
      "grad_norm": 413.5058898925781,
      "learning_rate": 6.29080118694362e-07,
      "logits/chosen": -0.46875,
      "logits/rejected": -0.46484375,
      "logps/chosen": -123.0,
      "logps/rejected": -115.5,
      "loss": 0.6223,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.0625,
      "rewards/margins": 0.236328125,
      "rewards/rejected": -0.173828125,
      "step": 231
    },
    {
      "epoch": 0.6874074074074074,
      "grad_norm": 396.54669189453125,
      "learning_rate": 6.231454005934718e-07,
      "logits/chosen": -0.455078125,
      "logits/rejected": -0.44921875,
      "logps/chosen": -109.0,
      "logps/rejected": -109.0,
      "loss": 0.6149,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1845703125,
      "rewards/margins": 0.220703125,
      "rewards/rejected": -0.03515625,
      "step": 232
    },
    {
      "epoch": 0.6903703703703704,
      "grad_norm": 324.2456359863281,
      "learning_rate": 6.172106824925815e-07,
      "logits/chosen": -0.48046875,
      "logits/rejected": -0.466796875,
      "logps/chosen": -121.0,
      "logps/rejected": -114.5,
      "loss": 0.6149,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.130859375,
      "rewards/margins": 0.193359375,
      "rewards/rejected": -0.0625,
      "step": 233
    },
    {
      "epoch": 0.6933333333333334,
      "grad_norm": 378.22955322265625,
      "learning_rate": 6.112759643916914e-07,
      "logits/chosen": -0.474609375,
      "logits/rejected": -0.474609375,
      "logps/chosen": -126.0,
      "logps/rejected": -117.0,
      "loss": 0.6306,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0986328125,
      "rewards/margins": 0.1865234375,
      "rewards/rejected": -0.08740234375,
      "step": 234
    },
    {
      "epoch": 0.6962962962962963,
      "grad_norm": 286.87713623046875,
      "learning_rate": 6.053412462908012e-07,
      "logits/chosen": -0.439453125,
      "logits/rejected": -0.46875,
      "logps/chosen": -96.0,
      "logps/rejected": -106.0,
      "loss": 0.5373,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.216796875,
      "rewards/margins": 0.412109375,
      "rewards/rejected": -0.1953125,
      "step": 235
    },
    {
      "epoch": 0.6992592592592592,
      "grad_norm": 287.8903503417969,
      "learning_rate": 5.99406528189911e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": -0.435546875,
      "logps/chosen": -100.0,
      "logps/rejected": -167.0,
      "loss": 0.5497,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1533203125,
      "rewards/margins": 0.5546875,
      "rewards/rejected": -0.404296875,
      "step": 236
    },
    {
      "epoch": 0.7022222222222222,
      "grad_norm": 389.0058288574219,
      "learning_rate": 5.934718100890207e-07,
      "logits/chosen": -0.5078125,
      "logits/rejected": -0.458984375,
      "logps/chosen": -115.5,
      "logps/rejected": -118.5,
      "loss": 0.5813,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1435546875,
      "rewards/margins": 0.291015625,
      "rewards/rejected": -0.1474609375,
      "step": 237
    },
    {
      "epoch": 0.7051851851851851,
      "grad_norm": 322.4640197753906,
      "learning_rate": 5.875370919881305e-07,
      "logits/chosen": -0.455078125,
      "logits/rejected": -0.48046875,
      "logps/chosen": -116.5,
      "logps/rejected": -127.5,
      "loss": 0.5622,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12890625,
      "rewards/margins": 0.3671875,
      "rewards/rejected": -0.23828125,
      "step": 238
    },
    {
      "epoch": 0.7081481481481482,
      "grad_norm": 356.6217346191406,
      "learning_rate": 5.816023738872403e-07,
      "logits/chosen": -0.50390625,
      "logits/rejected": -0.4765625,
      "logps/chosen": -134.0,
      "logps/rejected": -116.5,
      "loss": 0.5994,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.15234375,
      "rewards/margins": 0.2431640625,
      "rewards/rejected": -0.0908203125,
      "step": 239
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 352.3586120605469,
      "learning_rate": 5.756676557863502e-07,
      "logits/chosen": -0.421875,
      "logits/rejected": -0.453125,
      "logps/chosen": -120.0,
      "logps/rejected": -137.0,
      "loss": 0.557,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.1376953125,
      "rewards/margins": 0.404296875,
      "rewards/rejected": -0.267578125,
      "step": 240
    },
    {
      "epoch": 0.7140740740740741,
      "grad_norm": 319.5504455566406,
      "learning_rate": 5.697329376854599e-07,
      "logits/chosen": -0.419921875,
      "logits/rejected": -0.46875,
      "logps/chosen": -108.0,
      "logps/rejected": -166.0,
      "loss": 0.4836,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.287109375,
      "rewards/margins": 0.63671875,
      "rewards/rejected": -0.349609375,
      "step": 241
    },
    {
      "epoch": 0.717037037037037,
      "grad_norm": 279.9657287597656,
      "learning_rate": 5.637982195845697e-07,
      "logits/chosen": -0.443359375,
      "logits/rejected": -0.478515625,
      "logps/chosen": -102.0,
      "logps/rejected": -115.0,
      "loss": 0.5181,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21484375,
      "rewards/margins": 0.4609375,
      "rewards/rejected": -0.24609375,
      "step": 242
    },
    {
      "epoch": 0.72,
      "grad_norm": 352.0918273925781,
      "learning_rate": 5.578635014836796e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": -0.416015625,
      "logps/chosen": -103.5,
      "logps/rejected": -174.0,
      "loss": 0.6038,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.09619140625,
      "rewards/margins": 0.375,
      "rewards/rejected": -0.279296875,
      "step": 243
    },
    {
      "epoch": 0.7229629629629629,
      "grad_norm": 338.6324157714844,
      "learning_rate": 5.519287833827893e-07,
      "logits/chosen": -0.486328125,
      "logits/rejected": -0.53125,
      "logps/chosen": -115.0,
      "logps/rejected": -143.0,
      "loss": 0.5329,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17578125,
      "rewards/margins": 0.44921875,
      "rewards/rejected": -0.2734375,
      "step": 244
    },
    {
      "epoch": 0.725925925925926,
      "grad_norm": 355.90570068359375,
      "learning_rate": 5.45994065281899e-07,
      "logits/chosen": -0.421875,
      "logits/rejected": -0.44921875,
      "logps/chosen": -122.5,
      "logps/rejected": -116.0,
      "loss": 0.6306,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.10302734375,
      "rewards/margins": 0.1962890625,
      "rewards/rejected": -0.0927734375,
      "step": 245
    },
    {
      "epoch": 0.7288888888888889,
      "grad_norm": 315.7834777832031,
      "learning_rate": 5.400593471810089e-07,
      "logits/chosen": -0.458984375,
      "logits/rejected": -0.48046875,
      "logps/chosen": -116.5,
      "logps/rejected": -216.0,
      "loss": 0.5527,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1572265625,
      "rewards/margins": 0.486328125,
      "rewards/rejected": -0.330078125,
      "step": 246
    },
    {
      "epoch": 0.7318518518518519,
      "grad_norm": 355.6773376464844,
      "learning_rate": 5.341246290801187e-07,
      "logits/chosen": -0.4140625,
      "logits/rejected": -0.4765625,
      "logps/chosen": -99.5,
      "logps/rejected": -128.0,
      "loss": 0.5726,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.25390625,
      "rewards/margins": 0.34765625,
      "rewards/rejected": -0.09375,
      "step": 247
    },
    {
      "epoch": 0.7348148148148148,
      "grad_norm": 345.4238586425781,
      "learning_rate": 5.281899109792285e-07,
      "logits/chosen": -0.44140625,
      "logits/rejected": -0.482421875,
      "logps/chosen": -114.5,
      "logps/rejected": -103.0,
      "loss": 0.5754,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1767578125,
      "rewards/margins": 0.326171875,
      "rewards/rejected": -0.150390625,
      "step": 248
    },
    {
      "epoch": 0.7377777777777778,
      "grad_norm": 347.1054992675781,
      "learning_rate": 5.222551928783383e-07,
      "logits/chosen": -0.484375,
      "logits/rejected": -0.48828125,
      "logps/chosen": -117.0,
      "logps/rejected": -155.0,
      "loss": 0.5783,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1904296875,
      "rewards/margins": 0.419921875,
      "rewards/rejected": -0.2275390625,
      "step": 249
    },
    {
      "epoch": 0.7407407407407407,
      "grad_norm": 320.8048400878906,
      "learning_rate": 5.16320474777448e-07,
      "logits/chosen": -0.416015625,
      "logits/rejected": -0.462890625,
      "logps/chosen": -97.0,
      "logps/rejected": -112.5,
      "loss": 0.5957,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.18359375,
      "rewards/margins": 0.283203125,
      "rewards/rejected": -0.10009765625,
      "step": 250
    },
    {
      "epoch": 0.7437037037037038,
      "grad_norm": 338.7988586425781,
      "learning_rate": 5.103857566765578e-07,
      "logits/chosen": -0.447265625,
      "logits/rejected": -0.453125,
      "logps/chosen": -102.5,
      "logps/rejected": -118.0,
      "loss": 0.5697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.27734375,
      "rewards/margins": 0.361328125,
      "rewards/rejected": -0.08349609375,
      "step": 251
    },
    {
      "epoch": 0.7466666666666667,
      "grad_norm": 294.05718994140625,
      "learning_rate": 5.044510385756676e-07,
      "logits/chosen": -0.42578125,
      "logits/rejected": -0.478515625,
      "logps/chosen": -105.0,
      "logps/rejected": -107.5,
      "loss": 0.5615,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.162109375,
      "rewards/margins": 0.3515625,
      "rewards/rejected": -0.1904296875,
      "step": 252
    },
    {
      "epoch": 0.7496296296296296,
      "grad_norm": 338.3261413574219,
      "learning_rate": 4.985163204747774e-07,
      "logits/chosen": -0.474609375,
      "logits/rejected": -0.470703125,
      "logps/chosen": -115.5,
      "logps/rejected": -118.0,
      "loss": 0.5705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.27734375,
      "rewards/margins": 0.35546875,
      "rewards/rejected": -0.07958984375,
      "step": 253
    },
    {
      "epoch": 0.7525925925925926,
      "grad_norm": 303.49481201171875,
      "learning_rate": 4.925816023738872e-07,
      "logits/chosen": -0.4765625,
      "logits/rejected": -0.4765625,
      "logps/chosen": -105.0,
      "logps/rejected": -107.0,
      "loss": 0.5983,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.236328125,
      "rewards/margins": 0.251953125,
      "rewards/rejected": -0.0166015625,
      "step": 254
    },
    {
      "epoch": 0.7555555555555555,
      "grad_norm": 314.61431884765625,
      "learning_rate": 4.86646884272997e-07,
      "logits/chosen": -0.43359375,
      "logits/rejected": -0.384765625,
      "logps/chosen": -108.0,
      "logps/rejected": -248.0,
      "loss": 0.5377,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.140625,
      "rewards/margins": 0.74609375,
      "rewards/rejected": -0.60546875,
      "step": 255
    },
    {
      "epoch": 0.7585185185185185,
      "grad_norm": 338.1864318847656,
      "learning_rate": 4.807121661721068e-07,
      "logits/chosen": -0.453125,
      "logits/rejected": -0.470703125,
      "logps/chosen": -102.0,
      "logps/rejected": -111.5,
      "loss": 0.655,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1650390625,
      "rewards/margins": 0.12353515625,
      "rewards/rejected": 0.04150390625,
      "step": 256
    },
    {
      "epoch": 0.7614814814814815,
      "grad_norm": 338.2643127441406,
      "learning_rate": 4.747774480712166e-07,
      "logits/chosen": -0.443359375,
      "logits/rejected": -0.447265625,
      "logps/chosen": -118.5,
      "logps/rejected": -122.0,
      "loss": 0.56,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.2578125,
      "rewards/margins": 0.39453125,
      "rewards/rejected": -0.13671875,
      "step": 257
    },
    {
      "epoch": 0.7644444444444445,
      "grad_norm": 358.0396423339844,
      "learning_rate": 4.688427299703264e-07,
      "logits/chosen": -0.4609375,
      "logits/rejected": -0.4921875,
      "logps/chosen": -123.5,
      "logps/rejected": -131.0,
      "loss": 0.5951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.173828125,
      "rewards/margins": 0.31640625,
      "rewards/rejected": -0.1435546875,
      "step": 258
    },
    {
      "epoch": 0.7674074074074074,
      "grad_norm": 310.6124267578125,
      "learning_rate": 4.6290801186943617e-07,
      "logits/chosen": -0.5,
      "logits/rejected": -0.51171875,
      "logps/chosen": -91.5,
      "logps/rejected": -120.5,
      "loss": 0.5808,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.2177734375,
      "rewards/margins": 0.357421875,
      "rewards/rejected": -0.138671875,
      "step": 259
    },
    {
      "epoch": 0.7703703703703704,
      "grad_norm": 335.0301513671875,
      "learning_rate": 4.56973293768546e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": -0.451171875,
      "logps/chosen": -87.5,
      "logps/rejected": -109.0,
      "loss": 0.5374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.29296875,
      "rewards/margins": 0.40234375,
      "rewards/rejected": -0.1083984375,
      "step": 260
    },
    {
      "epoch": 0.7733333333333333,
      "grad_norm": 335.72308349609375,
      "learning_rate": 4.5103857566765573e-07,
      "logits/chosen": -0.404296875,
      "logits/rejected": -0.427734375,
      "logps/chosen": -90.5,
      "logps/rejected": -96.5,
      "loss": 0.6355,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.1826171875,
      "rewards/margins": 0.166015625,
      "rewards/rejected": 0.0164794921875,
      "step": 261
    },
    {
      "epoch": 0.7762962962962963,
      "grad_norm": 341.7281799316406,
      "learning_rate": 4.4510385756676557e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": -0.4453125,
      "logps/chosen": -108.0,
      "logps/rejected": -99.0,
      "loss": 0.6155,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.15234375,
      "rewards/margins": 0.208984375,
      "rewards/rejected": -0.05615234375,
      "step": 262
    },
    {
      "epoch": 0.7792592592592592,
      "grad_norm": 344.0790710449219,
      "learning_rate": 4.3916913946587536e-07,
      "logits/chosen": -0.458984375,
      "logits/rejected": -0.494140625,
      "logps/chosen": -92.5,
      "logps/rejected": -112.5,
      "loss": 0.5807,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.2216796875,
      "rewards/margins": 0.3125,
      "rewards/rejected": -0.09130859375,
      "step": 263
    },
    {
      "epoch": 0.7822222222222223,
      "grad_norm": 352.5789794921875,
      "learning_rate": 4.332344213649852e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": -0.46875,
      "logps/chosen": -115.0,
      "logps/rejected": -139.0,
      "loss": 0.5781,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.2294921875,
      "rewards/margins": 0.333984375,
      "rewards/rejected": -0.10498046875,
      "step": 264
    },
    {
      "epoch": 0.7851851851851852,
      "grad_norm": 297.2604675292969,
      "learning_rate": 4.272997032640949e-07,
      "logits/chosen": -0.486328125,
      "logits/rejected": -0.53125,
      "logps/chosen": -122.5,
      "logps/rejected": -175.0,
      "loss": 0.5029,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2470703125,
      "rewards/margins": 0.640625,
      "rewards/rejected": -0.392578125,
      "step": 265
    },
    {
      "epoch": 0.7881481481481482,
      "grad_norm": 311.6089782714844,
      "learning_rate": 4.2136498516320476e-07,
      "logits/chosen": -0.439453125,
      "logits/rejected": -0.458984375,
      "logps/chosen": -122.5,
      "logps/rejected": -125.0,
      "loss": 0.5132,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.2216796875,
      "rewards/margins": 0.4921875,
      "rewards/rejected": -0.271484375,
      "step": 266
    },
    {
      "epoch": 0.7911111111111111,
      "grad_norm": 313.684326171875,
      "learning_rate": 4.1543026706231454e-07,
      "logits/chosen": -0.427734375,
      "logits/rejected": -0.484375,
      "logps/chosen": -107.5,
      "logps/rejected": -106.5,
      "loss": 0.5455,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.30859375,
      "rewards/margins": 0.380859375,
      "rewards/rejected": -0.0712890625,
      "step": 267
    },
    {
      "epoch": 0.794074074074074,
      "grad_norm": 336.37811279296875,
      "learning_rate": 4.094955489614243e-07,
      "logits/chosen": -0.46875,
      "logits/rejected": -0.51171875,
      "logps/chosen": -130.0,
      "logps/rejected": -112.5,
      "loss": 0.5732,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.134765625,
      "rewards/margins": 0.322265625,
      "rewards/rejected": -0.1884765625,
      "step": 268
    },
    {
      "epoch": 0.797037037037037,
      "grad_norm": 357.6194152832031,
      "learning_rate": 4.035608308605341e-07,
      "logits/chosen": -0.486328125,
      "logits/rejected": -0.4921875,
      "logps/chosen": -115.0,
      "logps/rejected": -132.0,
      "loss": 0.6066,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1748046875,
      "rewards/margins": 0.267578125,
      "rewards/rejected": -0.0927734375,
      "step": 269
    },
    {
      "epoch": 0.8,
      "grad_norm": 362.451171875,
      "learning_rate": 3.976261127596439e-07,
      "logits/chosen": -0.470703125,
      "logits/rejected": -0.478515625,
      "logps/chosen": -119.0,
      "logps/rejected": -117.5,
      "loss": 0.5926,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.189453125,
      "rewards/margins": 0.31640625,
      "rewards/rejected": -0.126953125,
      "step": 270
    },
    {
      "epoch": 0.802962962962963,
      "grad_norm": 324.0819091796875,
      "learning_rate": 3.916913946587537e-07,
      "logits/chosen": -0.484375,
      "logits/rejected": -0.423828125,
      "logps/chosen": -113.5,
      "logps/rejected": -137.0,
      "loss": 0.5917,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.12890625,
      "rewards/margins": 0.328125,
      "rewards/rejected": -0.2001953125,
      "step": 271
    },
    {
      "epoch": 0.8059259259259259,
      "grad_norm": 333.343505859375,
      "learning_rate": 3.8575667655786346e-07,
      "logits/chosen": -0.447265625,
      "logits/rejected": -0.455078125,
      "logps/chosen": -112.0,
      "logps/rejected": -133.0,
      "loss": 0.5617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.203125,
      "rewards/margins": 0.345703125,
      "rewards/rejected": -0.142578125,
      "step": 272
    },
    {
      "epoch": 0.8088888888888889,
      "grad_norm": 379.63433837890625,
      "learning_rate": 3.798219584569733e-07,
      "logits/chosen": -0.44140625,
      "logits/rejected": -0.466796875,
      "logps/chosen": -93.0,
      "logps/rejected": -99.5,
      "loss": 0.6309,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.1884765625,
      "rewards/margins": 0.1748046875,
      "rewards/rejected": 0.01336669921875,
      "step": 273
    },
    {
      "epoch": 0.8118518518518518,
      "grad_norm": 336.7275390625,
      "learning_rate": 3.7388724035608303e-07,
      "logits/chosen": -0.466796875,
      "logits/rejected": -0.48046875,
      "logps/chosen": -107.5,
      "logps/rejected": -112.0,
      "loss": 0.5811,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.291015625,
      "rewards/margins": 0.33203125,
      "rewards/rejected": -0.04150390625,
      "step": 274
    },
    {
      "epoch": 0.8148148148148148,
      "grad_norm": 308.87542724609375,
      "learning_rate": 3.6795252225519287e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": -0.49609375,
      "logps/chosen": -108.5,
      "logps/rejected": -120.0,
      "loss": 0.532,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.259765625,
      "rewards/margins": 0.40234375,
      "rewards/rejected": -0.1435546875,
      "step": 275
    },
    {
      "epoch": 0.8177777777777778,
      "grad_norm": 349.4025573730469,
      "learning_rate": 3.6201780415430265e-07,
      "logits/chosen": -0.4140625,
      "logits/rejected": -0.439453125,
      "logps/chosen": -116.0,
      "logps/rejected": -111.0,
      "loss": 0.6223,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.154296875,
      "rewards/margins": 0.1728515625,
      "rewards/rejected": -0.017822265625,
      "step": 276
    },
    {
      "epoch": 0.8207407407407408,
      "grad_norm": 327.4034423828125,
      "learning_rate": 3.560830860534125e-07,
      "logits/chosen": -0.447265625,
      "logits/rejected": -0.474609375,
      "logps/chosen": -114.0,
      "logps/rejected": -105.0,
      "loss": 0.5864,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.2431640625,
      "rewards/margins": 0.28515625,
      "rewards/rejected": -0.04150390625,
      "step": 277
    },
    {
      "epoch": 0.8237037037037037,
      "grad_norm": 326.1805725097656,
      "learning_rate": 3.501483679525222e-07,
      "logits/chosen": -0.466796875,
      "logits/rejected": -0.515625,
      "logps/chosen": -112.0,
      "logps/rejected": -139.0,
      "loss": 0.4851,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.306640625,
      "rewards/margins": 0.54296875,
      "rewards/rejected": -0.2373046875,
      "step": 278
    },
    {
      "epoch": 0.8266666666666667,
      "grad_norm": 337.7111511230469,
      "learning_rate": 3.4421364985163206e-07,
      "logits/chosen": -0.427734375,
      "logits/rejected": -0.45703125,
      "logps/chosen": -107.0,
      "logps/rejected": -108.5,
      "loss": 0.5501,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.23828125,
      "rewards/margins": 0.400390625,
      "rewards/rejected": -0.162109375,
      "step": 279
    },
    {
      "epoch": 0.8296296296296296,
      "grad_norm": 305.23504638671875,
      "learning_rate": 3.3827893175074184e-07,
      "logits/chosen": -0.421875,
      "logits/rejected": -0.470703125,
      "logps/chosen": -115.5,
      "logps/rejected": -117.0,
      "loss": 0.5378,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.259765625,
      "rewards/margins": 0.431640625,
      "rewards/rejected": -0.1728515625,
      "step": 280
    },
    {
      "epoch": 0.8325925925925926,
      "grad_norm": 303.7311096191406,
      "learning_rate": 3.323442136498516e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": -0.4609375,
      "logps/chosen": -108.5,
      "logps/rejected": -119.0,
      "loss": 0.5375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2255859375,
      "rewards/margins": 0.4140625,
      "rewards/rejected": -0.189453125,
      "step": 281
    },
    {
      "epoch": 0.8355555555555556,
      "grad_norm": 306.6491394042969,
      "learning_rate": 3.264094955489614e-07,
      "logits/chosen": -0.45703125,
      "logits/rejected": -0.443359375,
      "logps/chosen": -113.5,
      "logps/rejected": -191.0,
      "loss": 0.5292,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.28125,
      "rewards/margins": 0.58203125,
      "rewards/rejected": -0.302734375,
      "step": 282
    },
    {
      "epoch": 0.8385185185185186,
      "grad_norm": 344.1654052734375,
      "learning_rate": 3.2047477744807125e-07,
      "logits/chosen": -0.458984375,
      "logits/rejected": -0.470703125,
      "logps/chosen": -124.5,
      "logps/rejected": -153.0,
      "loss": 0.4954,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.30859375,
      "rewards/margins": 0.58984375,
      "rewards/rejected": -0.279296875,
      "step": 283
    },
    {
      "epoch": 0.8414814814814815,
      "grad_norm": 387.68011474609375,
      "learning_rate": 3.14540059347181e-07,
      "logits/chosen": -0.48046875,
      "logits/rejected": -0.470703125,
      "logps/chosen": -130.0,
      "logps/rejected": -146.0,
      "loss": 0.597,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.1298828125,
      "rewards/margins": 0.326171875,
      "rewards/rejected": -0.1962890625,
      "step": 284
    },
    {
      "epoch": 0.8444444444444444,
      "grad_norm": 360.8407287597656,
      "learning_rate": 3.0860534124629076e-07,
      "logits/chosen": -0.46484375,
      "logits/rejected": -0.48046875,
      "logps/chosen": -100.0,
      "logps/rejected": -107.0,
      "loss": 0.5915,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1865234375,
      "rewards/margins": 0.28125,
      "rewards/rejected": -0.09423828125,
      "step": 285
    },
    {
      "epoch": 0.8474074074074074,
      "grad_norm": 328.58441162109375,
      "learning_rate": 3.026706231454006e-07,
      "logits/chosen": -0.4765625,
      "logits/rejected": -0.478515625,
      "logps/chosen": -105.0,
      "logps/rejected": -110.0,
      "loss": 0.5646,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.212890625,
      "rewards/margins": 0.34765625,
      "rewards/rejected": -0.1337890625,
      "step": 286
    },
    {
      "epoch": 0.8503703703703703,
      "grad_norm": 351.5137634277344,
      "learning_rate": 2.9673590504451033e-07,
      "logits/chosen": -0.412109375,
      "logits/rejected": -0.40234375,
      "logps/chosen": -98.5,
      "logps/rejected": -155.0,
      "loss": 0.6265,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.197265625,
      "rewards/margins": 0.33203125,
      "rewards/rejected": -0.1337890625,
      "step": 287
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 312.41815185546875,
      "learning_rate": 2.9080118694362016e-07,
      "logits/chosen": -0.5078125,
      "logits/rejected": -0.53515625,
      "logps/chosen": -120.0,
      "logps/rejected": -133.0,
      "loss": 0.5293,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1884765625,
      "rewards/margins": 0.48828125,
      "rewards/rejected": -0.30078125,
      "step": 288
    },
    {
      "epoch": 0.8562962962962963,
      "grad_norm": 340.3605041503906,
      "learning_rate": 2.8486646884272995e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": -0.474609375,
      "logps/chosen": -114.5,
      "logps/rejected": -107.5,
      "loss": 0.572,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1748046875,
      "rewards/margins": 0.330078125,
      "rewards/rejected": -0.1552734375,
      "step": 289
    },
    {
      "epoch": 0.8592592592592593,
      "grad_norm": 362.09100341796875,
      "learning_rate": 2.789317507418398e-07,
      "logits/chosen": -0.431640625,
      "logits/rejected": -0.4296875,
      "logps/chosen": -110.5,
      "logps/rejected": -113.0,
      "loss": 0.6093,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.1513671875,
      "rewards/margins": 0.2138671875,
      "rewards/rejected": -0.0625,
      "step": 290
    },
    {
      "epoch": 0.8622222222222222,
      "grad_norm": 329.35833740234375,
      "learning_rate": 2.729970326409495e-07,
      "logits/chosen": -0.48046875,
      "logits/rejected": -0.51953125,
      "logps/chosen": -106.5,
      "logps/rejected": -138.0,
      "loss": 0.4774,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.30859375,
      "rewards/margins": 0.578125,
      "rewards/rejected": -0.26953125,
      "step": 291
    },
    {
      "epoch": 0.8651851851851852,
      "grad_norm": 323.2444763183594,
      "learning_rate": 2.6706231454005935e-07,
      "logits/chosen": -0.470703125,
      "logits/rejected": -0.5234375,
      "logps/chosen": -93.0,
      "logps/rejected": -116.0,
      "loss": 0.6023,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.2216796875,
      "rewards/margins": 0.2578125,
      "rewards/rejected": -0.035888671875,
      "step": 292
    },
    {
      "epoch": 0.8681481481481481,
      "grad_norm": 350.0760498046875,
      "learning_rate": 2.6112759643916914e-07,
      "logits/chosen": -0.46484375,
      "logits/rejected": -0.52734375,
      "logps/chosen": -116.5,
      "logps/rejected": -153.0,
      "loss": 0.5517,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.154296875,
      "rewards/margins": 0.4765625,
      "rewards/rejected": -0.322265625,
      "step": 293
    },
    {
      "epoch": 0.8711111111111111,
      "grad_norm": 304.44775390625,
      "learning_rate": 2.551928783382789e-07,
      "logits/chosen": -0.46875,
      "logits/rejected": -0.50390625,
      "logps/chosen": -101.0,
      "logps/rejected": -109.0,
      "loss": 0.5915,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.318359375,
      "rewards/margins": 0.294921875,
      "rewards/rejected": 0.0235595703125,
      "step": 294
    },
    {
      "epoch": 0.8740740740740741,
      "grad_norm": 324.566162109375,
      "learning_rate": 2.492581602373887e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": -0.47265625,
      "logps/chosen": -110.0,
      "logps/rejected": -112.5,
      "loss": 0.5785,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.26953125,
      "rewards/margins": 0.318359375,
      "rewards/rejected": -0.04833984375,
      "step": 295
    },
    {
      "epoch": 0.8770370370370371,
      "grad_norm": 328.0279541015625,
      "learning_rate": 2.433234421364985e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": -0.431640625,
      "logps/chosen": -110.5,
      "logps/rejected": -108.0,
      "loss": 0.5424,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.26171875,
      "rewards/margins": 0.431640625,
      "rewards/rejected": -0.1689453125,
      "step": 296
    },
    {
      "epoch": 0.88,
      "grad_norm": 302.4769287109375,
      "learning_rate": 2.373887240356083e-07,
      "logits/chosen": -0.48828125,
      "logits/rejected": -0.5,
      "logps/chosen": -117.5,
      "logps/rejected": -158.0,
      "loss": 0.5054,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.2578125,
      "rewards/margins": 0.51953125,
      "rewards/rejected": -0.259765625,
      "step": 297
    },
    {
      "epoch": 0.882962962962963,
      "grad_norm": 312.2156677246094,
      "learning_rate": 2.3145400593471808e-07,
      "logits/chosen": -0.416015625,
      "logits/rejected": -0.443359375,
      "logps/chosen": -87.0,
      "logps/rejected": -102.5,
      "loss": 0.5788,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.251953125,
      "rewards/margins": 0.30859375,
      "rewards/rejected": -0.057373046875,
      "step": 298
    },
    {
      "epoch": 0.8859259259259259,
      "grad_norm": 295.56170654296875,
      "learning_rate": 2.2551928783382787e-07,
      "logits/chosen": -0.451171875,
      "logits/rejected": -0.4375,
      "logps/chosen": -118.0,
      "logps/rejected": -108.5,
      "loss": 0.5136,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.287109375,
      "rewards/margins": 0.4609375,
      "rewards/rejected": -0.1748046875,
      "step": 299
    },
    {
      "epoch": 0.8888888888888888,
      "grad_norm": 366.7168884277344,
      "learning_rate": 2.1958456973293768e-07,
      "logits/chosen": -0.439453125,
      "logits/rejected": -0.458984375,
      "logps/chosen": -87.0,
      "logps/rejected": -111.0,
      "loss": 0.635,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.26171875,
      "rewards/margins": 0.1767578125,
      "rewards/rejected": 0.0849609375,
      "step": 300
    },
    {
      "epoch": 0.8918518518518519,
      "grad_norm": 333.2161865234375,
      "learning_rate": 2.1364985163204746e-07,
      "logits/chosen": -0.427734375,
      "logits/rejected": -0.435546875,
      "logps/chosen": -95.5,
      "logps/rejected": -136.0,
      "loss": 0.6377,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.2392578125,
      "rewards/margins": 0.294921875,
      "rewards/rejected": -0.05615234375,
      "step": 301
    },
    {
      "epoch": 0.8948148148148148,
      "grad_norm": 296.5155334472656,
      "learning_rate": 2.0771513353115727e-07,
      "logits/chosen": -0.48828125,
      "logits/rejected": -0.458984375,
      "logps/chosen": -97.0,
      "logps/rejected": -149.0,
      "loss": 0.4789,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.345703125,
      "rewards/margins": 0.62890625,
      "rewards/rejected": -0.283203125,
      "step": 302
    },
    {
      "epoch": 0.8977777777777778,
      "grad_norm": 309.7770690917969,
      "learning_rate": 2.0178041543026706e-07,
      "logits/chosen": -0.4296875,
      "logits/rejected": -0.4765625,
      "logps/chosen": -99.5,
      "logps/rejected": -142.0,
      "loss": 0.5024,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.263671875,
      "rewards/margins": 0.5078125,
      "rewards/rejected": -0.2431640625,
      "step": 303
    },
    {
      "epoch": 0.9007407407407407,
      "grad_norm": 381.62310791015625,
      "learning_rate": 1.9584569732937684e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": -0.46484375,
      "logps/chosen": -115.5,
      "logps/rejected": -96.0,
      "loss": 0.6429,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.22265625,
      "rewards/margins": 0.1767578125,
      "rewards/rejected": 0.046142578125,
      "step": 304
    },
    {
      "epoch": 0.9037037037037037,
      "grad_norm": 328.7142639160156,
      "learning_rate": 1.8991097922848665e-07,
      "logits/chosen": -0.43359375,
      "logits/rejected": -0.451171875,
      "logps/chosen": -88.0,
      "logps/rejected": -99.5,
      "loss": 0.6071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.267578125,
      "rewards/margins": 0.26953125,
      "rewards/rejected": -0.0022430419921875,
      "step": 305
    },
    {
      "epoch": 0.9066666666666666,
      "grad_norm": 359.6824951171875,
      "learning_rate": 1.8397626112759643e-07,
      "logits/chosen": -0.515625,
      "logits/rejected": -0.5,
      "logps/chosen": -129.0,
      "logps/rejected": -103.5,
      "loss": 0.5629,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.236328125,
      "rewards/margins": 0.35546875,
      "rewards/rejected": -0.1181640625,
      "step": 306
    },
    {
      "epoch": 0.9096296296296297,
      "grad_norm": 362.33837890625,
      "learning_rate": 1.7804154302670624e-07,
      "logits/chosen": -0.396484375,
      "logits/rejected": -0.380859375,
      "logps/chosen": -110.5,
      "logps/rejected": -109.5,
      "loss": 0.5921,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.263671875,
      "rewards/margins": 0.275390625,
      "rewards/rejected": -0.0111083984375,
      "step": 307
    },
    {
      "epoch": 0.9125925925925926,
      "grad_norm": 323.42071533203125,
      "learning_rate": 1.7210682492581603e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": -0.4921875,
      "logps/chosen": -87.5,
      "logps/rejected": -102.0,
      "loss": 0.5582,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2734375,
      "rewards/margins": 0.34765625,
      "rewards/rejected": -0.07421875,
      "step": 308
    },
    {
      "epoch": 0.9155555555555556,
      "grad_norm": 311.4526672363281,
      "learning_rate": 1.661721068249258e-07,
      "logits/chosen": -0.427734375,
      "logits/rejected": -0.4375,
      "logps/chosen": -108.5,
      "logps/rejected": -104.5,
      "loss": 0.5549,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.310546875,
      "rewards/margins": 0.3984375,
      "rewards/rejected": -0.087890625,
      "step": 309
    },
    {
      "epoch": 0.9185185185185185,
      "grad_norm": 333.0624694824219,
      "learning_rate": 1.6023738872403562e-07,
      "logits/chosen": -0.46875,
      "logits/rejected": -0.453125,
      "logps/chosen": -119.0,
      "logps/rejected": -106.5,
      "loss": 0.5909,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1650390625,
      "rewards/margins": 0.26953125,
      "rewards/rejected": -0.10498046875,
      "step": 310
    },
    {
      "epoch": 0.9214814814814815,
      "grad_norm": 333.5960998535156,
      "learning_rate": 1.5430267062314538e-07,
      "logits/chosen": -0.458984375,
      "logits/rejected": -0.4765625,
      "logps/chosen": -119.5,
      "logps/rejected": -179.0,
      "loss": 0.5661,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.193359375,
      "rewards/margins": 0.40234375,
      "rewards/rejected": -0.208984375,
      "step": 311
    },
    {
      "epoch": 0.9244444444444444,
      "grad_norm": 347.4278259277344,
      "learning_rate": 1.4836795252225516e-07,
      "logits/chosen": -0.4765625,
      "logits/rejected": -0.46875,
      "logps/chosen": -125.5,
      "logps/rejected": -181.0,
      "loss": 0.5573,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.27734375,
      "rewards/margins": 0.4453125,
      "rewards/rejected": -0.1669921875,
      "step": 312
    },
    {
      "epoch": 0.9274074074074075,
      "grad_norm": 373.39276123046875,
      "learning_rate": 1.4243323442136497e-07,
      "logits/chosen": -0.427734375,
      "logits/rejected": -0.451171875,
      "logps/chosen": -99.0,
      "logps/rejected": -134.0,
      "loss": 0.5681,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.255859375,
      "rewards/margins": 0.361328125,
      "rewards/rejected": -0.1064453125,
      "step": 313
    },
    {
      "epoch": 0.9303703703703704,
      "grad_norm": 302.965576171875,
      "learning_rate": 1.3649851632047476e-07,
      "logits/chosen": -0.5078125,
      "logits/rejected": -0.4765625,
      "logps/chosen": -127.0,
      "logps/rejected": -210.0,
      "loss": 0.5341,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.208984375,
      "rewards/margins": 0.5625,
      "rewards/rejected": -0.3515625,
      "step": 314
    },
    {
      "epoch": 0.9333333333333333,
      "grad_norm": 305.0396728515625,
      "learning_rate": 1.3056379821958457e-07,
      "logits/chosen": -0.462890625,
      "logits/rejected": -0.51171875,
      "logps/chosen": -86.0,
      "logps/rejected": -108.5,
      "loss": 0.5503,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.271484375,
      "rewards/margins": 0.341796875,
      "rewards/rejected": -0.0703125,
      "step": 315
    },
    {
      "epoch": 0.9362962962962963,
      "grad_norm": 350.77337646484375,
      "learning_rate": 1.2462908011869435e-07,
      "logits/chosen": -0.375,
      "logits/rejected": -0.4375,
      "logps/chosen": -112.5,
      "logps/rejected": -138.0,
      "loss": 0.6021,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.193359375,
      "rewards/margins": 0.267578125,
      "rewards/rejected": -0.07470703125,
      "step": 316
    },
    {
      "epoch": 0.9392592592592592,
      "grad_norm": 320.9561462402344,
      "learning_rate": 1.1869436201780415e-07,
      "logits/chosen": -0.47265625,
      "logits/rejected": -0.466796875,
      "logps/chosen": -104.0,
      "logps/rejected": -183.0,
      "loss": 0.531,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.287109375,
      "rewards/margins": 0.55078125,
      "rewards/rejected": -0.265625,
      "step": 317
    },
    {
      "epoch": 0.9422222222222222,
      "grad_norm": 322.83856201171875,
      "learning_rate": 1.1275964391691393e-07,
      "logits/chosen": -0.41796875,
      "logits/rejected": -0.4609375,
      "logps/chosen": -99.5,
      "logps/rejected": -124.5,
      "loss": 0.5408,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.244140625,
      "rewards/margins": 0.392578125,
      "rewards/rejected": -0.1474609375,
      "step": 318
    },
    {
      "epoch": 0.9451851851851852,
      "grad_norm": 306.716796875,
      "learning_rate": 1.0682492581602373e-07,
      "logits/chosen": -0.4765625,
      "logits/rejected": -0.5078125,
      "logps/chosen": -99.0,
      "logps/rejected": -133.0,
      "loss": 0.5377,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2373046875,
      "rewards/margins": 0.42578125,
      "rewards/rejected": -0.1884765625,
      "step": 319
    },
    {
      "epoch": 0.9481481481481482,
      "grad_norm": 345.20361328125,
      "learning_rate": 1.0089020771513353e-07,
      "logits/chosen": -0.470703125,
      "logits/rejected": -0.46875,
      "logps/chosen": -131.0,
      "logps/rejected": -163.0,
      "loss": 0.5181,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.25,
      "rewards/margins": 0.53125,
      "rewards/rejected": -0.279296875,
      "step": 320
    },
    {
      "epoch": 0.9511111111111111,
      "grad_norm": 340.1747741699219,
      "learning_rate": 9.495548961424333e-08,
      "logits/chosen": -0.451171875,
      "logits/rejected": -0.48828125,
      "logps/chosen": -97.0,
      "logps/rejected": -98.5,
      "loss": 0.6364,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.15625,
      "rewards/margins": 0.17578125,
      "rewards/rejected": -0.0198974609375,
      "step": 321
    },
    {
      "epoch": 0.9540740740740741,
      "grad_norm": 363.9776916503906,
      "learning_rate": 8.902077151335312e-08,
      "logits/chosen": -0.48046875,
      "logits/rejected": -0.5078125,
      "logps/chosen": -105.0,
      "logps/rejected": -127.0,
      "loss": 0.5558,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.26953125,
      "rewards/margins": 0.431640625,
      "rewards/rejected": -0.162109375,
      "step": 322
    },
    {
      "epoch": 0.957037037037037,
      "grad_norm": 305.4539794921875,
      "learning_rate": 8.30860534124629e-08,
      "logits/chosen": -0.5,
      "logits/rejected": -0.5078125,
      "logps/chosen": -102.5,
      "logps/rejected": -90.5,
      "loss": 0.5662,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.2021484375,
      "rewards/margins": 0.326171875,
      "rewards/rejected": -0.125,
      "step": 323
    },
    {
      "epoch": 0.96,
      "grad_norm": 326.83056640625,
      "learning_rate": 7.715133531157269e-08,
      "logits/chosen": -0.51171875,
      "logits/rejected": -0.515625,
      "logps/chosen": -115.5,
      "logps/rejected": -112.5,
      "loss": 0.6027,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2197265625,
      "rewards/margins": 0.259765625,
      "rewards/rejected": -0.0400390625,
      "step": 324
    },
    {
      "epoch": 0.9629629629629629,
      "grad_norm": 316.9124755859375,
      "learning_rate": 7.121661721068249e-08,
      "logits/chosen": -0.458984375,
      "logits/rejected": -0.5,
      "logps/chosen": -116.0,
      "logps/rejected": -161.0,
      "loss": 0.5458,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.208984375,
      "rewards/margins": 0.4375,
      "rewards/rejected": -0.2294921875,
      "step": 325
    },
    {
      "epoch": 0.965925925925926,
      "grad_norm": 319.8908386230469,
      "learning_rate": 6.528189910979228e-08,
      "logits/chosen": -0.494140625,
      "logits/rejected": -0.515625,
      "logps/chosen": -104.0,
      "logps/rejected": -96.5,
      "loss": 0.5284,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.345703125,
      "rewards/margins": 0.4296875,
      "rewards/rejected": -0.08447265625,
      "step": 326
    },
    {
      "epoch": 0.9688888888888889,
      "grad_norm": 300.6716003417969,
      "learning_rate": 5.9347181008902075e-08,
      "logits/chosen": -0.474609375,
      "logits/rejected": -0.462890625,
      "logps/chosen": -92.0,
      "logps/rejected": -108.5,
      "loss": 0.5072,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.365234375,
      "rewards/margins": 0.47265625,
      "rewards/rejected": -0.10693359375,
      "step": 327
    },
    {
      "epoch": 0.9718518518518519,
      "grad_norm": 345.5276794433594,
      "learning_rate": 5.3412462908011865e-08,
      "logits/chosen": -0.431640625,
      "logits/rejected": -0.46484375,
      "logps/chosen": -110.5,
      "logps/rejected": -119.5,
      "loss": 0.6319,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.1005859375,
      "rewards/margins": 0.1806640625,
      "rewards/rejected": -0.07958984375,
      "step": 328
    },
    {
      "epoch": 0.9748148148148148,
      "grad_norm": 322.6155090332031,
      "learning_rate": 4.747774480712166e-08,
      "logits/chosen": -0.43359375,
      "logits/rejected": -0.48828125,
      "logps/chosen": -100.0,
      "logps/rejected": -130.0,
      "loss": 0.5236,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.298828125,
      "rewards/margins": 0.466796875,
      "rewards/rejected": -0.1689453125,
      "step": 329
    },
    {
      "epoch": 0.9777777777777777,
      "grad_norm": 331.7804260253906,
      "learning_rate": 4.154302670623145e-08,
      "logits/chosen": -0.453125,
      "logits/rejected": -0.48828125,
      "logps/chosen": -108.0,
      "logps/rejected": -146.0,
      "loss": 0.5869,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.158203125,
      "rewards/margins": 0.390625,
      "rewards/rejected": -0.2333984375,
      "step": 330
    },
    {
      "epoch": 0.9807407407407407,
      "grad_norm": 346.5263366699219,
      "learning_rate": 3.5608308605341244e-08,
      "logits/chosen": -0.482421875,
      "logits/rejected": -0.478515625,
      "logps/chosen": -129.0,
      "logps/rejected": -124.0,
      "loss": 0.5934,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.130859375,
      "rewards/margins": 0.279296875,
      "rewards/rejected": -0.1474609375,
      "step": 331
    },
    {
      "epoch": 0.9837037037037037,
      "grad_norm": 328.090576171875,
      "learning_rate": 2.9673590504451037e-08,
      "logits/chosen": -0.439453125,
      "logits/rejected": -0.47265625,
      "logps/chosen": -135.0,
      "logps/rejected": -149.0,
      "loss": 0.5096,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1630859375,
      "rewards/margins": 0.48828125,
      "rewards/rejected": -0.32421875,
      "step": 332
    },
    {
      "epoch": 0.9866666666666667,
      "grad_norm": 357.4970397949219,
      "learning_rate": 2.373887240356083e-08,
      "logits/chosen": -0.5,
      "logits/rejected": -0.484375,
      "logps/chosen": -112.0,
      "logps/rejected": -102.5,
      "loss": 0.5899,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.181640625,
      "rewards/margins": 0.30078125,
      "rewards/rejected": -0.11865234375,
      "step": 333
    },
    {
      "epoch": 0.9896296296296296,
      "grad_norm": 324.227783203125,
      "learning_rate": 1.7804154302670622e-08,
      "logits/chosen": -0.453125,
      "logits/rejected": -0.494140625,
      "logps/chosen": -102.5,
      "logps/rejected": -120.0,
      "loss": 0.4937,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.26171875,
      "rewards/margins": 0.53515625,
      "rewards/rejected": -0.2734375,
      "step": 334
    },
    {
      "epoch": 0.9925925925925926,
      "grad_norm": 320.154296875,
      "learning_rate": 1.1869436201780416e-08,
      "logits/chosen": -0.439453125,
      "logits/rejected": -0.462890625,
      "logps/chosen": -100.0,
      "logps/rejected": -111.5,
      "loss": 0.5704,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2353515625,
      "rewards/margins": 0.326171875,
      "rewards/rejected": -0.08984375,
      "step": 335
    },
    {
      "epoch": 0.9955555555555555,
      "grad_norm": 321.2402038574219,
      "learning_rate": 5.934718100890208e-09,
      "logits/chosen": -0.443359375,
      "logits/rejected": -0.5234375,
      "logps/chosen": -115.5,
      "logps/rejected": -121.5,
      "loss": 0.5919,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.189453125,
      "rewards/margins": 0.322265625,
      "rewards/rejected": -0.1328125,
      "step": 336
    },
    {
      "epoch": 0.9985185185185185,
      "grad_norm": 356.6434326171875,
      "learning_rate": 0.0,
      "logits/chosen": -0.46875,
      "logits/rejected": -0.482421875,
      "logps/chosen": -102.5,
      "logps/rejected": -120.5,
      "loss": 0.634,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.125,
      "rewards/margins": 0.1982421875,
      "rewards/rejected": -0.07275390625,
      "step": 337
    }
  ],
  "logging_steps": 1,
  "max_steps": 337,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}