{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 9.401709401709402,
  "eval_steps": 500,
  "global_step": 22000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.21367521367521367,
      "grad_norm": 4.928079605102539,
      "learning_rate": 5e-05,
      "logits/chosen": -17.893346786499023,
      "logits/rejected": -17.935224533081055,
      "logps/chosen": -247.3836669921875,
      "logps/rejected": -228.11244201660156,
      "loss": 0.7455,
      "rewards/accuracies": 0.5327500104904175,
      "rewards/chosen": 0.5997859835624695,
      "rewards/margins": 0.08256139606237411,
      "rewards/rejected": 0.5172246098518372,
      "step": 500
    },
    {
      "epoch": 0.21367521367521367,
      "eval_logits/chosen": -17.495702743530273,
      "eval_logits/rejected": -17.58708381652832,
      "eval_logps/chosen": -242.62672424316406,
      "eval_logps/rejected": -221.27163696289062,
      "eval_loss": 0.7328621745109558,
      "eval_rewards/accuracies": 0.5503731369972229,
      "eval_rewards/chosen": 1.2266465425491333,
      "eval_rewards/margins": 0.1979004591703415,
      "eval_rewards/rejected": 1.0287461280822754,
      "eval_runtime": 228.1137,
      "eval_samples_per_second": 11.718,
      "eval_steps_per_second": 1.469,
      "step": 500
    },
    {
      "epoch": 0.42735042735042733,
      "grad_norm": 4.208562850952148,
      "learning_rate": 4.890829694323144e-05,
      "logits/chosen": -16.971363067626953,
      "logits/rejected": -17.00401496887207,
      "logps/chosen": -247.00656127929688,
      "logps/rejected": -228.8548126220703,
      "loss": 0.7843,
      "rewards/accuracies": 0.5517500042915344,
      "rewards/chosen": 0.8434626460075378,
      "rewards/margins": 0.19367894530296326,
      "rewards/rejected": 0.649783730506897,
      "step": 1000
    },
    {
      "epoch": 0.42735042735042733,
      "eval_logits/chosen": -17.079561233520508,
      "eval_logits/rejected": -17.085018157958984,
      "eval_logps/chosen": -242.22113037109375,
      "eval_logps/rejected": -221.7228546142578,
      "eval_loss": 0.7642466425895691,
      "eval_rewards/accuracies": 0.5776119232177734,
      "eval_rewards/chosen": 1.2672061920166016,
      "eval_rewards/margins": 0.28358009457588196,
      "eval_rewards/rejected": 0.9836260676383972,
      "eval_runtime": 228.0897,
      "eval_samples_per_second": 11.719,
      "eval_steps_per_second": 1.469,
      "step": 1000
    },
    {
      "epoch": 0.6410256410256411,
      "grad_norm": 3.5691583156585693,
      "learning_rate": 4.7816593886462886e-05,
      "logits/chosen": -16.51204490661621,
      "logits/rejected": -16.47532844543457,
      "logps/chosen": -240.64077758789062,
      "logps/rejected": -222.9105987548828,
      "loss": 0.7908,
      "rewards/accuracies": 0.5792499780654907,
      "rewards/chosen": 1.2004680633544922,
      "rewards/margins": 0.30569130182266235,
      "rewards/rejected": 0.8947767615318298,
      "step": 1500
    },
    {
      "epoch": 0.6410256410256411,
      "eval_logits/chosen": -16.36088752746582,
      "eval_logits/rejected": -16.358932495117188,
      "eval_logps/chosen": -239.6025848388672,
      "eval_logps/rejected": -219.82733154296875,
      "eval_loss": 0.7727888226509094,
      "eval_rewards/accuracies": 0.5697761178016663,
      "eval_rewards/chosen": 1.5290582180023193,
      "eval_rewards/margins": 0.35588183999061584,
      "eval_rewards/rejected": 1.1731764078140259,
      "eval_runtime": 228.179,
      "eval_samples_per_second": 11.714,
      "eval_steps_per_second": 1.468,
      "step": 1500
    },
    {
      "epoch": 0.8547008547008547,
      "grad_norm": 6.145715713500977,
      "learning_rate": 4.672489082969432e-05,
      "logits/chosen": -16.44040870666504,
      "logits/rejected": -16.468050003051758,
      "logps/chosen": -245.83175659179688,
      "logps/rejected": -227.9639892578125,
      "loss": 0.7897,
      "rewards/accuracies": 0.5802500247955322,
      "rewards/chosen": 1.1456317901611328,
      "rewards/margins": 0.3306076228618622,
      "rewards/rejected": 0.8150242567062378,
      "step": 2000
    },
    {
      "epoch": 0.8547008547008547,
      "eval_logits/chosen": -16.609996795654297,
      "eval_logits/rejected": -16.722354888916016,
      "eval_logps/chosen": -241.63665771484375,
      "eval_logps/rejected": -222.01730346679688,
      "eval_loss": 0.7797393202781677,
      "eval_rewards/accuracies": 0.5716418027877808,
      "eval_rewards/chosen": 1.3256531953811646,
      "eval_rewards/margins": 0.37147435545921326,
      "eval_rewards/rejected": 0.9541788101196289,
      "eval_runtime": 227.8733,
      "eval_samples_per_second": 11.73,
      "eval_steps_per_second": 1.47,
      "step": 2000
    },
    {
      "epoch": 1.0683760683760684,
      "grad_norm": 8.053547859191895,
      "learning_rate": 4.563318777292577e-05,
      "logits/chosen": -16.771224975585938,
      "logits/rejected": -16.817113876342773,
      "logps/chosen": -241.07923889160156,
      "logps/rejected": -230.9042510986328,
      "loss": 0.6165,
      "rewards/accuracies": 0.6802499890327454,
      "rewards/chosen": 1.3186030387878418,
      "rewards/margins": 0.9475009441375732,
      "rewards/rejected": 0.37110206484794617,
      "step": 2500
    },
    {
      "epoch": 1.0683760683760684,
      "eval_logits/chosen": -18.916522979736328,
      "eval_logits/rejected": -19.013525009155273,
      "eval_logps/chosen": -245.43980407714844,
      "eval_logps/rejected": -226.0865936279297,
      "eval_loss": 0.8502845168113708,
      "eval_rewards/accuracies": 0.5727611780166626,
      "eval_rewards/chosen": 0.9453384876251221,
      "eval_rewards/margins": 0.39808395504951477,
      "eval_rewards/rejected": 0.5472545027732849,
      "eval_runtime": 227.8708,
      "eval_samples_per_second": 11.73,
      "eval_steps_per_second": 1.47,
      "step": 2500
    },
    {
      "epoch": 1.282051282051282,
      "grad_norm": 5.05163049697876,
      "learning_rate": 4.4541484716157205e-05,
      "logits/chosen": -18.562543869018555,
      "logits/rejected": -18.527542114257812,
      "logps/chosen": -240.0729217529297,
      "logps/rejected": -241.17103576660156,
      "loss": 0.2892,
      "rewards/accuracies": 0.8794999718666077,
      "rewards/chosen": 1.5824379920959473,
      "rewards/margins": 2.353099822998047,
      "rewards/rejected": -0.7706621289253235,
      "step": 3000
    },
    {
      "epoch": 1.282051282051282,
      "eval_logits/chosen": -18.576160430908203,
      "eval_logits/rejected": -18.738643646240234,
      "eval_logps/chosen": -244.65501403808594,
      "eval_logps/rejected": -225.6937713623047,
      "eval_loss": 0.8693752884864807,
      "eval_rewards/accuracies": 0.579104483127594,
      "eval_rewards/chosen": 1.0238155126571655,
      "eval_rewards/margins": 0.4372811019420624,
      "eval_rewards/rejected": 0.5865345597267151,
      "eval_runtime": 227.9156,
      "eval_samples_per_second": 11.728,
      "eval_steps_per_second": 1.47,
      "step": 3000
    },
    {
      "epoch": 1.4957264957264957,
      "grad_norm": 7.019180774688721,
      "learning_rate": 4.344978165938865e-05,
      "logits/chosen": -18.66375160217285,
      "logits/rejected": -18.701576232910156,
      "logps/chosen": -238.48146057128906,
      "logps/rejected": -242.01947021484375,
      "loss": 0.2854,
      "rewards/accuracies": 0.8837500214576721,
      "rewards/chosen": 1.5945905447006226,
      "rewards/margins": 2.4082257747650146,
      "rewards/rejected": -0.8136354088783264,
      "step": 3500
    },
    {
      "epoch": 1.4957264957264957,
      "eval_logits/chosen": -18.779308319091797,
      "eval_logits/rejected": -18.989845275878906,
      "eval_logps/chosen": -243.39840698242188,
      "eval_logps/rejected": -224.86245727539062,
      "eval_loss": 0.8864023685455322,
      "eval_rewards/accuracies": 0.5776119232177734,
      "eval_rewards/chosen": 1.1494779586791992,
      "eval_rewards/margins": 0.4798123836517334,
      "eval_rewards/rejected": 0.6696655750274658,
      "eval_runtime": 227.9853,
      "eval_samples_per_second": 11.724,
      "eval_steps_per_second": 1.469,
      "step": 3500
    },
    {
      "epoch": 1.7094017094017095,
      "grad_norm": 7.860182285308838,
      "learning_rate": 4.235807860262009e-05,
      "logits/chosen": -18.62232208251953,
      "logits/rejected": -18.68227767944336,
      "logps/chosen": -239.76577758789062,
      "logps/rejected": -242.22262573242188,
      "loss": 0.3174,
      "rewards/accuracies": 0.859499990940094,
      "rewards/chosen": 1.3996559381484985,
      "rewards/margins": 2.317948818206787,
      "rewards/rejected": -0.9182929396629333,
      "step": 4000
    },
    {
      "epoch": 1.7094017094017095,
      "eval_logits/chosen": -19.075952529907227,
      "eval_logits/rejected": -19.315759658813477,
      "eval_logps/chosen": -247.94288635253906,
      "eval_logps/rejected": -229.47616577148438,
      "eval_loss": 0.8939631581306458,
      "eval_rewards/accuracies": 0.5742537379264832,
      "eval_rewards/chosen": 0.6950293779373169,
      "eval_rewards/margins": 0.4867364168167114,
      "eval_rewards/rejected": 0.20829293131828308,
      "eval_runtime": 228.1073,
      "eval_samples_per_second": 11.718,
      "eval_steps_per_second": 1.469,
      "step": 4000
    },
    {
      "epoch": 1.9230769230769231,
      "grad_norm": 7.142192840576172,
      "learning_rate": 4.126637554585153e-05,
      "logits/chosen": -18.834980010986328,
      "logits/rejected": -18.85399627685547,
      "logps/chosen": -237.92694091796875,
      "logps/rejected": -241.12193298339844,
      "loss": 0.3407,
      "rewards/accuracies": 0.8569999933242798,
      "rewards/chosen": 1.4859353303909302,
      "rewards/margins": 2.1926088333129883,
      "rewards/rejected": -0.7066735625267029,
      "step": 4500
    },
    {
      "epoch": 1.9230769230769231,
      "eval_logits/chosen": -18.660301208496094,
      "eval_logits/rejected": -18.76552391052246,
      "eval_logps/chosen": -245.375,
      "eval_logps/rejected": -226.55128479003906,
      "eval_loss": 0.8752386569976807,
      "eval_rewards/accuracies": 0.579104483127594,
      "eval_rewards/chosen": 0.95181804895401,
      "eval_rewards/margins": 0.45103806257247925,
      "eval_rewards/rejected": 0.5007798671722412,
      "eval_runtime": 227.9353,
      "eval_samples_per_second": 11.727,
      "eval_steps_per_second": 1.47,
      "step": 4500
    },
    {
      "epoch": 2.1367521367521367,
      "grad_norm": 3.366326093673706,
      "learning_rate": 4.017467248908297e-05,
      "logits/chosen": -19.808855056762695,
      "logits/rejected": -19.856393814086914,
      "logps/chosen": -241.35366821289062,
      "logps/rejected": -259.6560363769531,
      "loss": 0.1671,
      "rewards/accuracies": 0.9362499713897705,
      "rewards/chosen": 1.4077472686767578,
      "rewards/margins": 3.8491322994232178,
      "rewards/rejected": -2.441384792327881,
      "step": 5000
    },
    {
      "epoch": 2.1367521367521367,
      "eval_logits/chosen": -22.046789169311523,
      "eval_logits/rejected": -22.61486053466797,
      "eval_logps/chosen": -268.6415710449219,
      "eval_logps/rejected": -251.24929809570312,
      "eval_loss": 1.0633221864700317,
      "eval_rewards/accuracies": 0.5742537379264832,
      "eval_rewards/chosen": -1.3748414516448975,
      "eval_rewards/margins": 0.5941786170005798,
      "eval_rewards/rejected": -1.9690202474594116,
      "eval_runtime": 228.035,
      "eval_samples_per_second": 11.722,
      "eval_steps_per_second": 1.469,
      "step": 5000
    },
    {
      "epoch": 2.3504273504273505,
      "grad_norm": 0.4629862606525421,
      "learning_rate": 3.9082969432314415e-05,
      "logits/chosen": -21.48691177368164,
      "logits/rejected": -21.680583953857422,
      "logps/chosen": -248.6252899169922,
      "logps/rejected": -278.22113037109375,
      "loss": 0.0894,
      "rewards/accuracies": 0.9677500128746033,
      "rewards/chosen": 0.5657418966293335,
      "rewards/margins": 4.973217964172363,
      "rewards/rejected": -4.407476425170898,
      "step": 5500
    },
    {
      "epoch": 2.3504273504273505,
      "eval_logits/chosen": -21.514081954956055,
      "eval_logits/rejected": -21.708589553833008,
      "eval_logps/chosen": -266.9635314941406,
      "eval_logps/rejected": -249.8363800048828,
      "eval_loss": 1.1009100675582886,
      "eval_rewards/accuracies": 0.562686562538147,
      "eval_rewards/chosen": -1.2070350646972656,
      "eval_rewards/margins": 0.6206951141357422,
      "eval_rewards/rejected": -1.8277301788330078,
      "eval_runtime": 228.1018,
      "eval_samples_per_second": 11.718,
      "eval_steps_per_second": 1.469,
      "step": 5500
    },
    {
      "epoch": 2.564102564102564,
      "grad_norm": 8.165853500366211,
      "learning_rate": 3.799126637554585e-05,
      "logits/chosen": -21.36423110961914,
      "logits/rejected": -21.592361450195312,
      "logps/chosen": -247.65838623046875,
      "logps/rejected": -276.07623291015625,
      "loss": 0.0929,
      "rewards/accuracies": 0.9667500257492065,
      "rewards/chosen": 0.566991925239563,
      "rewards/margins": 4.974726676940918,
      "rewards/rejected": -4.4077348709106445,
      "step": 6000
    },
    {
      "epoch": 2.564102564102564,
      "eval_logits/chosen": -21.843263626098633,
      "eval_logits/rejected": -22.647571563720703,
      "eval_logps/chosen": -271.59844970703125,
      "eval_logps/rejected": -254.54527282714844,
      "eval_loss": 1.1119718551635742,
      "eval_rewards/accuracies": 0.5697761178016663,
      "eval_rewards/chosen": -1.6705284118652344,
      "eval_rewards/margins": 0.6280881762504578,
      "eval_rewards/rejected": -2.298616647720337,
      "eval_runtime": 228.1658,
      "eval_samples_per_second": 11.715,
      "eval_steps_per_second": 1.468,
      "step": 6000
    },
    {
      "epoch": 2.7777777777777777,
      "grad_norm": 0.7172908782958984,
      "learning_rate": 3.68995633187773e-05,
      "logits/chosen": -21.93791389465332,
      "logits/rejected": -22.361909866333008,
      "logps/chosen": -255.6808319091797,
      "logps/rejected": -286.0832214355469,
      "loss": 0.0918,
      "rewards/accuracies": 0.965749979019165,
      "rewards/chosen": -0.06728626787662506,
      "rewards/margins": 5.06732177734375,
      "rewards/rejected": -5.134607315063477,
      "step": 6500
    },
    {
      "epoch": 2.7777777777777777,
      "eval_logits/chosen": -21.824033737182617,
      "eval_logits/rejected": -22.275728225708008,
      "eval_logps/chosen": -270.8564147949219,
      "eval_logps/rejected": -254.7839813232422,
      "eval_loss": 1.133668303489685,
      "eval_rewards/accuracies": 0.5802238583564758,
      "eval_rewards/chosen": -1.596319317817688,
      "eval_rewards/margins": 0.7261707186698914,
      "eval_rewards/rejected": -2.3224899768829346,
      "eval_runtime": 227.9485,
      "eval_samples_per_second": 11.726,
      "eval_steps_per_second": 1.47,
      "step": 6500
    },
    {
      "epoch": 2.9914529914529915,
      "grad_norm": 2.079650640487671,
      "learning_rate": 3.5807860262008734e-05,
      "logits/chosen": -22.237197875976562,
      "logits/rejected": -22.402421951293945,
      "logps/chosen": -257.87725830078125,
      "logps/rejected": -288.0488586425781,
      "loss": 0.1018,
      "rewards/accuracies": 0.9635000228881836,
      "rewards/chosen": -0.3456314504146576,
      "rewards/margins": 5.053824424743652,
      "rewards/rejected": -5.399456024169922,
      "step": 7000
    },
    {
      "epoch": 2.9914529914529915,
      "eval_logits/chosen": -22.45467758178711,
      "eval_logits/rejected": -22.827308654785156,
      "eval_logps/chosen": -275.2033996582031,
      "eval_logps/rejected": -259.20159912109375,
      "eval_loss": 1.1700248718261719,
      "eval_rewards/accuracies": 0.5701492428779602,
      "eval_rewards/chosen": -2.031022787094116,
      "eval_rewards/margins": 0.7332298755645752,
      "eval_rewards/rejected": -2.7642529010772705,
      "eval_runtime": 227.8703,
      "eval_samples_per_second": 11.73,
      "eval_steps_per_second": 1.47,
      "step": 7000
    },
    {
      "epoch": 3.2051282051282053,
      "grad_norm": 3.4078361988067627,
      "learning_rate": 3.4716157205240176e-05,
      "logits/chosen": -23.701507568359375,
      "logits/rejected": -23.8421688079834,
      "logps/chosen": -263.1128234863281,
      "logps/rejected": -315.31719970703125,
      "loss": 0.0299,
      "rewards/accuracies": 0.9897500276565552,
      "rewards/chosen": -0.8062784075737,
      "rewards/margins": 7.403973579406738,
      "rewards/rejected": -8.210251808166504,
      "step": 7500
    },
    {
      "epoch": 3.2051282051282053,
      "eval_logits/chosen": -24.26601791381836,
      "eval_logits/rejected": -24.537738800048828,
      "eval_logps/chosen": -295.60528564453125,
      "eval_logps/rejected": -280.7348327636719,
      "eval_loss": 1.3489611148834229,
      "eval_rewards/accuracies": 0.5686567425727844,
      "eval_rewards/chosen": -4.071210861206055,
      "eval_rewards/margins": 0.846366286277771,
      "eval_rewards/rejected": -4.917576789855957,
      "eval_runtime": 228.1695,
      "eval_samples_per_second": 11.715,
      "eval_steps_per_second": 1.468,
      "step": 7500
    },
    {
      "epoch": 3.4188034188034186,
      "grad_norm": 0.8472763895988464,
      "learning_rate": 3.362445414847162e-05,
      "logits/chosen": -25.03811264038086,
      "logits/rejected": -25.47620964050293,
      "logps/chosen": -272.96221923828125,
      "logps/rejected": -330.9865417480469,
      "loss": 0.0335,
      "rewards/accuracies": 0.9869999885559082,
      "rewards/chosen": -1.7266676425933838,
      "rewards/margins": 7.801348686218262,
      "rewards/rejected": -9.528016090393066,
      "step": 8000
    },
    {
      "epoch": 3.4188034188034186,
      "eval_logits/chosen": -25.045902252197266,
      "eval_logits/rejected": -26.036436080932617,
      "eval_logps/chosen": -298.59942626953125,
      "eval_logps/rejected": -283.6900329589844,
      "eval_loss": 1.3672035932540894,
      "eval_rewards/accuracies": 0.5697761178016663,
      "eval_rewards/chosen": -4.370626449584961,
      "eval_rewards/margins": 0.8424644470214844,
      "eval_rewards/rejected": -5.2130913734436035,
      "eval_runtime": 228.1698,
      "eval_samples_per_second": 11.715,
      "eval_steps_per_second": 1.468,
      "step": 8000
    },
    {
      "epoch": 3.6324786324786325,
      "grad_norm": 0.4369616210460663,
      "learning_rate": 3.2532751091703054e-05,
      "logits/chosen": -25.275936126708984,
      "logits/rejected": -25.85784339904785,
      "logps/chosen": -272.4268493652344,
      "logps/rejected": -328.8055725097656,
      "loss": 0.0342,
      "rewards/accuracies": 0.9862499833106995,
      "rewards/chosen": -1.7520734071731567,
      "rewards/margins": 7.620083332061768,
      "rewards/rejected": -9.372157096862793,
      "step": 8500
    },
    {
      "epoch": 3.6324786324786325,
      "eval_logits/chosen": -25.376535415649414,
      "eval_logits/rejected": -25.840946197509766,
      "eval_logps/chosen": -302.3465576171875,
      "eval_logps/rejected": -286.4819641113281,
      "eval_loss": 1.4073855876922607,
      "eval_rewards/accuracies": 0.5574626922607422,
      "eval_rewards/chosen": -4.745336055755615,
      "eval_rewards/margins": 0.7469544410705566,
      "eval_rewards/rejected": -5.492290019989014,
      "eval_runtime": 228.1692,
      "eval_samples_per_second": 11.715,
      "eval_steps_per_second": 1.468,
      "step": 8500
    },
    {
      "epoch": 3.8461538461538463,
      "grad_norm": 0.20020511746406555,
      "learning_rate": 3.14410480349345e-05,
      "logits/chosen": -25.79690933227539,
      "logits/rejected": -26.186351776123047,
      "logps/chosen": -275.40826416015625,
      "logps/rejected": -333.3716125488281,
      "loss": 0.0406,
      "rewards/accuracies": 0.9865000247955322,
      "rewards/chosen": -2.3063974380493164,
      "rewards/margins": 7.773219108581543,
      "rewards/rejected": -10.079617500305176,
      "step": 9000
    },
    {
      "epoch": 3.8461538461538463,
      "eval_logits/chosen": -25.633359909057617,
      "eval_logits/rejected": -26.33572769165039,
      "eval_logps/chosen": -305.1189270019531,
      "eval_logps/rejected": -290.773193359375,
      "eval_loss": 1.4024347066879272,
      "eval_rewards/accuracies": 0.5660447478294373,
      "eval_rewards/chosen": -5.022571086883545,
      "eval_rewards/margins": 0.8988366723060608,
      "eval_rewards/rejected": -5.921407699584961,
      "eval_runtime": 227.3147,
      "eval_samples_per_second": 11.759,
      "eval_steps_per_second": 1.474,
      "step": 9000
    },
    {
      "epoch": 4.05982905982906,
      "grad_norm": 0.4492078125476837,
      "learning_rate": 3.0349344978165938e-05,
      "logits/chosen": -25.544862747192383,
      "logits/rejected": -25.84202003479004,
      "logps/chosen": -275.60540771484375,
      "logps/rejected": -333.7979431152344,
      "loss": 0.0353,
      "rewards/accuracies": 0.9869999885559082,
      "rewards/chosen": -2.0801212787628174,
      "rewards/margins": 7.923834800720215,
      "rewards/rejected": -10.00395679473877,
      "step": 9500
    },
    {
      "epoch": 4.05982905982906,
      "eval_logits/chosen": -26.5059871673584,
      "eval_logits/rejected": -26.621540069580078,
      "eval_logps/chosen": -315.5564880371094,
      "eval_logps/rejected": -300.5618591308594,
      "eval_loss": 1.492741584777832,
      "eval_rewards/accuracies": 0.5660447478294373,
      "eval_rewards/chosen": -6.066328525543213,
      "eval_rewards/margins": 0.833947479724884,
      "eval_rewards/rejected": -6.900275707244873,
      "eval_runtime": 227.4988,
      "eval_samples_per_second": 11.75,
      "eval_steps_per_second": 1.473,
      "step": 9500
    },
    {
      "epoch": 4.273504273504273,
      "grad_norm": 0.786451518535614,
      "learning_rate": 2.9257641921397383e-05,
      "logits/chosen": -26.779680252075195,
      "logits/rejected": -26.80259132385254,
      "logps/chosen": -283.5688171386719,
      "logps/rejected": -356.3512268066406,
      "loss": 0.0127,
      "rewards/accuracies": 0.9944999814033508,
      "rewards/chosen": -2.842845916748047,
      "rewards/margins": 9.423320770263672,
      "rewards/rejected": -12.266166687011719,
      "step": 10000
    },
    {
      "epoch": 4.273504273504273,
      "eval_logits/chosen": -27.469865798950195,
      "eval_logits/rejected": -27.798917770385742,
      "eval_logps/chosen": -325.7637023925781,
      "eval_logps/rejected": -311.94171142578125,
      "eval_loss": 1.5876415967941284,
      "eval_rewards/accuracies": 0.5615671873092651,
      "eval_rewards/chosen": -7.0870537757873535,
      "eval_rewards/margins": 0.9512065649032593,
      "eval_rewards/rejected": -8.038260459899902,
      "eval_runtime": 227.4171,
      "eval_samples_per_second": 11.754,
      "eval_steps_per_second": 1.473,
      "step": 10000
    },
    {
      "epoch": 4.487179487179487,
      "grad_norm": 0.010021047666668892,
      "learning_rate": 2.816593886462882e-05,
      "logits/chosen": -27.027307510375977,
      "logits/rejected": -27.183887481689453,
      "logps/chosen": -284.8421325683594,
      "logps/rejected": -359.85992431640625,
      "loss": 0.0208,
      "rewards/accuracies": 0.9887499809265137,
      "rewards/chosen": -3.0061757564544678,
      "rewards/margins": 9.546944618225098,
      "rewards/rejected": -12.553121566772461,
      "step": 10500
    },
    {
      "epoch": 4.487179487179487,
      "eval_logits/chosen": -27.8445987701416,
      "eval_logits/rejected": -28.134754180908203,
      "eval_logps/chosen": -331.4013977050781,
      "eval_logps/rejected": -317.50494384765625,
      "eval_loss": 1.6331137418746948,
      "eval_rewards/accuracies": 0.5701492428779602,
      "eval_rewards/chosen": -7.650820732116699,
      "eval_rewards/margins": 0.9437649250030518,
      "eval_rewards/rejected": -8.594585418701172,
      "eval_runtime": 227.6795,
      "eval_samples_per_second": 11.74,
      "eval_steps_per_second": 1.471,
      "step": 10500
    },
    {
      "epoch": 4.700854700854701,
      "grad_norm": 0.36542195081710815,
      "learning_rate": 2.7074235807860267e-05,
      "logits/chosen": -27.794286727905273,
      "logits/rejected": -27.915931701660156,
      "logps/chosen": -291.84320068359375,
      "logps/rejected": -370.82891845703125,
      "loss": 0.0187,
      "rewards/accuracies": 0.9929999709129333,
      "rewards/chosen": -3.9194016456604004,
      "rewards/margins": 9.847352027893066,
      "rewards/rejected": -13.766753196716309,
      "step": 11000
    },
    {
      "epoch": 4.700854700854701,
      "eval_logits/chosen": -28.695451736450195,
      "eval_logits/rejected": -29.109943389892578,
      "eval_logps/chosen": -340.71160888671875,
      "eval_logps/rejected": -326.3081359863281,
      "eval_loss": 1.7010728120803833,
      "eval_rewards/accuracies": 0.5533581972122192,
      "eval_rewards/chosen": -8.581844329833984,
      "eval_rewards/margins": 0.8930591344833374,
      "eval_rewards/rejected": -9.474903106689453,
      "eval_runtime": 227.9038,
      "eval_samples_per_second": 11.729,
      "eval_steps_per_second": 1.47,
      "step": 11000
    },
    {
      "epoch": 4.914529914529915,
      "grad_norm": 0.3806762099266052,
      "learning_rate": 2.5982532751091705e-05,
      "logits/chosen": -28.41119384765625,
      "logits/rejected": -29.06536865234375,
      "logps/chosen": -296.35723876953125,
      "logps/rejected": -373.481201171875,
      "loss": 0.0225,
      "rewards/accuracies": 0.9915000200271606,
      "rewards/chosen": -4.082763671875,
      "rewards/margins": 9.720556259155273,
      "rewards/rejected": -13.803319931030273,
      "step": 11500
    },
    {
      "epoch": 4.914529914529915,
      "eval_logits/chosen": -29.32524871826172,
      "eval_logits/rejected": -30.48952865600586,
      "eval_logps/chosen": -336.1420593261719,
      "eval_logps/rejected": -321.3468933105469,
      "eval_loss": 1.700086236000061,
      "eval_rewards/accuracies": 0.5582089424133301,
      "eval_rewards/chosen": -8.124887466430664,
      "eval_rewards/margins": 0.8538958430290222,
      "eval_rewards/rejected": -8.978782653808594,
      "eval_runtime": 227.8283,
      "eval_samples_per_second": 11.733,
      "eval_steps_per_second": 1.47,
      "step": 11500
    },
    {
      "epoch": 5.128205128205128,
      "grad_norm": 0.2094322144985199,
      "learning_rate": 2.4890829694323144e-05,
      "logits/chosen": -28.792156219482422,
      "logits/rejected": -29.474102020263672,
      "logps/chosen": -296.71466064453125,
      "logps/rejected": -377.149658203125,
      "loss": 0.0158,
      "rewards/accuracies": 0.9934999942779541,
      "rewards/chosen": -4.169726848602295,
      "rewards/margins": 10.347012519836426,
      "rewards/rejected": -14.516740798950195,
      "step": 12000
    },
    {
      "epoch": 5.128205128205128,
      "eval_logits/chosen": -30.29293441772461,
      "eval_logits/rejected": -30.985925674438477,
      "eval_logps/chosen": -351.3580627441406,
      "eval_logps/rejected": -336.6136474609375,
      "eval_loss": 1.815493106842041,
      "eval_rewards/accuracies": 0.5555970072746277,
      "eval_rewards/chosen": -9.646490097045898,
      "eval_rewards/margins": 0.8589639663696289,
      "eval_rewards/rejected": -10.505454063415527,
      "eval_runtime": 227.9111,
      "eval_samples_per_second": 11.728,
      "eval_steps_per_second": 1.47,
      "step": 12000
    },
    {
      "epoch": 5.3418803418803416,
      "grad_norm": 0.10701989382505417,
      "learning_rate": 2.3799126637554586e-05,
      "logits/chosen": -29.149518966674805,
      "logits/rejected": -29.762508392333984,
      "logps/chosen": -296.66693115234375,
      "logps/rejected": -384.7826843261719,
      "loss": 0.0129,
      "rewards/accuracies": 0.9927499890327454,
      "rewards/chosen": -4.225397109985352,
      "rewards/margins": 10.851059913635254,
      "rewards/rejected": -15.076457023620605,
      "step": 12500
    },
    {
      "epoch": 5.3418803418803416,
      "eval_logits/chosen": -30.771709442138672,
      "eval_logits/rejected": -31.801382064819336,
      "eval_logps/chosen": -351.3876953125,
      "eval_logps/rejected": -337.5948181152344,
      "eval_loss": 1.822997808456421,
      "eval_rewards/accuracies": 0.5578358173370361,
      "eval_rewards/chosen": -9.649447441101074,
      "eval_rewards/margins": 0.9541246891021729,
      "eval_rewards/rejected": -10.603572845458984,
      "eval_runtime": 227.8818,
      "eval_samples_per_second": 11.73,
      "eval_steps_per_second": 1.47,
      "step": 12500
    },
    {
      "epoch": 5.555555555555555,
      "grad_norm": 5.323533535003662,
      "learning_rate": 2.2707423580786028e-05,
      "logits/chosen": -29.866849899291992,
      "logits/rejected": -30.677040100097656,
      "logps/chosen": -301.1296081542969,
      "logps/rejected": -388.8807678222656,
      "loss": 0.0148,
      "rewards/accuracies": 0.9940000176429749,
      "rewards/chosen": -4.681130886077881,
      "rewards/margins": 10.871023178100586,
      "rewards/rejected": -15.552155494689941,
      "step": 13000
    },
    {
      "epoch": 5.555555555555555,
      "eval_logits/chosen": -29.594120025634766,
      "eval_logits/rejected": -30.170185089111328,
      "eval_logps/chosen": -342.6488952636719,
      "eval_logps/rejected": -329.54034423828125,
      "eval_loss": 1.7344962358474731,
      "eval_rewards/accuracies": 0.5600746273994446,
      "eval_rewards/chosen": -8.775569915771484,
      "eval_rewards/margins": 1.0225567817687988,
      "eval_rewards/rejected": -9.798127174377441,
      "eval_runtime": 228.0608,
      "eval_samples_per_second": 11.721,
      "eval_steps_per_second": 1.469,
      "step": 13000
    },
    {
      "epoch": 5.769230769230769,
      "grad_norm": 0.2315172553062439,
      "learning_rate": 2.161572052401747e-05,
      "logits/chosen": -29.85978889465332,
      "logits/rejected": -30.509546279907227,
      "logps/chosen": -301.72674560546875,
      "logps/rejected": -391.1932373046875,
      "loss": 0.0189,
      "rewards/accuracies": 0.9927499890327454,
      "rewards/chosen": -4.796882152557373,
      "rewards/margins": 10.797426223754883,
      "rewards/rejected": -15.594307899475098,
      "step": 13500
    },
    {
      "epoch": 5.769230769230769,
      "eval_logits/chosen": -30.208812713623047,
      "eval_logits/rejected": -31.183263778686523,
      "eval_logps/chosen": -345.8949890136719,
      "eval_logps/rejected": -331.9636535644531,
      "eval_loss": 1.7506355047225952,
      "eval_rewards/accuracies": 0.5593283772468567,
      "eval_rewards/chosen": -9.100179672241211,
      "eval_rewards/margins": 0.9402767419815063,
      "eval_rewards/rejected": -10.040454864501953,
      "eval_runtime": 227.9921,
      "eval_samples_per_second": 11.724,
      "eval_steps_per_second": 1.469,
      "step": 13500
    },
    {
      "epoch": 5.982905982905983,
      "grad_norm": 4.712192058563232,
      "learning_rate": 2.052401746724891e-05,
      "logits/chosen": -29.906803131103516,
      "logits/rejected": -30.264978408813477,
      "logps/chosen": -307.0865173339844,
      "logps/rejected": -395.9546813964844,
      "loss": 0.0152,
      "rewards/accuracies": 0.9944999814033508,
      "rewards/chosen": -5.222396373748779,
      "rewards/margins": 10.86800479888916,
      "rewards/rejected": -16.09040069580078,
      "step": 14000
    },
    {
      "epoch": 5.982905982905983,
      "eval_logits/chosen": -29.900236129760742,
      "eval_logits/rejected": -30.382579803466797,
      "eval_logps/chosen": -348.7581787109375,
      "eval_logps/rejected": -334.68280029296875,
      "eval_loss": 1.7904853820800781,
      "eval_rewards/accuracies": 0.5559701323509216,
      "eval_rewards/chosen": -9.386497497558594,
      "eval_rewards/margins": 0.9258707165718079,
      "eval_rewards/rejected": -10.312368392944336,
      "eval_runtime": 227.9248,
      "eval_samples_per_second": 11.728,
      "eval_steps_per_second": 1.47,
      "step": 14000
    },
    {
      "epoch": 6.196581196581197,
      "grad_norm": 0.004079462960362434,
      "learning_rate": 1.943231441048035e-05,
      "logits/chosen": -30.478179931640625,
      "logits/rejected": -30.970232009887695,
      "logps/chosen": -313.88165283203125,
      "logps/rejected": -409.17962646484375,
      "loss": 0.0126,
      "rewards/accuracies": 0.9942499995231628,
      "rewards/chosen": -5.9417924880981445,
      "rewards/margins": 11.768869400024414,
      "rewards/rejected": -17.710660934448242,
      "step": 14500
    },
    {
      "epoch": 6.196581196581197,
      "eval_logits/chosen": -30.50238800048828,
      "eval_logits/rejected": -31.02549171447754,
      "eval_logps/chosen": -358.1636047363281,
      "eval_logps/rejected": -344.337890625,
      "eval_loss": 1.86916184425354,
      "eval_rewards/accuracies": 0.5600746273994446,
      "eval_rewards/chosen": -10.32703971862793,
      "eval_rewards/margins": 0.9508424401283264,
      "eval_rewards/rejected": -11.277880668640137,
      "eval_runtime": 227.418,
      "eval_samples_per_second": 11.754,
      "eval_steps_per_second": 1.473,
      "step": 14500
    },
    {
      "epoch": 6.410256410256411,
      "grad_norm": 0.06197773665189743,
      "learning_rate": 1.8340611353711792e-05,
      "logits/chosen": -30.11530876159668,
      "logits/rejected": -30.605209350585938,
      "logps/chosen": -308.64007568359375,
      "logps/rejected": -403.6331787109375,
      "loss": 0.0124,
      "rewards/accuracies": 0.9934999942779541,
      "rewards/chosen": -5.467129707336426,
      "rewards/margins": 11.565047264099121,
      "rewards/rejected": -17.032175064086914,
      "step": 15000
    },
    {
      "epoch": 6.410256410256411,
      "eval_logits/chosen": -30.82184600830078,
      "eval_logits/rejected": -31.706174850463867,
      "eval_logps/chosen": -357.69537353515625,
      "eval_logps/rejected": -344.0749206542969,
      "eval_loss": 1.8543611764907837,
      "eval_rewards/accuracies": 0.5597015023231506,
      "eval_rewards/chosen": -10.280218124389648,
      "eval_rewards/margins": 0.9713651537895203,
      "eval_rewards/rejected": -11.251583099365234,
      "eval_runtime": 227.9507,
      "eval_samples_per_second": 11.726,
      "eval_steps_per_second": 1.47,
      "step": 15000
    },
    {
      "epoch": 6.6239316239316235,
      "grad_norm": 0.03625645488500595,
      "learning_rate": 1.7248908296943234e-05,
      "logits/chosen": -31.255956649780273,
      "logits/rejected": -31.90044403076172,
      "logps/chosen": -316.18853759765625,
      "logps/rejected": -415.66796875,
      "loss": 0.0119,
      "rewards/accuracies": 0.9942499995231628,
      "rewards/chosen": -6.230967998504639,
      "rewards/margins": 11.93471908569336,
      "rewards/rejected": -18.165685653686523,
      "step": 15500
    },
    {
      "epoch": 6.6239316239316235,
      "eval_logits/chosen": -31.26849937438965,
      "eval_logits/rejected": -32.083431243896484,
      "eval_logps/chosen": -362.8932800292969,
      "eval_logps/rejected": -351.0294494628906,
      "eval_loss": 1.8719128370285034,
      "eval_rewards/accuracies": 0.5701492428779602,
      "eval_rewards/chosen": -10.800006866455078,
      "eval_rewards/margins": 1.1470307111740112,
      "eval_rewards/rejected": -11.947038650512695,
      "eval_runtime": 227.9667,
      "eval_samples_per_second": 11.725,
      "eval_steps_per_second": 1.47,
      "step": 15500
    },
    {
      "epoch": 6.837606837606837,
      "grad_norm": 0.20229032635688782,
      "learning_rate": 1.6157205240174673e-05,
      "logits/chosen": -31.66826629638672,
      "logits/rejected": -32.30759048461914,
      "logps/chosen": -319.5751647949219,
      "logps/rejected": -420.54534912109375,
      "loss": 0.0109,
      "rewards/accuracies": 0.9947500228881836,
      "rewards/chosen": -6.482852935791016,
      "rewards/margins": 12.024003982543945,
      "rewards/rejected": -18.50685691833496,
      "step": 16000
    },
    {
      "epoch": 6.837606837606837,
      "eval_logits/chosen": -31.825002670288086,
      "eval_logits/rejected": -32.77232360839844,
      "eval_logps/chosen": -362.9580078125,
      "eval_logps/rejected": -349.5175476074219,
      "eval_loss": 1.9149656295776367,
      "eval_rewards/accuracies": 0.562686562538147,
      "eval_rewards/chosen": -10.806483268737793,
      "eval_rewards/margins": 0.9893614649772644,
      "eval_rewards/rejected": -11.795844078063965,
      "eval_runtime": 228.1418,
      "eval_samples_per_second": 11.716,
      "eval_steps_per_second": 1.468,
      "step": 16000
    },
    {
      "epoch": 7.051282051282051,
      "grad_norm": 0.00038808645331300795,
      "learning_rate": 1.5065502183406113e-05,
      "logits/chosen": -32.047367095947266,
      "logits/rejected": -32.96037673950195,
      "logps/chosen": -321.9444274902344,
      "logps/rejected": -421.9002380371094,
      "loss": 0.0161,
      "rewards/accuracies": 0.9925000071525574,
      "rewards/chosen": -6.6494622230529785,
      "rewards/margins": 12.112425804138184,
      "rewards/rejected": -18.76188850402832,
      "step": 16500
    },
    {
      "epoch": 7.051282051282051,
      "eval_logits/chosen": -33.680885314941406,
      "eval_logits/rejected": -34.744083404541016,
      "eval_logps/chosen": -383.9889831542969,
      "eval_logps/rejected": -370.952392578125,
      "eval_loss": 2.086275577545166,
      "eval_rewards/accuracies": 0.5526119470596313,
      "eval_rewards/chosen": -12.909579277038574,
      "eval_rewards/margins": 1.0297467708587646,
      "eval_rewards/rejected": -13.939325332641602,
      "eval_runtime": 226.7868,
      "eval_samples_per_second": 11.786,
      "eval_steps_per_second": 1.477,
      "step": 16500
    },
    {
      "epoch": 7.264957264957265,
      "grad_norm": 0.022403132170438766,
      "learning_rate": 1.3973799126637555e-05,
      "logits/chosen": -33.031978607177734,
      "logits/rejected": -33.99634552001953,
      "logps/chosen": -326.0166931152344,
      "logps/rejected": -433.9259948730469,
      "loss": 0.0093,
      "rewards/accuracies": 0.9952499866485596,
      "rewards/chosen": -7.22238826751709,
      "rewards/margins": 12.905363082885742,
      "rewards/rejected": -20.12775230407715,
      "step": 17000
    },
    {
      "epoch": 7.264957264957265,
      "eval_logits/chosen": -32.838279724121094,
      "eval_logits/rejected": -33.81065368652344,
      "eval_logps/chosen": -374.7555847167969,
      "eval_logps/rejected": -361.67498779296875,
      "eval_loss": 1.9890711307525635,
      "eval_rewards/accuracies": 0.5570895671844482,
      "eval_rewards/chosen": -11.98624038696289,
      "eval_rewards/margins": 1.0253472328186035,
      "eval_rewards/rejected": -13.011587142944336,
      "eval_runtime": 227.8723,
      "eval_samples_per_second": 11.73,
      "eval_steps_per_second": 1.47,
      "step": 17000
    },
    {
      "epoch": 7.478632478632479,
      "grad_norm": 0.01652335375547409,
      "learning_rate": 1.2882096069868996e-05,
      "logits/chosen": -32.10330581665039,
      "logits/rejected": -32.913116455078125,
      "logps/chosen": -319.0428161621094,
      "logps/rejected": -421.7063293457031,
      "loss": 0.0116,
      "rewards/accuracies": 0.9944999814033508,
      "rewards/chosen": -6.535008907318115,
      "rewards/margins": 12.34512996673584,
      "rewards/rejected": -18.880136489868164,
      "step": 17500
    },
    {
      "epoch": 7.478632478632479,
      "eval_logits/chosen": -31.835594177246094,
      "eval_logits/rejected": -32.65978240966797,
      "eval_logps/chosen": -364.2667541503906,
      "eval_logps/rejected": -350.83392333984375,
      "eval_loss": 1.9134238958358765,
      "eval_rewards/accuracies": 0.5611940026283264,
      "eval_rewards/chosen": -10.937359809875488,
      "eval_rewards/margins": 0.9901263117790222,
      "eval_rewards/rejected": -11.927485466003418,
      "eval_runtime": 227.8103,
      "eval_samples_per_second": 11.733,
      "eval_steps_per_second": 1.471,
      "step": 17500
    },
    {
      "epoch": 7.6923076923076925,
      "grad_norm": 0.0030172993429005146,
      "learning_rate": 1.1790393013100438e-05,
      "logits/chosen": -31.859935760498047,
      "logits/rejected": -32.7746696472168,
      "logps/chosen": -316.1585388183594,
      "logps/rejected": -419.97430419921875,
      "loss": 0.0105,
      "rewards/accuracies": 0.9950000047683716,
      "rewards/chosen": -6.1652021408081055,
      "rewards/margins": 12.33544921875,
      "rewards/rejected": -18.500652313232422,
      "step": 18000
    },
    {
      "epoch": 7.6923076923076925,
      "eval_logits/chosen": -33.39469528198242,
      "eval_logits/rejected": -34.474544525146484,
      "eval_logps/chosen": -379.9172668457031,
      "eval_logps/rejected": -366.830322265625,
      "eval_loss": 2.0327441692352295,
      "eval_rewards/accuracies": 0.5638059973716736,
      "eval_rewards/chosen": -12.502408027648926,
      "eval_rewards/margins": 1.024713158607483,
      "eval_rewards/rejected": -13.527120590209961,
      "eval_runtime": 227.4415,
      "eval_samples_per_second": 11.752,
      "eval_steps_per_second": 1.473,
      "step": 18000
    },
    {
      "epoch": 7.905982905982906,
      "grad_norm": 16.508834838867188,
      "learning_rate": 1.0698689956331878e-05,
      "logits/chosen": -32.33524703979492,
      "logits/rejected": -33.17497253417969,
      "logps/chosen": -320.648193359375,
      "logps/rejected": -426.4129333496094,
      "loss": 0.0141,
      "rewards/accuracies": 0.9917500019073486,
      "rewards/chosen": -6.537839412689209,
      "rewards/margins": 12.470658302307129,
      "rewards/rejected": -19.008495330810547,
      "step": 18500
    },
    {
      "epoch": 7.905982905982906,
      "eval_logits/chosen": -32.687374114990234,
      "eval_logits/rejected": -33.56389236450195,
      "eval_logps/chosen": -370.7414855957031,
      "eval_logps/rejected": -357.1487731933594,
      "eval_loss": 1.9871249198913574,
      "eval_rewards/accuracies": 0.5548507571220398,
      "eval_rewards/chosen": -11.584826469421387,
      "eval_rewards/margins": 0.9741416573524475,
      "eval_rewards/rejected": -12.558969497680664,
      "eval_runtime": 227.2911,
      "eval_samples_per_second": 11.76,
      "eval_steps_per_second": 1.474,
      "step": 18500
    },
    {
      "epoch": 8.11965811965812,
      "grad_norm": 0.008148429915308952,
      "learning_rate": 9.60698689956332e-06,
      "logits/chosen": -33.27425003051758,
      "logits/rejected": -34.29746627807617,
      "logps/chosen": -327.61883544921875,
      "logps/rejected": -437.17681884765625,
      "loss": 0.008,
      "rewards/accuracies": 0.9957500100135803,
      "rewards/chosen": -7.342708110809326,
      "rewards/margins": 13.04981803894043,
      "rewards/rejected": -20.392526626586914,
      "step": 19000
    },
    {
      "epoch": 8.11965811965812,
      "eval_logits/chosen": -34.412513732910156,
      "eval_logits/rejected": -35.9024658203125,
      "eval_logps/chosen": -388.45513916015625,
      "eval_logps/rejected": -375.42486572265625,
      "eval_loss": 2.118881940841675,
      "eval_rewards/accuracies": 0.5604477524757385,
      "eval_rewards/chosen": -13.356197357177734,
      "eval_rewards/margins": 1.0303833484649658,
      "eval_rewards/rejected": -14.386581420898438,
      "eval_runtime": 227.1891,
      "eval_samples_per_second": 11.766,
      "eval_steps_per_second": 1.475,
      "step": 19000
    },
    {
      "epoch": 8.333333333333334,
      "grad_norm": 0.035114262253046036,
      "learning_rate": 8.51528384279476e-06,
      "logits/chosen": -33.662052154541016,
      "logits/rejected": -34.972896575927734,
      "logps/chosen": -332.39031982421875,
      "logps/rejected": -444.69793701171875,
      "loss": 0.0146,
      "rewards/accuracies": 0.9940000176429749,
      "rewards/chosen": -7.840022563934326,
      "rewards/margins": 13.208553314208984,
      "rewards/rejected": -21.04857635498047,
      "step": 19500
    },
    {
      "epoch": 8.333333333333334,
      "eval_logits/chosen": -34.22267532348633,
      "eval_logits/rejected": -35.91259765625,
      "eval_logps/chosen": -386.5843200683594,
      "eval_logps/rejected": -373.9736022949219,
      "eval_loss": 2.091254711151123,
      "eval_rewards/accuracies": 0.5593283772468567,
      "eval_rewards/chosen": -13.169110298156738,
      "eval_rewards/margins": 1.0723390579223633,
      "eval_rewards/rejected": -14.241450309753418,
      "eval_runtime": 227.9067,
      "eval_samples_per_second": 11.728,
      "eval_steps_per_second": 1.47,
      "step": 19500
    },
    {
      "epoch": 8.547008547008547,
      "grad_norm": 0.001891271909698844,
      "learning_rate": 7.423580786026202e-06,
      "logits/chosen": -33.975555419921875,
      "logits/rejected": -35.19069290161133,
      "logps/chosen": -332.5311279296875,
      "logps/rejected": -444.55194091796875,
      "loss": 0.01,
      "rewards/accuracies": 0.9952499866485596,
      "rewards/chosen": -7.8769989013671875,
      "rewards/margins": 13.22789192199707,
      "rewards/rejected": -21.104890823364258,
      "step": 20000
    },
    {
      "epoch": 8.547008547008547,
      "eval_logits/chosen": -34.138275146484375,
      "eval_logits/rejected": -35.49238586425781,
      "eval_logps/chosen": -389.0328369140625,
      "eval_logps/rejected": -376.4501953125,
      "eval_loss": 2.1092472076416016,
      "eval_rewards/accuracies": 0.5593283772468567,
      "eval_rewards/chosen": -13.413966178894043,
      "eval_rewards/margins": 1.0751436948776245,
      "eval_rewards/rejected": -14.48910903930664,
      "eval_runtime": 227.7973,
      "eval_samples_per_second": 11.734,
      "eval_steps_per_second": 1.471,
      "step": 20000
    },
    {
      "epoch": 8.760683760683762,
      "grad_norm": 1.0492886304855347,
      "learning_rate": 6.3318777292576415e-06,
      "logits/chosen": -33.439266204833984,
      "logits/rejected": -34.95777893066406,
      "logps/chosen": -331.6947326660156,
      "logps/rejected": -441.31298828125,
      "loss": 0.0126,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.511509418487549,
      "rewards/margins": 13.192078590393066,
      "rewards/rejected": -20.70358657836914,
      "step": 20500
    },
    {
      "epoch": 8.760683760683762,
      "eval_logits/chosen": -34.35652542114258,
      "eval_logits/rejected": -36.15835952758789,
      "eval_logps/chosen": -387.4725646972656,
      "eval_logps/rejected": -375.294189453125,
      "eval_loss": 2.09629487991333,
      "eval_rewards/accuracies": 0.562313437461853,
      "eval_rewards/chosen": -13.25793743133545,
      "eval_rewards/margins": 1.115572452545166,
      "eval_rewards/rejected": -14.373512268066406,
      "eval_runtime": 227.8406,
      "eval_samples_per_second": 11.732,
      "eval_steps_per_second": 1.47,
      "step": 20500
    },
    {
      "epoch": 8.974358974358974,
      "grad_norm": 6.9876251220703125,
      "learning_rate": 5.240174672489083e-06,
      "logits/chosen": -33.97458267211914,
      "logits/rejected": -35.557865142822266,
      "logps/chosen": -330.39239501953125,
      "logps/rejected": -444.573486328125,
      "loss": 0.0119,
      "rewards/accuracies": 0.9944999814033508,
      "rewards/chosen": -7.664352893829346,
      "rewards/margins": 13.433502197265625,
      "rewards/rejected": -21.09785270690918,
      "step": 21000
    },
    {
      "epoch": 8.974358974358974,
      "eval_logits/chosen": -34.37322235107422,
      "eval_logits/rejected": -36.26004409790039,
      "eval_logps/chosen": -386.285888671875,
      "eval_logps/rejected": -373.96063232421875,
      "eval_loss": 2.07737135887146,
      "eval_rewards/accuracies": 0.5649253726005554,
      "eval_rewards/chosen": -13.139269828796387,
      "eval_rewards/margins": 1.1008845567703247,
      "eval_rewards/rejected": -14.240155220031738,
      "eval_runtime": 227.6828,
      "eval_samples_per_second": 11.74,
      "eval_steps_per_second": 1.471,
      "step": 21000
    },
    {
      "epoch": 9.188034188034187,
      "grad_norm": 0.0024337973445653915,
      "learning_rate": 4.1484716157205246e-06,
      "logits/chosen": -33.696048736572266,
      "logits/rejected": -35.321414947509766,
      "logps/chosen": -330.987548828125,
      "logps/rejected": -445.3059387207031,
      "loss": 0.007,
      "rewards/accuracies": 0.9957500100135803,
      "rewards/chosen": -7.504194736480713,
      "rewards/margins": 13.350516319274902,
      "rewards/rejected": -20.854713439941406,
      "step": 21500
    },
    {
      "epoch": 9.188034188034187,
      "eval_logits/chosen": -34.3525505065918,
      "eval_logits/rejected": -36.3557243347168,
      "eval_logps/chosen": -384.0542297363281,
      "eval_logps/rejected": -371.8487243652344,
      "eval_loss": 2.050776958465576,
      "eval_rewards/accuracies": 0.5686567425727844,
      "eval_rewards/chosen": -12.916106224060059,
      "eval_rewards/margins": 1.1128581762313843,
      "eval_rewards/rejected": -14.028963088989258,
      "eval_runtime": 227.9747,
      "eval_samples_per_second": 11.725,
      "eval_steps_per_second": 1.469,
      "step": 21500
    },
    {
      "epoch": 9.401709401709402,
      "grad_norm": 0.03277166187763214,
      "learning_rate": 3.056768558951965e-06,
      "logits/chosen": -34.06507873535156,
      "logits/rejected": -35.69638442993164,
      "logps/chosen": -332.1161804199219,
      "logps/rejected": -444.4291076660156,
      "loss": 0.0106,
      "rewards/accuracies": 0.9950000047683716,
      "rewards/chosen": -7.72641658782959,
      "rewards/margins": 13.240401268005371,
      "rewards/rejected": -20.966815948486328,
      "step": 22000
    },
    {
      "epoch": 9.401709401709402,
      "eval_logits/chosen": -34.42190933227539,
      "eval_logits/rejected": -36.19350051879883,
      "eval_logps/chosen": -387.06207275390625,
      "eval_logps/rejected": -374.70416259765625,
      "eval_loss": 2.0756402015686035,
      "eval_rewards/accuracies": 0.5649253726005554,
      "eval_rewards/chosen": -13.216888427734375,
      "eval_rewards/margins": 1.0976189374923706,
      "eval_rewards/rejected": -14.314507484436035,
      "eval_runtime": 227.5352,
      "eval_samples_per_second": 11.748,
      "eval_steps_per_second": 1.472,
      "step": 22000
    }
  ],
  "logging_steps": 500,
  "max_steps": 23400,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 10,
  "save_steps": 1000,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}