{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.686698295554384,
  "eval_steps": 200,
  "global_step": 2400,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.035143208574942894,
      "grad_norm": Infinity,
      "learning_rate": 3.066666666666666e-07,
      "logits/chosen": -0.9749129414558411,
      "logits/rejected": -0.5760761499404907,
      "logps/chosen": -377.1766357421875,
      "logps/rejected": -303.9364929199219,
      "loss": 0.6863,
      "rewards/accuracies": 0.5674999952316284,
      "rewards/chosen": 0.004484061151742935,
      "rewards/margins": 0.014561166986823082,
      "rewards/rejected": -0.010077104903757572,
      "step": 50
    },
    {
      "epoch": 0.07028641714988579,
      "grad_norm": 118.19742584228516,
      "learning_rate": 6.333333333333332e-07,
      "logits/chosen": -0.7770336270332336,
      "logits/rejected": -0.5940005779266357,
      "logps/chosen": -369.6046447753906,
      "logps/rejected": -307.9879455566406,
      "loss": 0.64,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.05502090975642204,
      "rewards/margins": 0.13479986786842346,
      "rewards/rejected": -0.07977895438671112,
      "step": 100
    },
    {
      "epoch": 0.10542962572482868,
      "grad_norm": 95.5091552734375,
      "learning_rate": 9.666666666666666e-07,
      "logits/chosen": -0.7478348612785339,
      "logits/rejected": -0.3990992307662964,
      "logps/chosen": -372.7549133300781,
      "logps/rejected": -293.626708984375,
      "loss": 0.6121,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.017337413504719734,
      "rewards/margins": 0.2657831609249115,
      "rewards/rejected": -0.2484457641839981,
      "step": 150
    },
    {
      "epoch": 0.14057283429977158,
      "grad_norm": 129.95443725585938,
      "learning_rate": 9.997055316896116e-07,
      "logits/chosen": -0.8806526064872742,
      "logits/rejected": -0.6725770831108093,
      "logps/chosen": -357.8478088378906,
      "logps/rejected": -275.65130615234375,
      "loss": 0.5845,
      "rewards/accuracies": 0.6924999952316284,
      "rewards/chosen": 0.028637664392590523,
      "rewards/margins": 0.37213996052742004,
      "rewards/rejected": -0.34350234270095825,
      "step": 200
    },
    {
      "epoch": 0.14057283429977158,
      "eval_logits/chosen": -0.849940836429596,
      "eval_logits/rejected": -0.8586259484291077,
      "eval_logps/chosen": -126.63912963867188,
      "eval_logps/rejected": -153.3675079345703,
      "eval_loss": 0.6478162407875061,
      "eval_rewards/accuracies": 0.8101449012756348,
      "eval_rewards/chosen": -0.11397092789411545,
      "eval_rewards/margins": 0.11266002058982849,
      "eval_rewards/rejected": -0.22663094103336334,
      "eval_runtime": 44.6372,
      "eval_samples_per_second": 15.458,
      "eval_steps_per_second": 7.729,
      "step": 200
    },
    {
      "epoch": 0.17571604287471446,
      "grad_norm": 106.08065032958984,
      "learning_rate": 9.986880618329365e-07,
      "logits/chosen": -0.9304916858673096,
      "logits/rejected": -0.6669905185699463,
      "logps/chosen": -366.1148681640625,
      "logps/rejected": -284.8680725097656,
      "loss": 0.5127,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": 0.07616949081420898,
      "rewards/margins": 0.6659303307533264,
      "rewards/rejected": -0.5897608399391174,
      "step": 250
    },
    {
      "epoch": 0.21085925144965736,
      "grad_norm": 124.82818603515625,
      "learning_rate": 9.96945434200719e-07,
      "logits/chosen": -0.853944718837738,
      "logits/rejected": -0.5244762301445007,
      "logps/chosen": -367.5785827636719,
      "logps/rejected": -304.6045837402344,
      "loss": 0.5569,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.10503432899713516,
      "rewards/margins": 0.5734320282936096,
      "rewards/rejected": -0.6784663200378418,
      "step": 300
    },
    {
      "epoch": 0.24600246002460024,
      "grad_norm": 66.96507263183594,
      "learning_rate": 9.944801828018398e-07,
      "logits/chosen": -0.8821534514427185,
      "logits/rejected": -0.5591131448745728,
      "logps/chosen": -389.2870178222656,
      "logps/rejected": -304.467041015625,
      "loss": 0.5473,
      "rewards/accuracies": 0.7225000262260437,
      "rewards/chosen": -0.15399029850959778,
      "rewards/margins": 0.7008799910545349,
      "rewards/rejected": -0.8548702001571655,
      "step": 350
    },
    {
      "epoch": 0.28114566859954315,
      "grad_norm": 129.39100646972656,
      "learning_rate": 9.912958924348081e-07,
      "logits/chosen": -1.1357834339141846,
      "logits/rejected": -0.7058276534080505,
      "logps/chosen": -364.6184387207031,
      "logps/rejected": -316.7001037597656,
      "loss": 0.4865,
      "rewards/accuracies": 0.7649999856948853,
      "rewards/chosen": -0.28908756375312805,
      "rewards/margins": 0.9275381565093994,
      "rewards/rejected": -1.216625690460205,
      "step": 400
    },
    {
      "epoch": 0.28114566859954315,
      "eval_logits/chosen": -1.0553535223007202,
      "eval_logits/rejected": -1.0848628282546997,
      "eval_logps/chosen": -130.27931213378906,
      "eval_logps/rejected": -157.7180633544922,
      "eval_loss": 0.6393592953681946,
      "eval_rewards/accuracies": 0.7942029237747192,
      "eval_rewards/chosen": -0.4779890775680542,
      "eval_rewards/margins": 0.18369804322719574,
      "eval_rewards/rejected": -0.6616871356964111,
      "eval_runtime": 44.5728,
      "eval_samples_per_second": 15.48,
      "eval_steps_per_second": 7.74,
      "step": 400
    },
    {
      "epoch": 0.316288877174486,
      "grad_norm": 106.56148529052734,
      "learning_rate": 9.873971934749946e-07,
      "logits/chosen": -1.1408377885818481,
      "logits/rejected": -0.8068557977676392,
      "logps/chosen": -359.27313232421875,
      "logps/rejected": -309.91424560546875,
      "loss": 0.5123,
      "rewards/accuracies": 0.7325000166893005,
      "rewards/chosen": -0.2605482041835785,
      "rewards/margins": 0.8376699686050415,
      "rewards/rejected": -1.0982180833816528,
      "step": 450
    },
    {
      "epoch": 0.3514320857494289,
      "grad_norm": 111.91079711914062,
      "learning_rate": 9.827897551414597e-07,
      "logits/chosen": -1.0800330638885498,
      "logits/rejected": -0.7713271379470825,
      "logps/chosen": -383.8734130859375,
      "logps/rejected": -310.9490661621094,
      "loss": 0.5023,
      "rewards/accuracies": 0.7425000071525574,
      "rewards/chosen": -0.2419007271528244,
      "rewards/margins": 0.945677638053894,
      "rewards/rejected": -1.1875784397125244,
      "step": 500
    },
    {
      "epoch": 0.3865752943243718,
      "grad_norm": 165.75318908691406,
      "learning_rate": 9.77593298534426e-07,
      "logits/chosen": -1.2925091981887817,
      "logits/rejected": -1.0452824831008911,
      "logps/chosen": -394.8578796386719,
      "logps/rejected": -325.94842529296875,
      "loss": 0.4754,
      "rewards/accuracies": 0.7400000095367432,
      "rewards/chosen": -0.2650391459465027,
      "rewards/margins": 1.0042325258255005,
      "rewards/rejected": -1.269271731376648,
      "step": 550
    },
    {
      "epoch": 0.42171850289931473,
      "grad_norm": 93.74369812011719,
      "learning_rate": 9.71603305932502e-07,
      "logits/chosen": -1.145042896270752,
      "logits/rejected": -0.9232679009437561,
      "logps/chosen": -358.3184509277344,
      "logps/rejected": -306.0023498535156,
      "loss": 0.4767,
      "rewards/accuracies": 0.7825000286102295,
      "rewards/chosen": -0.32229503989219666,
      "rewards/margins": 1.0345232486724854,
      "rewards/rejected": -1.3568181991577148,
      "step": 600
    },
    {
      "epoch": 0.42171850289931473,
      "eval_logits/chosen": -1.2382392883300781,
      "eval_logits/rejected": -1.2902898788452148,
      "eval_logps/chosen": -133.6689910888672,
      "eval_logps/rejected": -161.33534240722656,
      "eval_loss": 0.6566920876502991,
      "eval_rewards/accuracies": 0.7739130258560181,
      "eval_rewards/chosen": -0.8169569969177246,
      "eval_rewards/margins": 0.2064574807882309,
      "eval_rewards/rejected": -1.0234146118164062,
      "eval_runtime": 44.7437,
      "eval_samples_per_second": 15.421,
      "eval_steps_per_second": 7.711,
      "step": 600
    },
    {
      "epoch": 0.4568617114742576,
      "grad_norm": 174.93446350097656,
      "learning_rate": 9.64927540338517e-07,
      "logits/chosen": -1.0424299240112305,
      "logits/rejected": -0.7404126524925232,
      "logps/chosen": -419.05157470703125,
      "logps/rejected": -319.6670227050781,
      "loss": 0.4469,
      "rewards/accuracies": 0.7724999785423279,
      "rewards/chosen": -0.3416347801685333,
      "rewards/margins": 1.252581238746643,
      "rewards/rejected": -1.5942161083221436,
      "step": 650
    },
    {
      "epoch": 0.4920049200492005,
      "grad_norm": 96.59870147705078,
      "learning_rate": 9.57575709190243e-07,
      "logits/chosen": -1.128604531288147,
      "logits/rejected": -0.8932415246963501,
      "logps/chosen": -401.80511474609375,
      "logps/rejected": -310.9868469238281,
      "loss": 0.4501,
      "rewards/accuracies": 0.7925000190734863,
      "rewards/chosen": -0.5053700804710388,
      "rewards/margins": 1.3491663932800293,
      "rewards/rejected": -1.854536533355713,
      "step": 700
    },
    {
      "epoch": 0.5271481286241434,
      "grad_norm": 107.97344207763672,
      "learning_rate": 9.49558503013341e-07,
      "logits/chosen": -1.060742974281311,
      "logits/rejected": -0.7900027632713318,
      "logps/chosen": -363.7101135253906,
      "logps/rejected": -296.1370849609375,
      "loss": 0.4578,
      "rewards/accuracies": 0.7774999737739563,
      "rewards/chosen": -0.41928350925445557,
      "rewards/margins": 1.2345094680786133,
      "rewards/rejected": -1.6537928581237793,
      "step": 750
    },
    {
      "epoch": 0.5622913371990863,
      "grad_norm": 115.42318725585938,
      "learning_rate": 9.408875798759345e-07,
      "logits/chosen": -1.1210286617279053,
      "logits/rejected": -0.8719848394393921,
      "logps/chosen": -383.5552673339844,
      "logps/rejected": -314.79168701171875,
      "loss": 0.5061,
      "rewards/accuracies": 0.7425000071525574,
      "rewards/chosen": -0.5828273892402649,
      "rewards/margins": 1.1859365701675415,
      "rewards/rejected": -1.7687640190124512,
      "step": 800
    },
    {
      "epoch": 0.5622913371990863,
      "eval_logits/chosen": -1.3269339799880981,
      "eval_logits/rejected": -1.360857605934143,
      "eval_logps/chosen": -136.08132934570312,
      "eval_logps/rejected": -163.81300354003906,
      "eval_loss": 0.6693784594535828,
      "eval_rewards/accuracies": 0.7550724744796753,
      "eval_rewards/chosen": -1.0581910610198975,
      "eval_rewards/margins": 0.21298907697200775,
      "eval_rewards/rejected": -1.271180272102356,
      "eval_runtime": 44.723,
      "eval_samples_per_second": 15.428,
      "eval_steps_per_second": 7.714,
      "step": 800
    },
    {
      "epoch": 0.5974345457740292,
      "grad_norm": 126.08724975585938,
      "learning_rate": 9.315755484362518e-07,
      "logits/chosen": -1.0859259366989136,
      "logits/rejected": -0.7706220149993896,
      "logps/chosen": -381.0041809082031,
      "logps/rejected": -318.6121826171875,
      "loss": 0.4827,
      "rewards/accuracies": 0.8025000095367432,
      "rewards/chosen": -0.5592462420463562,
      "rewards/margins": 1.1655195951461792,
      "rewards/rejected": -1.7247658967971802,
      "step": 850
    },
    {
      "epoch": 0.632577754348972,
      "grad_norm": 77.59442901611328,
      "learning_rate": 9.216359496079851e-07,
      "logits/chosen": -1.0959984064102173,
      "logits/rejected": -0.7341098189353943,
      "logps/chosen": -379.558837890625,
      "logps/rejected": -315.5170593261719,
      "loss": 0.4108,
      "rewards/accuracies": 0.8100000023841858,
      "rewards/chosen": -0.43343108892440796,
      "rewards/margins": 1.471386432647705,
      "rewards/rejected": -1.9048174619674683,
      "step": 900
    },
    {
      "epoch": 0.6677209629239149,
      "grad_norm": 93.53849792480469,
      "learning_rate": 9.110832368700279e-07,
      "logits/chosen": -1.11549711227417,
      "logits/rejected": -0.8405827283859253,
      "logps/chosen": -380.1915588378906,
      "logps/rejected": -323.5362548828125,
      "loss": 0.4607,
      "rewards/accuracies": 0.8100000023841858,
      "rewards/chosen": -0.6118125319480896,
      "rewards/margins": 1.3506970405578613,
      "rewards/rejected": -1.9625096321105957,
      "step": 950
    },
    {
      "epoch": 0.7028641714988578,
      "grad_norm": 69.78710174560547,
      "learning_rate": 8.999327552492229e-07,
      "logits/chosen": -1.0321812629699707,
      "logits/rejected": -0.8490170836448669,
      "logps/chosen": -370.0117492675781,
      "logps/rejected": -320.3147888183594,
      "loss": 0.518,
      "rewards/accuracies": 0.7400000095367432,
      "rewards/chosen": -0.527043342590332,
      "rewards/margins": 1.2954468727111816,
      "rewards/rejected": -1.8224902153015137,
      "step": 1000
    },
    {
      "epoch": 0.7028641714988578,
      "eval_logits/chosen": -1.3353734016418457,
      "eval_logits/rejected": -1.3637131452560425,
      "eval_logps/chosen": -137.6568603515625,
      "eval_logps/rejected": -165.5953369140625,
      "eval_loss": 0.6648596525192261,
      "eval_rewards/accuracies": 0.760869562625885,
      "eval_rewards/chosen": -1.2157429456710815,
      "eval_rewards/margins": 0.23366999626159668,
      "eval_rewards/rejected": -1.4494129419326782,
      "eval_runtime": 44.7345,
      "eval_samples_per_second": 15.424,
      "eval_steps_per_second": 7.712,
      "step": 1000
    },
    {
      "epoch": 0.7380073800738007,
      "grad_norm": 73.34844970703125,
      "learning_rate": 8.882007190066827e-07,
      "logits/chosen": -1.0483229160308838,
      "logits/rejected": -0.716176450252533,
      "logps/chosen": -356.2257080078125,
      "logps/rejected": -316.9234619140625,
      "loss": 0.4831,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.5482863187789917,
      "rewards/margins": 1.286440134048462,
      "rewards/rejected": -1.8347265720367432,
      "step": 1050
    },
    {
      "epoch": 0.7731505886487436,
      "grad_norm": 179.53550720214844,
      "learning_rate": 8.759041880601303e-07,
      "logits/chosen": -1.1389548778533936,
      "logits/rejected": -0.8437424302101135,
      "logps/chosen": -360.9399719238281,
      "logps/rejected": -327.319580078125,
      "loss": 0.4644,
      "rewards/accuracies": 0.7799999713897705,
      "rewards/chosen": -0.5854690074920654,
      "rewards/margins": 1.4202189445495605,
      "rewards/rejected": -2.005687952041626,
      "step": 1100
    },
    {
      "epoch": 0.8082937972236865,
      "grad_norm": 144.1688232421875,
      "learning_rate": 8.630610431765432e-07,
      "logits/chosen": -1.1540454626083374,
      "logits/rejected": -0.8791725635528564,
      "logps/chosen": -358.11578369140625,
      "logps/rejected": -325.086669921875,
      "loss": 0.4801,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.61565762758255,
      "rewards/margins": 1.2607437372207642,
      "rewards/rejected": -1.876401662826538,
      "step": 1150
    },
    {
      "epoch": 0.8434370057986295,
      "grad_norm": 87.58560943603516,
      "learning_rate": 8.496899599711758e-07,
      "logits/chosen": -1.1453388929367065,
      "logits/rejected": -0.8830125331878662,
      "logps/chosen": -380.4150390625,
      "logps/rejected": -328.78448486328125,
      "loss": 0.4577,
      "rewards/accuracies": 0.7950000166893005,
      "rewards/chosen": -0.6934986114501953,
      "rewards/margins": 1.3581026792526245,
      "rewards/rejected": -2.0516011714935303,
      "step": 1200
    },
    {
      "epoch": 0.8434370057986295,
      "eval_logits/chosen": -1.4031161069869995,
      "eval_logits/rejected": -1.433149814605713,
      "eval_logps/chosen": -138.8860321044922,
      "eval_logps/rejected": -167.08187866210938,
      "eval_loss": 0.6591169834136963,
      "eval_rewards/accuracies": 0.769565224647522,
      "eval_rewards/chosen": -1.3386619091033936,
      "eval_rewards/margins": 0.259408563375473,
      "eval_rewards/rejected": -1.5980706214904785,
      "eval_runtime": 44.7054,
      "eval_samples_per_second": 15.434,
      "eval_steps_per_second": 7.717,
      "step": 1200
    },
    {
      "epoch": 0.8785802143735723,
      "grad_norm": 62.81058883666992,
      "learning_rate": 8.358103817507679e-07,
      "logits/chosen": -1.0958822965621948,
      "logits/rejected": -0.7899726629257202,
      "logps/chosen": -380.1678161621094,
      "logps/rejected": -310.56219482421875,
      "loss": 0.4387,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.7097710371017456,
      "rewards/margins": 1.4620877504348755,
      "rewards/rejected": -2.171858787536621,
      "step": 1250
    },
    {
      "epoch": 0.9137234229485152,
      "grad_norm": 163.62998962402344,
      "learning_rate": 8.214424912404299e-07,
      "logits/chosen": -1.0615484714508057,
      "logits/rejected": -0.8271848559379578,
      "logps/chosen": -365.4388732910156,
      "logps/rejected": -329.86773681640625,
      "loss": 0.466,
      "rewards/accuracies": 0.7649999856948853,
      "rewards/chosen": -0.8122978210449219,
      "rewards/margins": 1.3591680526733398,
      "rewards/rejected": -2.1714658737182617,
      "step": 1300
    },
    {
      "epoch": 0.9488666315234581,
      "grad_norm": 123.21900939941406,
      "learning_rate": 8.066071812353162e-07,
      "logits/chosen": -1.2160460948944092,
      "logits/rejected": -0.9520894885063171,
      "logps/chosen": -378.2143249511719,
      "logps/rejected": -321.5575256347656,
      "loss": 0.4541,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.6522579789161682,
      "rewards/margins": 1.3640750646591187,
      "rewards/rejected": -2.0163331031799316,
      "step": 1350
    },
    {
      "epoch": 0.984009840098401,
      "grad_norm": 111.00737762451172,
      "learning_rate": 7.913260242197632e-07,
      "logits/chosen": -1.2698862552642822,
      "logits/rejected": -1.0099891424179077,
      "logps/chosen": -360.263427734375,
      "logps/rejected": -302.4189147949219,
      "loss": 0.4738,
      "rewards/accuracies": 0.7599999904632568,
      "rewards/chosen": -0.7399062514305115,
      "rewards/margins": 1.3410731554031372,
      "rewards/rejected": -2.080979347229004,
      "step": 1400
    },
    {
      "epoch": 0.984009840098401,
      "eval_logits/chosen": -1.3625684976577759,
      "eval_logits/rejected": -1.3784757852554321,
      "eval_logps/chosen": -138.63931274414062,
      "eval_logps/rejected": -166.5675506591797,
      "eval_loss": 0.6629493236541748,
      "eval_rewards/accuracies": 0.7579709887504578,
      "eval_rewards/chosen": -1.3139899969100952,
      "eval_rewards/margins": 0.23264573514461517,
      "eval_rewards/rejected": -1.5466355085372925,
      "eval_runtime": 44.7686,
      "eval_samples_per_second": 15.413,
      "eval_steps_per_second": 7.706,
      "step": 1400
    },
    {
      "epoch": 1.0189773326304692,
      "grad_norm": 25.185258865356445,
      "learning_rate": 7.756212409980718e-07,
      "logits/chosen": -1.2253597974777222,
      "logits/rejected": -0.9460395574569702,
      "logps/chosen": -358.0578308105469,
      "logps/rejected": -312.42144775390625,
      "loss": 0.2854,
      "rewards/accuracies": 0.8718593120574951,
      "rewards/chosen": -0.2665604054927826,
      "rewards/margins": 2.4454708099365234,
      "rewards/rejected": -2.712031126022339,
      "step": 1450
    },
    {
      "epoch": 1.054120541205412,
      "grad_norm": 47.889488220214844,
      "learning_rate": 7.595156683825462e-07,
      "logits/chosen": -1.3037954568862915,
      "logits/rejected": -1.0524487495422363,
      "logps/chosen": -385.7896728515625,
      "logps/rejected": -340.14324951171875,
      "loss": 0.1089,
      "rewards/accuracies": 0.9725000262260437,
      "rewards/chosen": 0.14200635254383087,
      "rewards/margins": 3.677000045776367,
      "rewards/rejected": -3.5349936485290527,
      "step": 1500
    },
    {
      "epoch": 1.089263749780355,
      "grad_norm": 7.548559665679932,
      "learning_rate": 7.430327259857772e-07,
      "logits/chosen": -1.3957204818725586,
      "logits/rejected": -1.155929446220398,
      "logps/chosen": -379.749267578125,
      "logps/rejected": -325.2372741699219,
      "loss": 0.1278,
      "rewards/accuracies": 0.9674999713897705,
      "rewards/chosen": 0.007118640001863241,
      "rewards/margins": 3.5875449180603027,
      "rewards/rejected": -3.5804266929626465,
      "step": 1550
    },
    {
      "epoch": 1.1244069583552978,
      "grad_norm": 10.904348373413086,
      "learning_rate": 7.261963821654566e-07,
      "logits/chosen": -1.4134939908981323,
      "logits/rejected": -1.1053929328918457,
      "logps/chosen": -354.8056335449219,
      "logps/rejected": -326.7030029296875,
      "loss": 0.1133,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.18568024039268494,
      "rewards/margins": 3.7941460609436035,
      "rewards/rejected": -3.9798266887664795,
      "step": 1600
    },
    {
      "epoch": 1.1244069583552978,
      "eval_logits/chosen": -1.8039968013763428,
      "eval_logits/rejected": -1.8495479822158813,
      "eval_logps/chosen": -146.17001342773438,
      "eval_logps/rejected": -175.72039794921875,
      "eval_loss": 0.6601921319961548,
      "eval_rewards/accuracies": 0.7811594009399414,
      "eval_rewards/chosen": -2.0670576095581055,
      "eval_rewards/margins": 0.3948650062084198,
      "eval_rewards/rejected": -2.4619228839874268,
      "eval_runtime": 45.1384,
      "eval_samples_per_second": 15.286,
      "eval_steps_per_second": 7.643,
      "step": 1600
    },
    {
      "epoch": 1.1595501669302408,
      "grad_norm": 20.5295352935791,
      "learning_rate": 7.090311191712463e-07,
      "logits/chosen": -1.632176399230957,
      "logits/rejected": -1.37174654006958,
      "logps/chosen": -380.2718505859375,
      "logps/rejected": -352.9892578125,
      "loss": 0.0908,
      "rewards/accuracies": 0.9800000190734863,
      "rewards/chosen": -0.38161003589630127,
      "rewards/margins": 4.1729817390441895,
      "rewards/rejected": -4.554591655731201,
      "step": 1650
    },
    {
      "epoch": 1.1946933755051836,
      "grad_norm": 21.824186325073242,
      "learning_rate": 6.915618975443784e-07,
      "logits/chosen": -1.6156671047210693,
      "logits/rejected": -1.4324374198913574,
      "logps/chosen": -387.204345703125,
      "logps/rejected": -339.71954345703125,
      "loss": 0.1449,
      "rewards/accuracies": 0.9449999928474426,
      "rewards/chosen": -0.5762972235679626,
      "rewards/margins": 4.003431797027588,
      "rewards/rejected": -4.579729080200195,
      "step": 1700
    },
    {
      "epoch": 1.2298365840801266,
      "grad_norm": 66.74718475341797,
      "learning_rate": 6.738141198217591e-07,
      "logits/chosen": -1.6413328647613525,
      "logits/rejected": -1.3644522428512573,
      "logps/chosen": -363.44366455078125,
      "logps/rejected": -322.4190368652344,
      "loss": 0.1191,
      "rewards/accuracies": 0.9599999785423279,
      "rewards/chosen": -0.37116390466690063,
      "rewards/margins": 3.9141128063201904,
      "rewards/rejected": -4.285276889801025,
      "step": 1750
    },
    {
      "epoch": 1.2649797926550694,
      "grad_norm": 19.022262573242188,
      "learning_rate": 6.558135935973521e-07,
      "logits/chosen": -1.715476632118225,
      "logits/rejected": -1.4770665168762207,
      "logps/chosen": -366.1575012207031,
      "logps/rejected": -329.7523193359375,
      "loss": 0.1085,
      "rewards/accuracies": 0.9725000262260437,
      "rewards/chosen": -0.46892687678337097,
      "rewards/margins": 4.068582534790039,
      "rewards/rejected": -4.537509918212891,
      "step": 1800
    },
    {
      "epoch": 1.2649797926550694,
      "eval_logits/chosen": -1.8528786897659302,
      "eval_logits/rejected": -1.8930330276489258,
      "eval_logps/chosen": -147.26092529296875,
      "eval_logps/rejected": -177.1022491455078,
      "eval_loss": 0.6557580232620239,
      "eval_rewards/accuracies": 0.7840579748153687,
      "eval_rewards/chosen": -2.1761505603790283,
      "eval_rewards/margins": 0.4239543676376343,
      "eval_rewards/rejected": -2.600104808807373,
      "eval_runtime": 44.8492,
      "eval_samples_per_second": 15.385,
      "eval_steps_per_second": 7.692,
      "step": 1800
    },
    {
      "epoch": 1.3001230012300122,
      "grad_norm": 27.70209503173828,
      "learning_rate": 6.375864939945548e-07,
      "logits/chosen": -1.5819531679153442,
      "logits/rejected": -1.3250428438186646,
      "logps/chosen": -358.1719665527344,
      "logps/rejected": -327.7480773925781,
      "loss": 0.101,
      "rewards/accuracies": 0.9775000214576721,
      "rewards/chosen": -0.38992682099342346,
      "rewards/margins": 4.033656597137451,
      "rewards/rejected": -4.423582553863525,
      "step": 1850
    },
    {
      "epoch": 1.3352662098049553,
      "grad_norm": 17.87761878967285,
      "learning_rate": 6.191593256041411e-07,
      "logits/chosen": -1.6258714199066162,
      "logits/rejected": -1.3926899433135986,
      "logps/chosen": -362.2528381347656,
      "logps/rejected": -326.35888671875,
      "loss": 0.1036,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.3824487626552582,
      "rewards/margins": 4.142745494842529,
      "rewards/rejected": -4.52519416809082,
      "step": 1900
    },
    {
      "epoch": 1.370409418379898,
      "grad_norm": 6.942252159118652,
      "learning_rate": 6.005588839431128e-07,
      "logits/chosen": -1.7623356580734253,
      "logits/rejected": -1.4591953754425049,
      "logps/chosen": -347.2608947753906,
      "logps/rejected": -348.46173095703125,
      "loss": 0.1031,
      "rewards/accuracies": 0.9674999713897705,
      "rewards/chosen": -0.635872483253479,
      "rewards/margins": 4.266874313354492,
      "rewards/rejected": -4.902746677398682,
      "step": 1950
    },
    {
      "epoch": 1.405552626954841,
      "grad_norm": 18.128665924072266,
      "learning_rate": 5.81812216490508e-07,
      "logits/chosen": -1.6600449085235596,
      "logits/rejected": -1.6073315143585205,
      "logps/chosen": -392.4981384277344,
      "logps/rejected": -345.29913330078125,
      "loss": 0.1079,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.8337967395782471,
      "rewards/margins": 4.29391622543335,
      "rewards/rejected": -5.127712249755859,
      "step": 2000
    },
    {
      "epoch": 1.405552626954841,
      "eval_logits/chosen": -2.016140937805176,
      "eval_logits/rejected": -2.087994337081909,
      "eval_logps/chosen": -151.57797241210938,
      "eval_logps/rejected": -181.83580017089844,
      "eval_loss": 0.6645065546035767,
      "eval_rewards/accuracies": 0.7840579748153687,
      "eval_rewards/chosen": -2.607856035232544,
      "eval_rewards/margins": 0.4656042158603668,
      "eval_rewards/rejected": -3.073460102081299,
      "eval_runtime": 45.2045,
      "eval_samples_per_second": 15.264,
      "eval_steps_per_second": 7.632,
      "step": 2000
    },
    {
      "epoch": 1.4406958355297839,
      "grad_norm": 132.83291625976562,
      "learning_rate": 5.629465833568234e-07,
      "logits/chosen": -1.6978211402893066,
      "logits/rejected": -1.537874698638916,
      "logps/chosen": -382.0157165527344,
      "logps/rejected": -358.0167541503906,
      "loss": 0.1307,
      "rewards/accuracies": 0.9599999785423279,
      "rewards/chosen": -0.7373409271240234,
      "rewards/margins": 4.2708845138549805,
      "rewards/rejected": -5.008224964141846,
      "step": 2050
    },
    {
      "epoch": 1.4758390441047267,
      "grad_norm": 5.127355575561523,
      "learning_rate": 5.439894176442409e-07,
      "logits/chosen": -1.7916449308395386,
      "logits/rejected": -1.5541598796844482,
      "logps/chosen": -390.88214111328125,
      "logps/rejected": -364.38250732421875,
      "loss": 0.0953,
      "rewards/accuracies": 0.9700000286102295,
      "rewards/chosen": -0.5856018662452698,
      "rewards/margins": 4.4867262840271,
      "rewards/rejected": -5.072328090667725,
      "step": 2100
    },
    {
      "epoch": 1.5109822526796697,
      "grad_norm": 11.737075805664062,
      "learning_rate": 5.24968285555305e-07,
      "logits/chosen": -1.7472648620605469,
      "logits/rejected": -1.5027780532836914,
      "logps/chosen": -390.71307373046875,
      "logps/rejected": -361.4422607421875,
      "loss": 0.1223,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.861807644367218,
      "rewards/margins": 4.3698410987854,
      "rewards/rejected": -5.2316484451293945,
      "step": 2150
    },
    {
      "epoch": 1.5461254612546127,
      "grad_norm": 36.32992172241211,
      "learning_rate": 5.059108463080506e-07,
      "logits/chosen": -1.7441960573196411,
      "logits/rejected": -1.5256434679031372,
      "logps/chosen": -400.3003845214844,
      "logps/rejected": -368.2380065917969,
      "loss": 0.136,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -0.8741735816001892,
      "rewards/margins": 4.250377178192139,
      "rewards/rejected": -5.124550819396973,
      "step": 2200
    },
    {
      "epoch": 1.5461254612546127,
      "eval_logits/chosen": -2.029924154281616,
      "eval_logits/rejected": -2.0764176845550537,
      "eval_logps/chosen": -152.37820434570312,
      "eval_logps/rejected": -182.8094940185547,
      "eval_loss": 0.6600757837295532,
      "eval_rewards/accuracies": 0.7898550629615784,
      "eval_rewards/chosen": -2.6878771781921387,
      "eval_rewards/margins": 0.48295170068740845,
      "eval_rewards/rejected": -3.1708288192749023,
      "eval_runtime": 44.7936,
      "eval_samples_per_second": 15.404,
      "eval_steps_per_second": 7.702,
      "step": 2200
    },
    {
      "epoch": 1.5812686698295555,
      "grad_norm": 15.144909858703613,
      "learning_rate": 4.868448119158783e-07,
      "logits/chosen": -1.6984503269195557,
      "logits/rejected": -1.5325889587402344,
      "logps/chosen": -387.57586669921875,
      "logps/rejected": -374.7725830078125,
      "loss": 0.114,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -0.7923544049263,
      "rewards/margins": 4.450418949127197,
      "rewards/rejected": -5.242773532867432,
      "step": 2250
    },
    {
      "epoch": 1.6164118784044983,
      "grad_norm": 9.436891555786133,
      "learning_rate": 4.6779790689065506e-07,
      "logits/chosen": -1.8316223621368408,
      "logits/rejected": -1.594365119934082,
      "logps/chosen": -364.552978515625,
      "logps/rejected": -339.114013671875,
      "loss": 0.1108,
      "rewards/accuracies": 0.9674999713897705,
      "rewards/chosen": -0.9853540658950806,
      "rewards/margins": 4.360680103302002,
      "rewards/rejected": -5.346034049987793,
      "step": 2300
    },
    {
      "epoch": 1.651555086979441,
      "grad_norm": 6.352813243865967,
      "learning_rate": 4.487978279276421e-07,
      "logits/chosen": -1.6773854494094849,
      "logits/rejected": -1.5818564891815186,
      "logps/chosen": -388.2464599609375,
      "logps/rejected": -352.1225891113281,
      "loss": 0.1065,
      "rewards/accuracies": 0.9700000286102295,
      "rewards/chosen": -0.989799976348877,
      "rewards/margins": 4.535504341125488,
      "rewards/rejected": -5.525304794311523,
      "step": 2350
    },
    {
      "epoch": 1.686698295554384,
      "grad_norm": 21.942838668823242,
      "learning_rate": 4.2987220363087207e-07,
      "logits/chosen": -1.911537528038025,
      "logits/rejected": -1.6945948600769043,
      "logps/chosen": -389.4797668457031,
      "logps/rejected": -358.8553161621094,
      "loss": 0.1128,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -0.8928723335266113,
      "rewards/margins": 4.473836898803711,
      "rewards/rejected": -5.366708278656006,
      "step": 2400
    },
    {
      "epoch": 1.686698295554384,
      "eval_logits/chosen": -2.15362811088562,
      "eval_logits/rejected": -2.2070846557617188,
      "eval_logps/chosen": -154.57290649414062,
      "eval_logps/rejected": -185.60618591308594,
      "eval_loss": 0.6540037989616394,
      "eval_rewards/accuracies": 0.7898550629615784,
      "eval_rewards/chosen": -2.9073493480682373,
      "eval_rewards/margins": 0.5431488156318665,
      "eval_rewards/rejected": -3.450498104095459,
      "eval_runtime": 44.9193,
      "eval_samples_per_second": 15.361,
      "eval_steps_per_second": 7.68,
      "step": 2400
    }
  ],
  "logging_steps": 50,
  "max_steps": 4269,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 300,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}