trainer_state.json

{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.7248506220001959,
  "eval_steps": 200,
  "global_step": 1850,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.019590557351356647,
      "grad_norm": 152.70944213867188,
      "learning_rate": 3.2666666666666663e-07,
      "logits/chosen": -2.876490592956543,
      "logits/rejected": -2.730497121810913,
      "logps/chosen": -467.5262756347656,
      "logps/rejected": -378.40655517578125,
      "loss": 0.64,
      "rewards/accuracies": 0.6175000071525574,
      "rewards/chosen": 0.30685558915138245,
      "rewards/margins": 0.14467713236808777,
      "rewards/rejected": 0.16217847168445587,
      "step": 50
    },
    {
      "epoch": 0.03918111470271329,
      "grad_norm": 169.46041870117188,
      "learning_rate": 6.6e-07,
      "logits/chosen": -2.9327621459960938,
      "logits/rejected": -2.691812038421631,
      "logps/chosen": -459.0450439453125,
      "logps/rejected": -358.7035217285156,
      "loss": 0.5732,
      "rewards/accuracies": 0.7200000286102295,
      "rewards/chosen": 0.8860471844673157,
      "rewards/margins": 0.42572346329689026,
      "rewards/rejected": 0.4603237509727478,
      "step": 100
    },
    {
      "epoch": 0.05877167205406994,
      "grad_norm": 163.8626708984375,
      "learning_rate": 9.933333333333333e-07,
      "logits/chosen": -2.9648749828338623,
      "logits/rejected": -2.746995449066162,
      "logps/chosen": -450.51348876953125,
      "logps/rejected": -383.9017333984375,
      "loss": 0.5708,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.1387741565704346,
      "rewards/margins": 0.5126526355743408,
      "rewards/rejected": 0.626121461391449,
      "step": 150
    },
    {
      "epoch": 0.07836222940542659,
      "grad_norm": 101.9367446899414,
      "learning_rate": 9.99894936347371e-07,
      "logits/chosen": -3.0323171615600586,
      "logits/rejected": -2.8769636154174805,
      "logps/chosen": -467.7211608886719,
      "logps/rejected": -375.92816162109375,
      "loss": 0.5266,
      "rewards/accuracies": 0.7174999713897705,
      "rewards/chosen": 1.2015265226364136,
      "rewards/margins": 0.7129627466201782,
      "rewards/rejected": 0.48856377601623535,
      "step": 200
    },
    {
      "epoch": 0.07836222940542659,
      "eval_logits/chosen": -2.2541391849517822,
      "eval_logits/rejected": -2.285670042037964,
      "eval_logps/chosen": -172.0572509765625,
      "eval_logps/rejected": -200.56900024414062,
      "eval_loss": 0.6408354640007019,
      "eval_rewards/accuracies": 0.7420290112495422,
      "eval_rewards/chosen": 0.0233369879424572,
      "eval_rewards/margins": 0.1684449017047882,
      "eval_rewards/rejected": -0.1451079100370407,
      "eval_runtime": 45.4466,
      "eval_samples_per_second": 15.183,
      "eval_steps_per_second": 7.591,
      "step": 200
    },
    {
      "epoch": 0.09795278675678323,
      "grad_norm": 106.43968200683594,
      "learning_rate": 9.995711712979657e-07,
      "logits/chosen": -2.953029155731201,
      "logits/rejected": -2.7987329959869385,
      "logps/chosen": -462.841796875,
      "logps/rejected": -378.1797790527344,
      "loss": 0.4989,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.3110496997833252,
      "rewards/margins": 0.854219377040863,
      "rewards/rejected": 0.4568302035331726,
      "step": 250
    },
    {
      "epoch": 0.11754334410813988,
      "grad_norm": 105.10195922851562,
      "learning_rate": 9.990288027658056e-07,
      "logits/chosen": -2.9163262844085693,
      "logits/rejected": -2.7455270290374756,
      "logps/chosen": -438.2953186035156,
      "logps/rejected": -367.7381591796875,
      "loss": 0.5314,
      "rewards/accuracies": 0.7400000095367432,
      "rewards/chosen": 1.436720848083496,
      "rewards/margins": 0.8745988607406616,
      "rewards/rejected": 0.5621219277381897,
      "step": 300
    },
    {
      "epoch": 0.13713390145949653,
      "grad_norm": 73.22066497802734,
      "learning_rate": 9.982680680817391e-07,
      "logits/chosen": -2.966728448867798,
      "logits/rejected": -2.765995740890503,
      "logps/chosen": -447.7353210449219,
      "logps/rejected": -349.2236633300781,
      "loss": 0.4596,
      "rewards/accuracies": 0.7900000214576721,
      "rewards/chosen": 1.517472267150879,
      "rewards/margins": 1.1284364461898804,
      "rewards/rejected": 0.3890356719493866,
      "step": 350
    },
    {
      "epoch": 0.15672445881085317,
      "grad_norm": 193.4586181640625,
      "learning_rate": 9.972893001297698e-07,
      "logits/chosen": -2.92069935798645,
      "logits/rejected": -2.7497365474700928,
      "logps/chosen": -460.17755126953125,
      "logps/rejected": -371.06011962890625,
      "loss": 0.51,
      "rewards/accuracies": 0.7350000143051147,
      "rewards/chosen": 1.5156035423278809,
      "rewards/margins": 1.070378065109253,
      "rewards/rejected": 0.44522520899772644,
      "step": 400
    },
    {
      "epoch": 0.15672445881085317,
      "eval_logits/chosen": -2.2648189067840576,
      "eval_logits/rejected": -2.2913312911987305,
      "eval_logps/chosen": -172.1621856689453,
      "eval_logps/rejected": -201.6964874267578,
      "eval_loss": 0.6150330305099487,
      "eval_rewards/accuracies": 0.7710145115852356,
      "eval_rewards/chosen": 0.012844223529100418,
      "eval_rewards/margins": 0.2707003951072693,
      "eval_rewards/rejected": -0.25785619020462036,
      "eval_runtime": 45.871,
      "eval_samples_per_second": 15.042,
      "eval_steps_per_second": 7.521,
      "step": 400
    },
    {
      "epoch": 0.17631501616220982,
      "grad_norm": 142.24362182617188,
      "learning_rate": 9.96092927201391e-07,
      "logits/chosen": -2.922132968902588,
      "logits/rejected": -2.754628896713257,
      "logps/chosen": -446.7215270996094,
      "logps/rejected": -385.2563171386719,
      "loss": 0.4712,
      "rewards/accuracies": 0.7574999928474426,
      "rewards/chosen": 1.4054139852523804,
      "rewards/margins": 1.1510262489318848,
      "rewards/rejected": 0.25438761711120605,
      "step": 450
    },
    {
      "epoch": 0.19590557351356647,
      "grad_norm": 99.2269515991211,
      "learning_rate": 9.946794728081737e-07,
      "logits/chosen": -3.007890224456787,
      "logits/rejected": -2.8413093090057373,
      "logps/chosen": -465.13348388671875,
      "logps/rejected": -390.49591064453125,
      "loss": 0.4768,
      "rewards/accuracies": 0.7799999713897705,
      "rewards/chosen": 1.604026198387146,
      "rewards/margins": 1.1249028444290161,
      "rewards/rejected": 0.47912338376045227,
      "step": 500
    },
    {
      "epoch": 0.2154961308649231,
      "grad_norm": 169.93678283691406,
      "learning_rate": 9.930495554526878e-07,
      "logits/chosen": -2.952453851699829,
      "logits/rejected": -2.812680959701538,
      "logps/chosen": -449.0242919921875,
      "logps/rejected": -357.1375732421875,
      "loss": 0.4962,
      "rewards/accuracies": 0.7724999785423279,
      "rewards/chosen": 1.6039185523986816,
      "rewards/margins": 1.0116550922393799,
      "rewards/rejected": 0.5922635793685913,
      "step": 550
    },
    {
      "epoch": 0.23508668821627976,
      "grad_norm": 272.4690856933594,
      "learning_rate": 9.912038883578552e-07,
      "logits/chosen": -2.9756484031677246,
      "logits/rejected": -2.760322332382202,
      "logps/chosen": -460.68865966796875,
      "logps/rejected": -377.5042724609375,
      "loss": 0.5151,
      "rewards/accuracies": 0.7799999713897705,
      "rewards/chosen": 1.6494457721710205,
      "rewards/margins": 1.0801762342453003,
      "rewards/rejected": 0.5692696571350098,
      "step": 600
    },
    {
      "epoch": 0.23508668821627976,
      "eval_logits/chosen": -2.328936815261841,
      "eval_logits/rejected": -2.3488407135009766,
      "eval_logps/chosen": -173.3373565673828,
      "eval_logps/rejected": -202.97079467773438,
      "eval_loss": 0.6263740062713623,
      "eval_rewards/accuracies": 0.7507246136665344,
      "eval_rewards/chosen": -0.10467354953289032,
      "eval_rewards/margins": 0.2806117534637451,
      "eval_rewards/rejected": -0.38528531789779663,
      "eval_runtime": 45.7131,
      "eval_samples_per_second": 15.094,
      "eval_steps_per_second": 7.547,
      "step": 600
    },
    {
      "epoch": 0.2546772455676364,
      "grad_norm": 72.5123291015625,
      "learning_rate": 9.891432791548562e-07,
      "logits/chosen": -2.9478814601898193,
      "logits/rejected": -2.767559766769409,
      "logps/chosen": -429.18499755859375,
      "logps/rejected": -369.6413269042969,
      "loss": 0.4703,
      "rewards/accuracies": 0.7649999856948853,
      "rewards/chosen": 1.570642113685608,
      "rewards/margins": 1.231786847114563,
      "rewards/rejected": 0.3388552963733673,
      "step": 650
    },
    {
      "epoch": 0.27426780291899305,
      "grad_norm": 146.4647979736328,
      "learning_rate": 9.868686295297258e-07,
      "logits/chosen": -2.9772324562072754,
      "logits/rejected": -2.7840633392333984,
      "logps/chosen": -452.4783630371094,
      "logps/rejected": -379.3604736328125,
      "loss": 0.4646,
      "rewards/accuracies": 0.7674999833106995,
      "rewards/chosen": 1.685501217842102,
      "rewards/margins": 1.2647416591644287,
      "rewards/rejected": 0.4207596480846405,
      "step": 700
    },
    {
      "epoch": 0.29385836027034967,
      "grad_norm": 115.44526672363281,
      "learning_rate": 9.8438093482879e-07,
      "logits/chosen": -3.0253145694732666,
      "logits/rejected": -2.8723714351654053,
      "logps/chosen": -461.543212890625,
      "logps/rejected": -378.514892578125,
      "loss": 0.5084,
      "rewards/accuracies": 0.7549999952316284,
      "rewards/chosen": 1.5600999593734741,
      "rewards/margins": 1.2082923650741577,
      "rewards/rejected": 0.3518076241016388,
      "step": 750
    },
    {
      "epoch": 0.31344891762170635,
      "grad_norm": 49.29530334472656,
      "learning_rate": 9.81681283623121e-07,
      "logits/chosen": -3.0215282440185547,
      "logits/rejected": -2.831230401992798,
      "logps/chosen": -469.9809875488281,
      "logps/rejected": -369.4506530761719,
      "loss": 0.4662,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 1.9505739212036133,
      "rewards/margins": 1.3873133659362793,
      "rewards/rejected": 0.5632606744766235,
      "step": 800
    },
    {
      "epoch": 0.31344891762170635,
      "eval_logits/chosen": -2.373304843902588,
      "eval_logits/rejected": -2.391378402709961,
      "eval_logps/chosen": -174.7285614013672,
      "eval_logps/rejected": -204.59425354003906,
      "eval_loss": 0.6305972933769226,
      "eval_rewards/accuracies": 0.7681159377098083,
      "eval_rewards/chosen": -0.2437940239906311,
      "eval_rewards/margins": 0.30383673310279846,
      "eval_rewards/rejected": -0.547630786895752,
      "eval_runtime": 45.7764,
      "eval_samples_per_second": 15.073,
      "eval_steps_per_second": 7.537,
      "step": 800
    },
    {
      "epoch": 0.33303947497306297,
      "grad_norm": 144.90708923339844,
      "learning_rate": 9.787708572321983e-07,
      "logits/chosen": -2.910259485244751,
      "logits/rejected": -2.762113332748413,
      "logps/chosen": -452.485107421875,
      "logps/rejected": -337.2020568847656,
      "loss": 0.5209,
      "rewards/accuracies": 0.7400000095367432,
      "rewards/chosen": 2.068610906600952,
      "rewards/margins": 1.3179844617843628,
      "rewards/rejected": 0.7506266236305237,
      "step": 850
    },
    {
      "epoch": 0.35263003232441964,
      "grad_norm": 130.08224487304688,
      "learning_rate": 9.756509292069825e-07,
      "logits/chosen": -2.9319863319396973,
      "logits/rejected": -2.783926010131836,
      "logps/chosen": -419.07110595703125,
      "logps/rejected": -382.4843444824219,
      "loss": 0.5003,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.7963037490844727,
      "rewards/margins": 1.2497668266296387,
      "rewards/rejected": 0.5465368032455444,
      "step": 900
    },
    {
      "epoch": 0.37222058967577626,
      "grad_norm": 151.3379669189453,
      "learning_rate": 9.72322864772634e-07,
      "logits/chosen": -3.0067360401153564,
      "logits/rejected": -2.8380017280578613,
      "logps/chosen": -463.267822265625,
      "logps/rejected": -379.6971130371094,
      "loss": 0.4351,
      "rewards/accuracies": 0.8050000071525574,
      "rewards/chosen": 1.8822674751281738,
      "rewards/margins": 1.517218828201294,
      "rewards/rejected": 0.36504870653152466,
      "step": 950
    },
    {
      "epoch": 0.39181114702713293,
      "grad_norm": 127.39634704589844,
      "learning_rate": 9.687881202311132e-07,
      "logits/chosen": -2.975214958190918,
      "logits/rejected": -2.7904016971588135,
      "logps/chosen": -459.6684875488281,
      "logps/rejected": -384.0487060546875,
      "loss": 0.5026,
      "rewards/accuracies": 0.7574999928474426,
      "rewards/chosen": 2.054124355316162,
      "rewards/margins": 1.4798840284347534,
      "rewards/rejected": 0.5742404460906982,
      "step": 1000
    },
    {
      "epoch": 0.39181114702713293,
      "eval_logits/chosen": -2.285802125930786,
      "eval_logits/rejected": -2.3008267879486084,
      "eval_logps/chosen": -173.751953125,
      "eval_logps/rejected": -203.65634155273438,
      "eval_loss": 0.6286672353744507,
      "eval_rewards/accuracies": 0.7579709887504578,
      "eval_rewards/chosen": -0.14613337814807892,
      "eval_rewards/margins": 0.3077085614204407,
      "eval_rewards/rejected": -0.453841894865036,
      "eval_runtime": 45.6208,
      "eval_samples_per_second": 15.125,
      "eval_steps_per_second": 7.562,
      "step": 1000
    },
    {
      "epoch": 0.41140170437848955,
      "grad_norm": 113.51675415039062,
      "learning_rate": 9.65048242323929e-07,
      "logits/chosen": -2.9107511043548584,
      "logits/rejected": -2.766993522644043,
      "logps/chosen": -419.68035888671875,
      "logps/rejected": -354.5930480957031,
      "loss": 0.4776,
      "rewards/accuracies": 0.7825000286102295,
      "rewards/chosen": 1.6637612581253052,
      "rewards/margins": 1.2595511674880981,
      "rewards/rejected": 0.40421026945114136,
      "step": 1050
    },
    {
      "epoch": 0.4309922617298462,
      "grad_norm": 142.44851684570312,
      "learning_rate": 9.611048675553127e-07,
      "logits/chosen": -2.945481061935425,
      "logits/rejected": -2.7998950481414795,
      "logps/chosen": -473.3480529785156,
      "logps/rejected": -378.8951416015625,
      "loss": 0.4899,
      "rewards/accuracies": 0.7799999713897705,
      "rewards/chosen": 1.8652592897415161,
      "rewards/margins": 1.3835867643356323,
      "rewards/rejected": 0.48167240619659424,
      "step": 1100
    },
    {
      "epoch": 0.45058281908120285,
      "grad_norm": 110.11934661865234,
      "learning_rate": 9.569597214761124e-07,
      "logits/chosen": -2.9105591773986816,
      "logits/rejected": -2.7662746906280518,
      "logps/chosen": -445.8592834472656,
      "logps/rejected": -364.0835266113281,
      "loss": 0.4642,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 1.9219467639923096,
      "rewards/margins": 1.473950982093811,
      "rewards/rejected": 0.4479956328868866,
      "step": 1150
    },
    {
      "epoch": 0.4701733764325595,
      "grad_norm": 180.00526428222656,
      "learning_rate": 9.526146179287222e-07,
      "logits/chosen": -2.8630211353302,
      "logits/rejected": -2.6804027557373047,
      "logps/chosen": -449.76812744140625,
      "logps/rejected": -364.3941650390625,
      "loss": 0.5097,
      "rewards/accuracies": 0.7325000166893005,
      "rewards/chosen": 1.7920465469360352,
      "rewards/margins": 1.2713873386383057,
      "rewards/rejected": 0.5206592082977295,
      "step": 1200
    },
    {
      "epoch": 0.4701733764325595,
      "eval_logits/chosen": -2.2779555320739746,
      "eval_logits/rejected": -2.2900030612945557,
      "eval_logps/chosen": -173.5331573486328,
      "eval_logps/rejected": -203.9847869873047,
      "eval_loss": 0.6131945848464966,
      "eval_rewards/accuracies": 0.7449275255203247,
      "eval_rewards/chosen": -0.12425270676612854,
      "eval_rewards/margins": 0.36243319511413574,
      "eval_rewards/rejected": -0.4866859018802643,
      "eval_runtime": 45.8929,
      "eval_samples_per_second": 15.035,
      "eval_steps_per_second": 7.517,
      "step": 1200
    },
    {
      "epoch": 0.48976393378391614,
      "grad_norm": 160.4936065673828,
      "learning_rate": 9.480714582533773e-07,
      "logits/chosen": -2.8644652366638184,
      "logits/rejected": -2.7200751304626465,
      "logps/chosen": -454.3363952636719,
      "logps/rejected": -376.8122253417969,
      "loss": 0.4958,
      "rewards/accuracies": 0.7524999976158142,
      "rewards/chosen": 2.13558292388916,
      "rewards/margins": 1.546592354774475,
      "rewards/rejected": 0.5889905691146851,
      "step": 1250
    },
    {
      "epoch": 0.5093544911352728,
      "grad_norm": 108.44024658203125,
      "learning_rate": 9.433322304561614e-07,
      "logits/chosen": -2.942742347717285,
      "logits/rejected": -2.7668235301971436,
      "logps/chosen": -450.0126647949219,
      "logps/rejected": -390.0963134765625,
      "loss": 0.4247,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.9560456275939941,
      "rewards/margins": 1.6855427026748657,
      "rewards/rejected": 0.27050289511680603,
      "step": 1300
    },
    {
      "epoch": 0.5289450484866295,
      "grad_norm": 280.08380126953125,
      "learning_rate": 9.383990083390903e-07,
      "logits/chosen": -2.957747220993042,
      "logits/rejected": -2.795564889907837,
      "logps/chosen": -453.2030334472656,
      "logps/rejected": -413.6679992675781,
      "loss": 0.4633,
      "rewards/accuracies": 0.7674999833106995,
      "rewards/chosen": 1.6891446113586426,
      "rewards/margins": 1.4608780145645142,
      "rewards/rejected": 0.22826646268367767,
      "step": 1350
    },
    {
      "epoch": 0.5485356058379861,
      "grad_norm": 214.8365478515625,
      "learning_rate": 9.332739505926528e-07,
      "logits/chosen": -2.917468309402466,
      "logits/rejected": -2.753814697265625,
      "logps/chosen": -471.89483642578125,
      "logps/rejected": -402.4897155761719,
      "loss": 0.4122,
      "rewards/accuracies": 0.8025000095367432,
      "rewards/chosen": 1.7866332530975342,
      "rewards/margins": 1.777306318283081,
      "rewards/rejected": 0.009327063336968422,
      "step": 1400
    },
    {
      "epoch": 0.5485356058379861,
      "eval_logits/chosen": -2.384613275527954,
      "eval_logits/rejected": -2.4019815921783447,
      "eval_logps/chosen": -176.3463592529297,
      "eval_logps/rejected": -207.75599670410156,
      "eval_loss": 0.5938982367515564,
      "eval_rewards/accuracies": 0.7782608866691589,
      "eval_rewards/chosen": -0.40557393431663513,
      "eval_rewards/margins": 0.4582330286502838,
      "eval_rewards/rejected": -0.863806962966919,
      "eval_runtime": 45.9899,
      "eval_samples_per_second": 15.003,
      "eval_steps_per_second": 7.502,
      "step": 1400
    },
    {
      "epoch": 0.5681261631893427,
      "grad_norm": 143.27638244628906,
      "learning_rate": 9.279592998512064e-07,
      "logits/chosen": -2.954049587249756,
      "logits/rejected": -2.828245162963867,
      "logps/chosen": -467.3232116699219,
      "logps/rejected": -384.35601806640625,
      "loss": 0.4616,
      "rewards/accuracies": 0.7900000214576721,
      "rewards/chosen": 1.6394932270050049,
      "rewards/margins": 1.5439642667770386,
      "rewards/rejected": 0.09552911669015884,
      "step": 1450
    },
    {
      "epoch": 0.5877167205406993,
      "grad_norm": 70.37281799316406,
      "learning_rate": 9.224573817116396e-07,
      "logits/chosen": -2.9961767196655273,
      "logits/rejected": -2.8868143558502197,
      "logps/chosen": -478.9124755859375,
      "logps/rejected": -402.163330078125,
      "loss": 0.3946,
      "rewards/accuracies": 0.8174999952316284,
      "rewards/chosen": 2.2281653881073,
      "rewards/margins": 2.0543277263641357,
      "rewards/rejected": 0.17383766174316406,
      "step": 1500
    },
    {
      "epoch": 0.6073072778920561,
      "grad_norm": 116.5025863647461,
      "learning_rate": 9.16770603715733e-07,
      "logits/chosen": -2.935271978378296,
      "logits/rejected": -2.8107409477233887,
      "logps/chosen": -448.86444091796875,
      "logps/rejected": -377.48944091796875,
      "loss": 0.5143,
      "rewards/accuracies": 0.7599999904632568,
      "rewards/chosen": 1.8645941019058228,
      "rewards/margins": 1.5373382568359375,
      "rewards/rejected": 0.3272559642791748,
      "step": 1550
    },
    {
      "epoch": 0.6268978352434127,
      "grad_norm": 115.91889190673828,
      "learning_rate": 9.109014542966609e-07,
      "logits/chosen": -3.030916452407837,
      "logits/rejected": -2.8699285984039307,
      "logps/chosen": -430.56219482421875,
      "logps/rejected": -374.2171936035156,
      "loss": 0.5026,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.8679478168487549,
      "rewards/margins": 1.4932770729064941,
      "rewards/rejected": 0.3746708631515503,
      "step": 1600
    },
    {
      "epoch": 0.6268978352434127,
      "eval_logits/chosen": -2.468580484390259,
      "eval_logits/rejected": -2.4822895526885986,
      "eval_logps/chosen": -177.13941955566406,
      "eval_logps/rejected": -209.1125030517578,
      "eval_loss": 0.590223491191864,
      "eval_rewards/accuracies": 0.7927536368370056,
      "eval_rewards/chosen": -0.484881192445755,
      "eval_rewards/margins": 0.514574408531189,
      "eval_rewards/rejected": -0.9994555711746216,
      "eval_runtime": 45.9912,
      "eval_samples_per_second": 15.003,
      "eval_steps_per_second": 7.501,
      "step": 1600
    },
    {
      "epoch": 0.6464883925947693,
      "grad_norm": 144.60536193847656,
      "learning_rate": 9.04852501690097e-07,
      "logits/chosen": -3.0554418563842773,
      "logits/rejected": -2.916806221008301,
      "logps/chosen": -448.33453369140625,
      "logps/rejected": -382.4766540527344,
      "loss": 0.4249,
      "rewards/accuracies": 0.7950000166893005,
      "rewards/chosen": 1.6690967082977295,
      "rewards/margins": 1.7366982698440552,
      "rewards/rejected": -0.06760149449110031,
      "step": 1650
    },
    {
      "epoch": 0.6660789499461259,
      "grad_norm": 204.64212036132812,
      "learning_rate": 8.986263928104007e-07,
      "logits/chosen": -2.9387013912200928,
      "logits/rejected": -2.8714663982391357,
      "logps/chosen": -462.238525390625,
      "logps/rejected": -362.3359069824219,
      "loss": 0.4699,
      "rewards/accuracies": 0.7724999785423279,
      "rewards/chosen": 2.0341713428497314,
      "rewards/margins": 1.6428416967391968,
      "rewards/rejected": 0.39132973551750183,
      "step": 1700
    },
    {
      "epoch": 0.6856695072974827,
      "grad_norm": 158.81417846679688,
      "learning_rate": 8.922258520923739e-07,
      "logits/chosen": -2.928734064102173,
      "logits/rejected": -2.7294297218322754,
      "logps/chosen": -432.1619873046875,
      "logps/rejected": -378.280517578125,
      "loss": 0.5039,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": 1.977944016456604,
      "rewards/margins": 1.4837253093719482,
      "rewards/rejected": 0.4942188262939453,
      "step": 1750
    },
    {
      "epoch": 0.7052600646488393,
      "grad_norm": 128.84994506835938,
      "learning_rate": 8.856536802990969e-07,
      "logits/chosen": -2.9586968421936035,
      "logits/rejected": -2.868234634399414,
      "logps/chosen": -432.3280334472656,
      "logps/rejected": -376.2671203613281,
      "loss": 0.461,
      "rewards/accuracies": 0.7799999713897705,
      "rewards/chosen": 1.9399315118789673,
      "rewards/margins": 1.5768976211547852,
      "rewards/rejected": 0.36303389072418213,
      "step": 1800
    },
    {
      "epoch": 0.7052600646488393,
      "eval_logits/chosen": -2.4123268127441406,
      "eval_logits/rejected": -2.4266409873962402,
      "eval_logps/chosen": -174.25506591796875,
      "eval_logps/rejected": -205.96014404296875,
      "eval_loss": 0.5846713185310364,
      "eval_rewards/accuracies": 0.8202898502349854,
      "eval_rewards/chosen": -0.19644570350646973,
      "eval_rewards/margins": 0.4877746105194092,
      "eval_rewards/rejected": -0.6842203140258789,
      "eval_runtime": 45.7164,
      "eval_samples_per_second": 15.093,
      "eval_steps_per_second": 7.547,
      "step": 1800
    },
    {
      "epoch": 0.7248506220001959,
      "grad_norm": 227.02520751953125,
      "learning_rate": 8.789127532963639e-07,
      "logits/chosen": -2.9502925872802734,
      "logits/rejected": -2.8222861289978027,
      "logps/chosen": -459.9781188964844,
      "logps/rejected": -375.967041015625,
      "loss": 0.3961,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 2.2383108139038086,
      "rewards/margins": 1.8207529783248901,
      "rewards/rejected": 0.4175575375556946,
      "step": 1850
    }
  ],
  "logging_steps": 50,
  "max_steps": 7659,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 370,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}