trainer_state.json

{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9957264957264957,
  "eval_steps": 500,
  "global_step": 233,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.021367521367521368,
      "grad_norm": 1.5138773918151855,
      "learning_rate": 2.085563709106038e-06,
      "logits/chosen": -8.083470344543457,
      "logits/rejected": -8.251802444458008,
      "logps/chosen": -47.38351821899414,
      "logps/rejected": -84.17253112792969,
      "loss": 0.6923,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": 0.00038603864959441125,
      "rewards/margins": 0.001732213655486703,
      "rewards/rejected": -0.0013461749767884612,
      "step": 5
    },
    {
      "epoch": 0.042735042735042736,
      "grad_norm": 1.7986823320388794,
      "learning_rate": 4.692518345488585e-06,
      "logits/chosen": -8.110440254211426,
      "logits/rejected": -8.255338668823242,
      "logps/chosen": -46.66815185546875,
      "logps/rejected": -82.66679382324219,
      "loss": 0.6843,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.002308158902451396,
      "rewards/margins": 0.017809158191084862,
      "rewards/rejected": -0.020117320120334625,
      "step": 10
    },
    {
      "epoch": 0.0641025641025641,
      "grad_norm": 1.4440377950668335,
      "learning_rate": 7.299472981871133e-06,
      "logits/chosen": -8.137468338012695,
      "logits/rejected": -8.269254684448242,
      "logps/chosen": -46.37355422973633,
      "logps/rejected": -85.70413970947266,
      "loss": 0.6507,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.0037229328881949186,
      "rewards/margins": 0.08850056678056717,
      "rewards/rejected": -0.09222349524497986,
      "step": 15
    },
    {
      "epoch": 0.08547008547008547,
      "grad_norm": 1.5997639894485474,
      "learning_rate": 9.90642761825368e-06,
      "logits/chosen": -8.082623481750488,
      "logits/rejected": -8.237975120544434,
      "logps/chosen": -45.760292053222656,
      "logps/rejected": -83.62718200683594,
      "loss": 0.5913,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.010417555458843708,
      "rewards/margins": 0.22176532447338104,
      "rewards/rejected": -0.23218289017677307,
      "step": 20
    },
    {
      "epoch": 0.10683760683760683,
      "grad_norm": 1.4740291833877563,
      "learning_rate": 1.2513382254636227e-05,
      "logits/chosen": -8.118743896484375,
      "logits/rejected": -8.245091438293457,
      "logps/chosen": -45.908843994140625,
      "logps/rejected": -87.47322082519531,
      "loss": 0.4777,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -0.002890002680942416,
      "rewards/margins": 0.5190014839172363,
      "rewards/rejected": -0.5218915343284607,
      "step": 25
    },
    {
      "epoch": 0.1282051282051282,
      "grad_norm": 1.1871259212493896,
      "learning_rate": 1.5120336891018775e-05,
      "logits/chosen": -8.102651596069336,
      "logits/rejected": -8.204320907592773,
      "logps/chosen": -47.10442352294922,
      "logps/rejected": -91.00077819824219,
      "loss": 0.3566,
      "rewards/accuracies": 0.9916666746139526,
      "rewards/chosen": 0.011888022534549236,
      "rewards/margins": 0.9453158378601074,
      "rewards/rejected": -0.9334278106689453,
      "step": 30
    },
    {
      "epoch": 0.14957264957264957,
      "grad_norm": 1.3960121870040894,
      "learning_rate": 1.772729152740132e-05,
      "logits/chosen": -8.014056205749512,
      "logits/rejected": -8.112325668334961,
      "logps/chosen": -46.329681396484375,
      "logps/rejected": -98.96195983886719,
      "loss": 0.2151,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": 0.0061798011884093285,
      "rewards/margins": 1.721108078956604,
      "rewards/rejected": -1.7149282693862915,
      "step": 35
    },
    {
      "epoch": 0.17094017094017094,
      "grad_norm": 0.48299625515937805,
      "learning_rate": 1.824746798129707e-05,
      "logits/chosen": -7.782778263092041,
      "logits/rejected": -7.81784200668335,
      "logps/chosen": -47.68151092529297,
      "logps/rejected": -116.3075942993164,
      "loss": 0.0833,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -0.1560879796743393,
      "rewards/margins": 3.2540345191955566,
      "rewards/rejected": -3.4101223945617676,
      "step": 40
    },
    {
      "epoch": 0.19230769230769232,
      "grad_norm": 0.9764330387115479,
      "learning_rate": 1.8242534921967367e-05,
      "logits/chosen": -7.5903191566467285,
      "logits/rejected": -7.652113437652588,
      "logps/chosen": -48.73347854614258,
      "logps/rejected": -129.20016479492188,
      "loss": 0.0504,
      "rewards/accuracies": 0.9916666746139526,
      "rewards/chosen": -0.3716394007205963,
      "rewards/margins": 4.439803123474121,
      "rewards/rejected": -4.8114423751831055,
      "step": 45
    },
    {
      "epoch": 0.21367521367521367,
      "grad_norm": 0.8377407193183899,
      "learning_rate": 1.8233810105797565e-05,
      "logits/chosen": -7.339984893798828,
      "logits/rejected": -7.358351707458496,
      "logps/chosen": -53.820945739746094,
      "logps/rejected": -150.31130981445312,
      "loss": 0.0227,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -0.6324827671051025,
      "rewards/margins": 6.093559265136719,
      "rewards/rejected": -6.726041316986084,
      "step": 50
    },
    {
      "epoch": 0.23504273504273504,
      "grad_norm": 0.06649806350469589,
      "learning_rate": 1.822129837143867e-05,
      "logits/chosen": -7.078668117523193,
      "logits/rejected": -7.0982465744018555,
      "logps/chosen": -51.893882751464844,
      "logps/rejected": -167.0825958251953,
      "loss": 0.0123,
      "rewards/accuracies": 0.9958332777023315,
      "rewards/chosen": -0.5823885202407837,
      "rewards/margins": 7.8044114112854,
      "rewards/rejected": -8.386799812316895,
      "step": 55
    },
    {
      "epoch": 0.2564102564102564,
      "grad_norm": 0.26297417283058167,
      "learning_rate": 1.8205006657709207e-05,
      "logits/chosen": -6.831668853759766,
      "logits/rejected": -6.80808162689209,
      "logps/chosen": -51.43288040161133,
      "logps/rejected": -171.9197998046875,
      "loss": 0.0053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6035412549972534,
      "rewards/margins": 8.397494316101074,
      "rewards/rejected": -9.001035690307617,
      "step": 60
    },
    {
      "epoch": 0.2777777777777778,
      "grad_norm": 2.2538387775421143,
      "learning_rate": 1.8184943999747055e-05,
      "logits/chosen": -6.655425071716309,
      "logits/rejected": -6.645463466644287,
      "logps/chosen": -56.01396560668945,
      "logps/rejected": -174.80201721191406,
      "loss": 0.0094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9524188041687012,
      "rewards/margins": 8.350458145141602,
      "rewards/rejected": -9.302877426147461,
      "step": 65
    },
    {
      "epoch": 0.29914529914529914,
      "grad_norm": 0.24203453958034515,
      "learning_rate": 1.8161121523998697e-05,
      "logits/chosen": -6.6167120933532715,
      "logits/rejected": -6.533728122711182,
      "logps/chosen": -55.92729568481445,
      "logps/rejected": -180.21670532226562,
      "loss": 0.0069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0188896656036377,
      "rewards/margins": 8.844462394714355,
      "rewards/rejected": -9.863351821899414,
      "step": 70
    },
    {
      "epoch": 0.32051282051282054,
      "grad_norm": 0.07127346843481064,
      "learning_rate": 1.813355244204865e-05,
      "logits/chosen": -6.511422634124756,
      "logits/rejected": -6.383130073547363,
      "logps/chosen": -54.2319221496582,
      "logps/rejected": -183.2593536376953,
      "loss": 0.007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8627808690071106,
      "rewards/margins": 9.315163612365723,
      "rewards/rejected": -10.177945137023926,
      "step": 75
    },
    {
      "epoch": 0.3418803418803419,
      "grad_norm": 0.23009130358695984,
      "learning_rate": 1.810225204329254e-05,
      "logits/chosen": -6.263828277587891,
      "logits/rejected": -6.193525314331055,
      "logps/chosen": -51.82039260864258,
      "logps/rejected": -188.51785278320312,
      "loss": 0.0036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5680203437805176,
      "rewards/margins": 10.208035469055176,
      "rewards/rejected": -10.776056289672852,
      "step": 80
    },
    {
      "epoch": 0.36324786324786323,
      "grad_norm": 0.07301731407642365,
      "learning_rate": 1.8067237686457828e-05,
      "logits/chosen": -6.14719820022583,
      "logits/rejected": -5.998934745788574,
      "logps/chosen": -53.6826286315918,
      "logps/rejected": -189.89035034179688,
      "loss": 0.0085,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6072945594787598,
      "rewards/margins": 10.168685913085938,
      "rewards/rejected": -10.775979995727539,
      "step": 85
    },
    {
      "epoch": 0.38461538461538464,
      "grad_norm": 0.04785608872771263,
      "learning_rate": 1.8028528789976905e-05,
      "logits/chosen": -6.031782627105713,
      "logits/rejected": -5.81381368637085,
      "logps/chosen": -51.576385498046875,
      "logps/rejected": -195.06924438476562,
      "loss": 0.0071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5760863423347473,
      "rewards/margins": 10.668878555297852,
      "rewards/rejected": -11.244962692260742,
      "step": 90
    },
    {
      "epoch": 0.405982905982906,
      "grad_norm": 0.3428807258605957,
      "learning_rate": 1.7986146821217944e-05,
      "logits/chosen": -5.902842998504639,
      "logits/rejected": -5.718190670013428,
      "logps/chosen": -52.10051727294922,
      "logps/rejected": -200.8454132080078,
      "loss": 0.0064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6000415682792664,
      "rewards/margins": 10.976484298706055,
      "rewards/rejected": -11.576525688171387,
      "step": 95
    },
    {
      "epoch": 0.42735042735042733,
      "grad_norm": 0.23059943318367004,
      "learning_rate": 1.7940115284579423e-05,
      "logits/chosen": -5.684418678283691,
      "logits/rejected": -5.488518714904785,
      "logps/chosen": -54.800445556640625,
      "logps/rejected": -194.55435180664062,
      "loss": 0.0071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8389965295791626,
      "rewards/margins": 10.36409854888916,
      "rewards/rejected": -11.203096389770508,
      "step": 100
    },
    {
      "epoch": 0.44871794871794873,
      "grad_norm": 0.3416368067264557,
      "learning_rate": 1.7890459708454927e-05,
      "logits/chosen": -5.714560508728027,
      "logits/rejected": -5.499309539794922,
      "logps/chosen": -55.89985275268555,
      "logps/rejected": -212.3295440673828,
      "loss": 0.0034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9213576316833496,
      "rewards/margins": 11.838143348693848,
      "rewards/rejected": -12.759500503540039,
      "step": 105
    },
    {
      "epoch": 0.4700854700854701,
      "grad_norm": 0.13027356564998627,
      "learning_rate": 1.7837207631075485e-05,
      "logits/chosen": -5.563567161560059,
      "logits/rejected": -5.320518493652344,
      "logps/chosen": -57.118194580078125,
      "logps/rejected": -209.5955047607422,
      "loss": 0.0022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0772572755813599,
      "rewards/margins": 11.747246742248535,
      "rewards/rejected": -12.824502944946289,
      "step": 110
    },
    {
      "epoch": 0.49145299145299143,
      "grad_norm": 0.5294904112815857,
      "learning_rate": 1.7780388585237314e-05,
      "logits/chosen": -5.495570182800293,
      "logits/rejected": -5.353681564331055,
      "logps/chosen": -62.526214599609375,
      "logps/rejected": -215.9052734375,
      "loss": 0.006,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.5267938375473022,
      "rewards/margins": 11.732142448425293,
      "rewards/rejected": -13.258936882019043,
      "step": 115
    },
    {
      "epoch": 0.5128205128205128,
      "grad_norm": 0.015581418760120869,
      "learning_rate": 1.772003408192334e-05,
      "logits/chosen": -5.3723015785217285,
      "logits/rejected": -5.171557426452637,
      "logps/chosen": -63.449729919433594,
      "logps/rejected": -221.8417205810547,
      "loss": 0.0028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6488335132598877,
      "rewards/margins": 12.199827194213867,
      "rewards/rejected": -13.848661422729492,
      "step": 120
    },
    {
      "epoch": 0.5341880341880342,
      "grad_norm": 0.815427839756012,
      "learning_rate": 1.76561775928277e-05,
      "logits/chosen": -5.303497314453125,
      "logits/rejected": -5.043176651000977,
      "logps/chosen": -61.9741096496582,
      "logps/rejected": -221.53762817382812,
      "loss": 0.0034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5428324937820435,
      "rewards/margins": 12.496816635131836,
      "rewards/rejected": -14.039648056030273,
      "step": 125
    },
    {
      "epoch": 0.5555555555555556,
      "grad_norm": 0.005422074813395739,
      "learning_rate": 1.7588854531792862e-05,
      "logits/chosen": -5.377820014953613,
      "logits/rejected": -5.087519645690918,
      "logps/chosen": -63.00715255737305,
      "logps/rejected": -232.4966278076172,
      "loss": 0.0081,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.6413911581039429,
      "rewards/margins": 13.287958145141602,
      "rewards/rejected": -14.929348945617676,
      "step": 130
    },
    {
      "epoch": 0.5769230769230769,
      "grad_norm": 0.026918945834040642,
      "learning_rate": 1.7518102235169614e-05,
      "logits/chosen": -5.284510135650635,
      "logits/rejected": -4.935328483581543,
      "logps/chosen": -70.90217590332031,
      "logps/rejected": -237.47158813476562,
      "loss": 0.0031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3692615032196045,
      "rewards/margins": 13.011067390441895,
      "rewards/rejected": -15.380328178405762,
      "step": 135
    },
    {
      "epoch": 0.5982905982905983,
      "grad_norm": 0.10641518980264664,
      "learning_rate": 1.7443959941110907e-05,
      "logits/chosen": -5.173069953918457,
      "logits/rejected": -4.842183589935303,
      "logps/chosen": -76.1960220336914,
      "logps/rejected": -236.0799560546875,
      "loss": 0.0078,
      "rewards/accuracies": 0.9916666746139526,
      "rewards/chosen": -3.056429624557495,
      "rewards/margins": 12.243330001831055,
      "rewards/rejected": -15.299758911132812,
      "step": 140
    },
    {
      "epoch": 0.6196581196581197,
      "grad_norm": 0.0042958855628967285,
      "learning_rate": 1.736646876781097e-05,
      "logits/chosen": -5.071074962615967,
      "logits/rejected": -4.795034408569336,
      "logps/chosen": -75.36505126953125,
      "logps/rejected": -239.0994415283203,
      "loss": 0.0062,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.8842122554779053,
      "rewards/margins": 12.760270118713379,
      "rewards/rejected": -15.644482612609863,
      "step": 145
    },
    {
      "epoch": 0.6410256410256411,
      "grad_norm": 0.0037026714999228716,
      "learning_rate": 1.7285671690701772e-05,
      "logits/chosen": -5.1612749099731445,
      "logits/rejected": -4.891432285308838,
      "logps/chosen": -73.67475891113281,
      "logps/rejected": -243.776611328125,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.849099636077881,
      "rewards/margins": 13.369132995605469,
      "rewards/rejected": -16.218233108520508,
      "step": 150
    },
    {
      "epoch": 0.6623931623931624,
      "grad_norm": 1.1098724603652954,
      "learning_rate": 1.720161351861951e-05,
      "logits/chosen": -4.937078952789307,
      "logits/rejected": -4.7936506271362305,
      "logps/chosen": -81.33653259277344,
      "logps/rejected": -253.2895050048828,
      "loss": 0.0173,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -3.4761550426483154,
      "rewards/margins": 13.513101577758789,
      "rewards/rejected": -16.989253997802734,
      "step": 155
    },
    {
      "epoch": 0.6837606837606838,
      "grad_norm": 0.048093151301145554,
      "learning_rate": 1.7114340868954298e-05,
      "logits/chosen": -4.8120503425598145,
      "logits/rejected": -4.629734992980957,
      "logps/chosen": -66.40604400634766,
      "logps/rejected": -245.8269500732422,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.142770528793335,
      "rewards/margins": 14.375267028808594,
      "rewards/rejected": -16.51803970336914,
      "step": 160
    },
    {
      "epoch": 0.7051282051282052,
      "grad_norm": 0.22269146144390106,
      "learning_rate": 1.702390214179687e-05,
      "logits/chosen": -4.921370506286621,
      "logits/rejected": -4.65984582901001,
      "logps/chosen": -61.0145149230957,
      "logps/rejected": -255.3472442626953,
      "loss": 0.0025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5794414281845093,
      "rewards/margins": 15.753924369812012,
      "rewards/rejected": -17.3333683013916,
      "step": 165
    },
    {
      "epoch": 0.7264957264957265,
      "grad_norm": 0.005028157494962215,
      "learning_rate": 1.6930347493096617e-05,
      "logits/chosen": -4.911020278930664,
      "logits/rejected": -4.5893473625183105,
      "logps/chosen": -59.54716873168945,
      "logps/rejected": -254.0550994873047,
      "loss": 0.0108,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.2242438793182373,
      "rewards/margins": 15.8722562789917,
      "rewards/rejected": -17.096500396728516,
      "step": 170
    },
    {
      "epoch": 0.7478632478632479,
      "grad_norm": 0.002056506462395191,
      "learning_rate": 1.683372880684587e-05,
      "logits/chosen": -4.788331031799316,
      "logits/rejected": -4.532805442810059,
      "logps/chosen": -59.84217071533203,
      "logps/rejected": -256.4787292480469,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3549582958221436,
      "rewards/margins": 15.981206893920898,
      "rewards/rejected": -17.336164474487305,
      "step": 175
    },
    {
      "epoch": 0.7692307692307693,
      "grad_norm": 0.002407959895208478,
      "learning_rate": 1.6734099666305807e-05,
      "logits/chosen": -4.8239545822143555,
      "logits/rejected": -4.522343635559082,
      "logps/chosen": -60.510772705078125,
      "logps/rejected": -255.5074005126953,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.45245361328125,
      "rewards/margins": 15.896041870117188,
      "rewards/rejected": -17.348495483398438,
      "step": 180
    },
    {
      "epoch": 0.7905982905982906,
      "grad_norm": 0.03240586817264557,
      "learning_rate": 1.6631515324290002e-05,
      "logits/chosen": -4.732089519500732,
      "logits/rejected": -4.461452007293701,
      "logps/chosen": -62.0566291809082,
      "logps/rejected": -255.7787628173828,
      "loss": 0.0035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.531470537185669,
      "rewards/margins": 15.87073802947998,
      "rewards/rejected": -17.402210235595703,
      "step": 185
    },
    {
      "epoch": 0.811965811965812,
      "grad_norm": 0.5220297574996948,
      "learning_rate": 1.6526032672522037e-05,
      "logits/chosen": -4.666471004486084,
      "logits/rejected": -4.46518611907959,
      "logps/chosen": -60.214569091796875,
      "logps/rejected": -257.39501953125,
      "loss": 0.0049,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.40105402469635,
      "rewards/margins": 16.15066909790039,
      "rewards/rejected": -17.551721572875977,
      "step": 190
    },
    {
      "epoch": 0.8333333333333334,
      "grad_norm": 0.05552729591727257,
      "learning_rate": 1.6417710210084233e-05,
      "logits/chosen": -4.780084133148193,
      "logits/rejected": -4.495057582855225,
      "logps/chosen": -65.10442352294922,
      "logps/rejected": -255.4351043701172,
      "loss": 0.0036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6861228942871094,
      "rewards/margins": 15.603243827819824,
      "rewards/rejected": -17.289369583129883,
      "step": 195
    },
    {
      "epoch": 0.8547008547008547,
      "grad_norm": 0.0821588784456253,
      "learning_rate": 1.6306608010974917e-05,
      "logits/chosen": -4.595023155212402,
      "logits/rejected": -4.374794960021973,
      "logps/chosen": -57.596336364746094,
      "logps/rejected": -258.0520935058594,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3521565198898315,
      "rewards/margins": 16.3614501953125,
      "rewards/rejected": -17.713607788085938,
      "step": 200
    },
    {
      "epoch": 0.8760683760683761,
      "grad_norm": 1.1318793296813965,
      "learning_rate": 1.619278769079231e-05,
      "logits/chosen": -4.529976844787598,
      "logits/rejected": -4.339434623718262,
      "logps/chosen": -63.430381774902344,
      "logps/rejected": -265.7686462402344,
      "loss": 0.0079,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.6948257684707642,
      "rewards/margins": 16.656055450439453,
      "rewards/rejected": -18.350879669189453,
      "step": 205
    },
    {
      "epoch": 0.8974358974358975,
      "grad_norm": 0.1861945390701294,
      "learning_rate": 1.607631237256344e-05,
      "logits/chosen": -4.55942440032959,
      "logits/rejected": -4.32992696762085,
      "logps/chosen": -65.634033203125,
      "logps/rejected": -268.8421630859375,
      "loss": 0.0193,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.9223676919937134,
      "rewards/margins": 16.714584350585938,
      "rewards/rejected": -18.636951446533203,
      "step": 210
    },
    {
      "epoch": 0.9188034188034188,
      "grad_norm": 0.08688800781965256,
      "learning_rate": 1.595724665173709e-05,
      "logits/chosen": -4.457498073577881,
      "logits/rejected": -4.274403095245361,
      "logps/chosen": -67.5849838256836,
      "logps/rejected": -264.041259765625,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0491080284118652,
      "rewards/margins": 16.044025421142578,
      "rewards/rejected": -18.0931339263916,
      "step": 215
    },
    {
      "epoch": 0.9401709401709402,
      "grad_norm": 1.0533666610717773,
      "learning_rate": 1.5835656560360096e-05,
      "logits/chosen": -4.608490943908691,
      "logits/rejected": -4.405177116394043,
      "logps/chosen": -69.20387268066406,
      "logps/rejected": -255.1839599609375,
      "loss": 0.0037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.211890935897827,
      "rewards/margins": 15.11512565612793,
      "rewards/rejected": -17.327014923095703,
      "step": 220
    },
    {
      "epoch": 0.9615384615384616,
      "grad_norm": 0.12808182835578918,
      "learning_rate": 1.5711609530457018e-05,
      "logits/chosen": -4.415229797363281,
      "logits/rejected": -4.2678608894348145,
      "logps/chosen": -69.80118560791016,
      "logps/rejected": -263.7221374511719,
      "loss": 0.0083,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.4672863483428955,
      "rewards/margins": 15.749577522277832,
      "rewards/rejected": -18.21686363220215,
      "step": 225
    },
    {
      "epoch": 0.9829059829059829,
      "grad_norm": 5.840671539306641,
      "learning_rate": 1.5585174356633322e-05,
      "logits/chosen": -4.42140007019043,
      "logits/rejected": -4.332382678985596,
      "logps/chosen": -71.861572265625,
      "logps/rejected": -265.5335998535156,
      "loss": 0.006,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.7037765979766846,
      "rewards/margins": 15.881362915039062,
      "rewards/rejected": -18.585140228271484,
      "step": 230
    },
    {
      "epoch": 0.9957264957264957,
      "eval_logits/chosen": -4.4525227546691895,
      "eval_logits/rejected": -4.340153217315674,
      "eval_logps/chosen": -71.37000274658203,
      "eval_logps/rejected": -273.5752258300781,
      "eval_loss": 0.00013485502859111875,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -2.484865427017212,
      "eval_rewards/margins": 16.606918334960938,
      "eval_rewards/rejected": -19.091785430908203,
      "eval_runtime": 14.45,
      "eval_samples_per_second": 13.841,
      "eval_steps_per_second": 6.92,
      "step": 233
    }
  ],
  "logging_steps": 5,
  "max_steps": 702,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 12,
  "trial_name": null,
  "trial_params": null
}