{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.995910949568378,
  "eval_steps": 400,
  "global_step": 137,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.03634711494775102,
      "grad_norm": 242.63721380836233,
      "learning_rate": 2.857142857142857e-07,
      "logits/chosen": -9.921992301940918,
      "logits/rejected": -10.590700149536133,
      "logps/chosen": -1.3840388059616089,
      "logps/rejected": -0.9648998379707336,
      "loss": 9.0059,
      "rewards/accuracies": 0.18125000596046448,
      "rewards/chosen": -13.840387344360352,
      "rewards/margins": -4.191389560699463,
      "rewards/rejected": -9.648998260498047,
      "step": 5
    },
    {
      "epoch": 0.07269422989550205,
      "grad_norm": 125.7278409072736,
      "learning_rate": 5.714285714285714e-07,
      "logits/chosen": -9.401021957397461,
      "logits/rejected": -10.126498222351074,
      "logps/chosen": -1.224568486213684,
      "logps/rejected": -0.8530822992324829,
      "loss": 8.149,
      "rewards/accuracies": 0.19374999403953552,
      "rewards/chosen": -12.245683670043945,
      "rewards/margins": -3.714862108230591,
      "rewards/rejected": -8.530821800231934,
      "step": 10
    },
    {
      "epoch": 0.10904134484325306,
      "grad_norm": 118.11240526235882,
      "learning_rate": 7.998695344323425e-07,
      "logits/chosen": -8.648290634155273,
      "logits/rejected": -9.154404640197754,
      "logps/chosen": -1.1302968263626099,
      "logps/rejected": -0.8277978897094727,
      "loss": 8.183,
      "rewards/accuracies": 0.16875000298023224,
      "rewards/chosen": -11.30296802520752,
      "rewards/margins": -3.024988889694214,
      "rewards/rejected": -8.277978897094727,
      "step": 15
    },
    {
      "epoch": 0.1453884597910041,
      "grad_norm": 76.19191775941484,
      "learning_rate": 7.953121695121394e-07,
      "logits/chosen": -7.632864475250244,
      "logits/rejected": -8.276620864868164,
      "logps/chosen": -0.7727211713790894,
      "logps/rejected": -0.6470556259155273,
      "loss": 6.3176,
      "rewards/accuracies": 0.21875,
      "rewards/chosen": -7.727211952209473,
      "rewards/margins": -1.2566553354263306,
      "rewards/rejected": -6.470555782318115,
      "step": 20
    },
    {
      "epoch": 0.18173557473875512,
      "grad_norm": 72.77010466261328,
      "learning_rate": 7.843163833184991e-07,
      "logits/chosen": -6.38211727142334,
      "logits/rejected": -6.964330196380615,
      "logps/chosen": -0.7217488288879395,
      "logps/rejected": -0.6721780896186829,
      "loss": 5.6946,
      "rewards/accuracies": 0.2750000059604645,
      "rewards/chosen": -7.217488765716553,
      "rewards/margins": -0.49570804834365845,
      "rewards/rejected": -6.721780300140381,
      "step": 25
    },
    {
      "epoch": 0.21808268968650613,
      "grad_norm": 50.91204324013054,
      "learning_rate": 7.670612634414511e-07,
      "logits/chosen": -5.789036273956299,
      "logits/rejected": -6.50485897064209,
      "logps/chosen": -0.6984373331069946,
      "logps/rejected": -0.6544702649116516,
      "loss": 5.5012,
      "rewards/accuracies": 0.3062500059604645,
      "rewards/chosen": -6.984373569488525,
      "rewards/margins": -0.43967121839523315,
      "rewards/rejected": -6.544702053070068,
      "step": 30
    },
    {
      "epoch": 0.25442980463425713,
      "grad_norm": 68.27173982618704,
      "learning_rate": 7.438278427948805e-07,
      "logits/chosen": -6.224788665771484,
      "logits/rejected": -6.794752597808838,
      "logps/chosen": -0.6720889806747437,
      "logps/rejected": -0.6537488698959351,
      "loss": 5.2665,
      "rewards/accuracies": 0.3062500059604645,
      "rewards/chosen": -6.720890045166016,
      "rewards/margins": -0.18340133130550385,
      "rewards/rejected": -6.537489414215088,
      "step": 35
    },
    {
      "epoch": 0.2907769195820082,
      "grad_norm": 52.462578532583635,
      "learning_rate": 7.149945224533862e-07,
      "logits/chosen": -5.9340128898620605,
      "logits/rejected": -6.362427234649658,
      "logps/chosen": -0.6735531091690063,
      "logps/rejected": -0.712183952331543,
      "loss": 5.0443,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -6.735530853271484,
      "rewards/margins": 0.3863077461719513,
      "rewards/rejected": -7.1218390464782715,
      "step": 40
    },
    {
      "epoch": 0.3271240345297592,
      "grad_norm": 49.32067133417458,
      "learning_rate": 6.810309086608129e-07,
      "logits/chosen": -6.20428466796875,
      "logits/rejected": -6.80216121673584,
      "logps/chosen": -0.6294387578964233,
      "logps/rejected": -0.7196067571640015,
      "loss": 5.0107,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -6.294387340545654,
      "rewards/margins": 0.9016799926757812,
      "rewards/rejected": -7.196066856384277,
      "step": 45
    },
    {
      "epoch": 0.36347114947751025,
      "grad_norm": 58.79656712268972,
      "learning_rate": 6.424901643866552e-07,
      "logits/chosen": -5.74347448348999,
      "logits/rejected": -6.245406627655029,
      "logps/chosen": -0.6845372319221497,
      "logps/rejected": -0.7552808523178101,
      "loss": 5.1229,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -6.845372200012207,
      "rewards/margins": 0.7074359655380249,
      "rewards/rejected": -7.5528082847595215,
      "step": 50
    },
    {
      "epoch": 0.39981826442526125,
      "grad_norm": 50.23542787679931,
      "learning_rate": 6e-07,
      "logits/chosen": -5.800081729888916,
      "logits/rejected": -6.2036285400390625,
      "logps/chosen": -0.5938673615455627,
      "logps/rejected": -0.7371848821640015,
      "loss": 4.9249,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -5.938673973083496,
      "rewards/margins": 1.4331753253936768,
      "rewards/rejected": -7.371849060058594,
      "step": 55
    },
    {
      "epoch": 0.43616537937301225,
      "grad_norm": 48.19402877634858,
      "learning_rate": 5.542524497952543e-07,
      "logits/chosen": -5.9060564041137695,
      "logits/rejected": -6.406619071960449,
      "logps/chosen": -0.5991919040679932,
      "logps/rejected": -0.7929545640945435,
      "loss": 4.6185,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -5.99191951751709,
      "rewards/margins": 1.9376264810562134,
      "rewards/rejected": -7.9295454025268555,
      "step": 60
    },
    {
      "epoch": 0.4725124943207633,
      "grad_norm": 57.342852083624614,
      "learning_rate": 5.059926008786647e-07,
      "logits/chosen": -6.52951192855835,
      "logits/rejected": -6.941732883453369,
      "logps/chosen": -0.5963850021362305,
      "logps/rejected": -0.6861479878425598,
      "loss": 4.8488,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -5.963850021362305,
      "rewards/margins": 0.8976295590400696,
      "rewards/rejected": -6.86147928237915,
      "step": 65
    },
    {
      "epoch": 0.5088596092685143,
      "grad_norm": 50.7899004379066,
      "learning_rate": 4.5600645798745166e-07,
      "logits/chosen": -6.482469081878662,
      "logits/rejected": -6.863368988037109,
      "logps/chosen": -0.6205894947052002,
      "logps/rejected": -0.7471215128898621,
      "loss": 4.7473,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -6.205894470214844,
      "rewards/margins": 1.265321135520935,
      "rewards/rejected": -7.47121524810791,
      "step": 70
    },
    {
      "epoch": 0.5452067242162654,
      "grad_norm": 51.764811999912986,
      "learning_rate": 4.051081418863895e-07,
      "logits/chosen": -5.929373264312744,
      "logits/rejected": -6.43304443359375,
      "logps/chosen": -0.6067383885383606,
      "logps/rejected": -0.7068555951118469,
      "loss": 4.6926,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -6.067384243011475,
      "rewards/margins": 1.0011712312698364,
      "rewards/rejected": -7.068556308746338,
      "step": 75
    },
    {
      "epoch": 0.5815538391640164,
      "grad_norm": 52.24384814108085,
      "learning_rate": 3.541266298406398e-07,
      "logits/chosen": -6.554868221282959,
      "logits/rejected": -6.9319915771484375,
      "logps/chosen": -0.5482149720191956,
      "logps/rejected": -0.761601984500885,
      "loss": 4.8385,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -5.482149600982666,
      "rewards/margins": 2.1338706016540527,
      "rewards/rejected": -7.616020202636719,
      "step": 80
    },
    {
      "epoch": 0.6179009541117674,
      "grad_norm": 49.865022637467455,
      "learning_rate": 3.0389225412181565e-07,
      "logits/chosen": -6.109275817871094,
      "logits/rejected": -6.693246364593506,
      "logps/chosen": -0.5953459739685059,
      "logps/rejected": -0.7325998544692993,
      "loss": 4.5675,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -5.953459739685059,
      "rewards/margins": 1.3725389242172241,
      "rewards/rejected": -7.325998783111572,
      "step": 85
    },
    {
      "epoch": 0.6542480690595184,
      "grad_norm": 55.385656081268344,
      "learning_rate": 2.5522317844515273e-07,
      "logits/chosen": -6.353059768676758,
      "logits/rejected": -6.821288108825684,
      "logps/chosen": -0.5759108662605286,
      "logps/rejected": -0.710253119468689,
      "loss": 4.5135,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -5.759108066558838,
      "rewards/margins": 1.3434231281280518,
      "rewards/rejected": -7.102531433105469,
      "step": 90
    },
    {
      "epoch": 0.6905951840072694,
      "grad_norm": 54.711123059457066,
      "learning_rate": 2.0891207259509476e-07,
      "logits/chosen": -6.575620174407959,
      "logits/rejected": -6.845908164978027,
      "logps/chosen": -0.6202897429466248,
      "logps/rejected": -0.7272644639015198,
      "loss": 4.576,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -6.202897548675537,
      "rewards/margins": 1.0697463750839233,
      "rewards/rejected": -7.27264404296875,
      "step": 95
    },
    {
      "epoch": 0.7269422989550205,
      "grad_norm": 51.133951393232486,
      "learning_rate": 1.6571320226872206e-07,
      "logits/chosen": -6.456846714019775,
      "logits/rejected": -7.050930023193359,
      "logps/chosen": -0.5731553435325623,
      "logps/rejected": -0.722934365272522,
      "loss": 4.5131,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -5.731553554534912,
      "rewards/margins": 1.4977900981903076,
      "rewards/rejected": -7.229344367980957,
      "step": 100
    },
    {
      "epoch": 0.7632894139027715,
      "grad_norm": 51.6827958226427,
      "learning_rate": 1.2633014440382787e-07,
      "logits/chosen": -6.841757774353027,
      "logits/rejected": -7.067150115966797,
      "logps/chosen": -0.6003233194351196,
      "logps/rejected": -0.7174679636955261,
      "loss": 4.4139,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -6.003233432769775,
      "rewards/margins": 1.1714468002319336,
      "rewards/rejected": -7.174680233001709,
      "step": 105
    },
    {
      "epoch": 0.7996365288505225,
      "grad_norm": 63.215678605250034,
      "learning_rate": 9.14043280712228e-08,
      "logits/chosen": -6.239346504211426,
      "logits/rejected": -6.602126121520996,
      "logps/chosen": -0.614771842956543,
      "logps/rejected": -0.8130232691764832,
      "loss": 4.6241,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -6.147718906402588,
      "rewards/margins": 1.9825141429901123,
      "rewards/rejected": -8.130231857299805,
      "step": 110
    },
    {
      "epoch": 0.8359836437982735,
      "grad_norm": 59.93834778024071,
      "learning_rate": 6.150458756494239e-08,
      "logits/chosen": -6.464346885681152,
      "logits/rejected": -6.851041316986084,
      "logps/chosen": -0.6194056272506714,
      "logps/rejected": -0.776168704032898,
      "loss": 4.4725,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -6.194056510925293,
      "rewards/margins": 1.567630410194397,
      "rewards/rejected": -7.7616868019104,
      "step": 115
    },
    {
      "epoch": 0.8723307587460245,
      "grad_norm": 63.957410261640334,
      "learning_rate": 3.711789783843522e-08,
      "logits/chosen": -5.914224147796631,
      "logits/rejected": -6.372625350952148,
      "logps/chosen": -0.5956039428710938,
      "logps/rejected": -0.7622672319412231,
      "loss": 4.5261,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -5.9560394287109375,
      "rewards/margins": 1.6666326522827148,
      "rewards/rejected": -7.622672080993652,
      "step": 120
    },
    {
      "epoch": 0.9086778736937755,
      "grad_norm": 64.67587211778695,
      "learning_rate": 1.8641443178027784e-08,
      "logits/chosen": -6.29254674911499,
      "logits/rejected": -6.541565895080566,
      "logps/chosen": -0.6281149387359619,
      "logps/rejected": -0.7664488554000854,
      "loss": 4.6393,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -6.281149864196777,
      "rewards/margins": 1.3833396434783936,
      "rewards/rejected": -7.664488792419434,
      "step": 125
    },
    {
      "epoch": 0.9450249886415266,
      "grad_norm": 53.764099269870464,
      "learning_rate": 6.376148290617145e-09,
      "logits/chosen": -6.527040958404541,
      "logits/rejected": -7.045149803161621,
      "logps/chosen": -0.6423445343971252,
      "logps/rejected": -0.7968176603317261,
      "loss": 4.4187,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -6.423445701599121,
      "rewards/margins": 1.5447312593460083,
      "rewards/rejected": -7.968176364898682,
      "step": 130
    },
    {
      "epoch": 0.9813721035892776,
      "grad_norm": 54.882883609182315,
      "learning_rate": 5.217771643080127e-10,
      "logits/chosen": -6.546690940856934,
      "logits/rejected": -6.896824836730957,
      "logps/chosen": -0.6224032640457153,
      "logps/rejected": -0.7323735952377319,
      "loss": 4.5659,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -6.224032878875732,
      "rewards/margins": 1.0997024774551392,
      "rewards/rejected": -7.32373571395874,
      "step": 135
    },
    {
      "epoch": 0.995910949568378,
      "step": 137,
      "total_flos": 0.0,
      "train_loss": 5.239574314033898,
      "train_runtime": 2514.0727,
      "train_samples_per_second": 7.003,
      "train_steps_per_second": 0.054
    }
  ],
  "logging_steps": 5,
  "max_steps": 137,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}