{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9964513839602555,
  "eval_steps": 500,
  "global_step": 162,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "eta": 0.0010000000474974513,
      "grad_norm": 22.254964893351787,
      "learning_rate": 2.941176470588235e-08,
      "logits/chosen": -2.4821925163269043,
      "logits/rejected": -2.235710382461548,
      "logps/chosen": -280.2124328613281,
      "logps/pi_response": -436.6075744628906,
      "logps/ref_response": -436.6075744628906,
      "logps/rejected": -273.4680480957031,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06,
      "eta": 0.0010000000474974513,
      "grad_norm": 23.328818767881323,
      "learning_rate": 2.941176470588235e-07,
      "logits/chosen": -2.3237369060516357,
      "logits/rejected": -2.2622690200805664,
      "logps/chosen": -239.0484619140625,
      "logps/pi_response": -371.8123474121094,
      "logps/ref_response": -370.81756591796875,
      "logps/rejected": -253.18133544921875,
      "loss": 0.6927,
      "rewards/accuracies": 0.4444444477558136,
      "rewards/chosen": -0.009806615300476551,
      "rewards/margins": 0.001764079206623137,
      "rewards/rejected": -0.011570693925023079,
      "step": 10
    },
    {
      "epoch": 0.12,
      "eta": 0.0010000000474974513,
      "grad_norm": 31.176954929712068,
      "learning_rate": 4.99472085783721e-07,
      "logits/chosen": -2.121244430541992,
      "logits/rejected": -2.1339948177337646,
      "logps/chosen": -252.5571746826172,
      "logps/pi_response": -376.398193359375,
      "logps/ref_response": -357.7356872558594,
      "logps/rejected": -262.6872863769531,
      "loss": 0.6923,
      "rewards/accuracies": 0.5076923370361328,
      "rewards/chosen": -0.21289017796516418,
      "rewards/margins": -0.0055158319883048534,
      "rewards/rejected": -0.20737436413764954,
      "step": 20
    },
    {
      "epoch": 0.18,
      "eta": 0.0010000000474974513,
      "grad_norm": 29.374574206663485,
      "learning_rate": 4.901488388458247e-07,
      "logits/chosen": -2.0692079067230225,
      "logits/rejected": -2.033362865447998,
      "logps/chosen": -240.89402770996094,
      "logps/pi_response": -404.3094787597656,
      "logps/ref_response": -375.89013671875,
      "logps/rejected": -253.24954223632812,
      "loss": 0.6895,
      "rewards/accuracies": 0.5038461685180664,
      "rewards/chosen": -0.07824164628982544,
      "rewards/margins": 0.01707359030842781,
      "rewards/rejected": -0.09531523287296295,
      "step": 30
    },
    {
      "epoch": 0.25,
      "eta": 0.0010000000474974513,
      "grad_norm": 30.44936672759969,
      "learning_rate": 4.695964991097616e-07,
      "logits/chosen": -1.8107295036315918,
      "logits/rejected": -2.005711555480957,
      "logps/chosen": -303.3603515625,
      "logps/pi_response": -455.744873046875,
      "logps/ref_response": -391.4946594238281,
      "logps/rejected": -326.20098876953125,
      "loss": 0.6898,
      "rewards/accuracies": 0.5307692289352417,
      "rewards/chosen": -0.6260919570922852,
      "rewards/margins": 0.034338854253292084,
      "rewards/rejected": -0.6604308485984802,
      "step": 40
    },
    {
      "epoch": 0.31,
      "eta": 0.0010000000474974513,
      "grad_norm": 23.599398572955327,
      "learning_rate": 4.3877607113930516e-07,
      "logits/chosen": -1.907410979270935,
      "logits/rejected": -1.855884075164795,
      "logps/chosen": -248.3097686767578,
      "logps/pi_response": -406.1575012207031,
      "logps/ref_response": -367.650390625,
      "logps/rejected": -256.7185974121094,
      "loss": 0.7017,
      "rewards/accuracies": 0.48076921701431274,
      "rewards/chosen": -0.12874366343021393,
      "rewards/margins": -0.0029275938868522644,
      "rewards/rejected": -0.12581607699394226,
      "step": 50
    },
    {
      "epoch": 0.37,
      "eta": 0.0010000000474974513,
      "grad_norm": 23.113382825281022,
      "learning_rate": 3.991286838919086e-07,
      "logits/chosen": -1.7167693376541138,
      "logits/rejected": -1.6973148584365845,
      "logps/chosen": -320.3218688964844,
      "logps/pi_response": -487.9947204589844,
      "logps/ref_response": -405.68280029296875,
      "logps/rejected": -322.4259948730469,
      "loss": 0.6945,
      "rewards/accuracies": 0.5615384578704834,
      "rewards/chosen": -0.5926837921142578,
      "rewards/margins": 0.038080714643001556,
      "rewards/rejected": -0.6307645440101624,
      "step": 60
    },
    {
      "epoch": 0.43,
      "eta": 0.0010000000474974513,
      "grad_norm": 31.784636012816986,
      "learning_rate": 3.52508205130354e-07,
      "logits/chosen": -1.5467545986175537,
      "logits/rejected": -1.5477180480957031,
      "logps/chosen": -306.8106994628906,
      "logps/pi_response": -480.5911865234375,
      "logps/ref_response": -377.5798034667969,
      "logps/rejected": -334.5196533203125,
      "loss": 0.6801,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.787551760673523,
      "rewards/margins": 0.06417767703533173,
      "rewards/rejected": -0.8517293930053711,
      "step": 70
    },
    {
      "epoch": 0.49,
      "eta": 0.0010000000474974513,
      "grad_norm": 35.2255241905511,
      "learning_rate": 3.010945566265912e-07,
      "logits/chosen": -1.584012508392334,
      "logits/rejected": -1.7565481662750244,
      "logps/chosen": -292.3354797363281,
      "logps/pi_response": -463.38507080078125,
      "logps/ref_response": -351.9272766113281,
      "logps/rejected": -315.16680908203125,
      "loss": 0.6865,
      "rewards/accuracies": 0.5961538553237915,
      "rewards/chosen": -0.6284902691841125,
      "rewards/margins": 0.07952240109443665,
      "rewards/rejected": -0.7080127596855164,
      "step": 80
    },
    {
      "epoch": 0.55,
      "eta": 0.0010000000474974513,
      "grad_norm": 24.755608967697253,
      "learning_rate": 2.4729178344249006e-07,
      "logits/chosen": -1.6239264011383057,
      "logits/rejected": -1.7038928270339966,
      "logps/chosen": -333.8011474609375,
      "logps/pi_response": -518.841064453125,
      "logps/ref_response": -389.2355651855469,
      "logps/rejected": -362.0622253417969,
      "loss": 0.6905,
      "rewards/accuracies": 0.5538461804389954,
      "rewards/chosen": -0.9721028804779053,
      "rewards/margins": 0.06480876356363297,
      "rewards/rejected": -1.0369116067886353,
      "step": 90
    },
    {
      "epoch": 0.62,
      "eta": 0.0010000000474974513,
      "grad_norm": 28.32503077830092,
      "learning_rate": 1.9361564345465145e-07,
      "logits/chosen": -1.6863784790039062,
      "logits/rejected": -1.576540470123291,
      "logps/chosen": -324.2234802246094,
      "logps/pi_response": -477.94110107421875,
      "logps/ref_response": -352.850830078125,
      "logps/rejected": -337.5308532714844,
      "loss": 0.6817,
      "rewards/accuracies": 0.5692307949066162,
      "rewards/chosen": -0.95728600025177,
      "rewards/margins": 0.05953861400485039,
      "rewards/rejected": -1.0168246030807495,
      "step": 100
    },
    {
      "epoch": 0.68,
      "eta": 0.0010000000474974513,
      "grad_norm": 24.75749478420196,
      "learning_rate": 1.4257597331216208e-07,
      "logits/chosen": -1.5470343828201294,
      "logits/rejected": -1.6905183792114258,
      "logps/chosen": -339.9206848144531,
      "logps/pi_response": -517.6537475585938,
      "logps/ref_response": -383.1593322753906,
      "logps/rejected": -354.73150634765625,
      "loss": 0.6791,
      "rewards/accuracies": 0.4923076927661896,
      "rewards/chosen": -1.0359457731246948,
      "rewards/margins": 0.02877073362469673,
      "rewards/rejected": -1.0647164583206177,
      "step": 110
    },
    {
      "epoch": 0.74,
      "eta": 0.0010000000474974513,
      "grad_norm": 27.374335177920894,
      "learning_rate": 9.655933126436563e-08,
      "logits/chosen": -1.7334721088409424,
      "logits/rejected": -1.6775918006896973,
      "logps/chosen": -323.307861328125,
      "logps/pi_response": -500.0389709472656,
      "logps/ref_response": -373.8336181640625,
      "logps/rejected": -337.35687255859375,
      "loss": 0.6743,
      "rewards/accuracies": 0.5730769038200378,
      "rewards/chosen": -0.884036660194397,
      "rewards/margins": 0.05840235576033592,
      "rewards/rejected": -0.9424390196800232,
      "step": 120
    },
    {
      "epoch": 0.8,
      "eta": 0.0010000000474974513,
      "grad_norm": 29.06627362451846,
      "learning_rate": 5.771740434959277e-08,
      "logits/chosen": -1.7559294700622559,
      "logits/rejected": -1.656353235244751,
      "logps/chosen": -322.9090270996094,
      "logps/pi_response": -490.60504150390625,
      "logps/ref_response": -358.2728271484375,
      "logps/rejected": -327.773681640625,
      "loss": 0.677,
      "rewards/accuracies": 0.5884615182876587,
      "rewards/chosen": -0.8856447339057922,
      "rewards/margins": 0.060253795236349106,
      "rewards/rejected": -0.9458985328674316,
      "step": 130
    },
    {
      "epoch": 0.86,
      "eta": 0.0010000000474974513,
      "grad_norm": 29.276601449078793,
      "learning_rate": 2.7866397900677185e-08,
      "logits/chosen": -1.5905346870422363,
      "logits/rejected": -1.5064502954483032,
      "logps/chosen": -327.3086242675781,
      "logps/pi_response": -507.3574523925781,
      "logps/ref_response": -365.5458984375,
      "logps/rejected": -337.4426574707031,
      "loss": 0.6696,
      "rewards/accuracies": 0.6307692527770996,
      "rewards/chosen": -0.9561350345611572,
      "rewards/margins": 0.08353108912706375,
      "rewards/rejected": -1.0396660566329956,
      "step": 140
    },
    {
      "epoch": 0.92,
      "eta": 0.0010000000474974513,
      "grad_norm": 30.496204004580633,
      "learning_rate": 8.402111802159412e-09,
      "logits/chosen": -1.74466872215271,
      "logits/rejected": -1.6440544128417969,
      "logps/chosen": -315.1639709472656,
      "logps/pi_response": -511.9558410644531,
      "logps/ref_response": -375.8587951660156,
      "logps/rejected": -326.924560546875,
      "loss": 0.6791,
      "rewards/accuracies": 0.5846154093742371,
      "rewards/chosen": -0.8547829389572144,
      "rewards/margins": 0.06981422752141953,
      "rewards/rejected": -0.9245970845222473,
      "step": 150
    },
    {
      "epoch": 0.98,
      "eta": 0.0010000000474974513,
      "grad_norm": 29.5797991936398,
      "learning_rate": 2.3467443900582197e-10,
      "logits/chosen": -1.836174726486206,
      "logits/rejected": -1.7468369007110596,
      "logps/chosen": -307.39251708984375,
      "logps/pi_response": -489.5311279296875,
      "logps/ref_response": -352.3800354003906,
      "logps/rejected": -338.456787109375,
      "loss": 0.6738,
      "rewards/accuracies": 0.5846154093742371,
      "rewards/chosen": -0.8461953401565552,
      "rewards/margins": 0.09575623273849487,
      "rewards/rejected": -0.9419515132904053,
      "step": 160
    },
    {
      "epoch": 1.0,
      "step": 162,
      "total_flos": 0.0,
      "train_loss": 0.6842105340810469,
      "train_runtime": 43334.2513,
      "train_samples_per_second": 0.488,
      "train_steps_per_second": 0.004
    }
  ],
  "logging_steps": 10,
  "max_steps": 162,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}