{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9968652037617555,
  "eval_steps": 500,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "learning_rate": 3.125e-08,
      "logits/chosen": -2.0781588554382324,
      "logits/rejected": -1.9858250617980957,
      "logps/chosen": -453.2818603515625,
      "logps/pi_response": -300.6171875,
      "logps/ref_response": -300.6171875,
      "logps/rejected": -393.94146728515625,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.1030797958374023,
      "logits/rejected": -2.0334737300872803,
      "logps/chosen": -286.7795715332031,
      "logps/pi_response": -192.09445190429688,
      "logps/ref_response": -192.19212341308594,
      "logps/rejected": -342.5543518066406,
      "loss": 0.6687,
      "rewards/accuracies": 0.4930555522441864,
      "rewards/chosen": -0.049170926213264465,
      "rewards/margins": 0.10100988298654556,
      "rewards/rejected": -0.15018080174922943,
      "step": 10
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -2.0767107009887695,
      "logits/rejected": -2.057764768600464,
      "logps/chosen": -252.1197967529297,
      "logps/pi_response": -182.93057250976562,
      "logps/ref_response": -184.64120483398438,
      "logps/rejected": -385.4762268066406,
      "loss": 0.7043,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.17670336365699768,
      "rewards/margins": 0.7420909404754639,
      "rewards/rejected": -0.9187943339347839,
      "step": 20
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -2.079202175140381,
      "logits/rejected": -2.032349109649658,
      "logps/chosen": -291.5123596191406,
      "logps/pi_response": -202.2268829345703,
      "logps/ref_response": -192.75753784179688,
      "logps/rejected": -345.61810302734375,
      "loss": 0.7312,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.801366925239563,
      "rewards/margins": 0.6456394195556641,
      "rewards/rejected": -1.4470064640045166,
      "step": 30
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -2.147491931915283,
      "logits/rejected": -2.0794780254364014,
      "logps/chosen": -282.3897705078125,
      "logps/pi_response": -198.35958862304688,
      "logps/ref_response": -189.3666534423828,
      "logps/rejected": -389.67364501953125,
      "loss": 0.8956,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.9823302030563354,
      "rewards/margins": 0.8020473718643188,
      "rewards/rejected": -1.7843774557113647,
      "step": 40
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -2.226250171661377,
      "logits/rejected": -2.1124267578125,
      "logps/chosen": -302.74493408203125,
      "logps/pi_response": -224.74600219726562,
      "logps/ref_response": -208.11865234375,
      "logps/rejected": -445.48974609375,
      "loss": 0.8241,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.6344465017318726,
      "rewards/margins": 1.1299909353256226,
      "rewards/rejected": -2.764437198638916,
      "step": 50
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -2.119874954223633,
      "logits/rejected": -2.081808090209961,
      "logps/chosen": -285.70880126953125,
      "logps/pi_response": -211.44058227539062,
      "logps/ref_response": -197.88754272460938,
      "logps/rejected": -408.64111328125,
      "loss": 0.7844,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.3133118152618408,
      "rewards/margins": 0.963096022605896,
      "rewards/rejected": -2.2764081954956055,
      "step": 60
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -2.0851354598999023,
      "logits/rejected": -2.0019078254699707,
      "logps/chosen": -282.61920166015625,
      "logps/pi_response": -194.21969604492188,
      "logps/ref_response": -180.25257873535156,
      "logps/rejected": -390.8282775878906,
      "loss": 0.8222,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2267100811004639,
      "rewards/margins": 1.4668184518814087,
      "rewards/rejected": -2.693528652191162,
      "step": 70
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -2.161830186843872,
      "logits/rejected": -2.095283269882202,
      "logps/chosen": -327.2430114746094,
      "logps/pi_response": -215.4277801513672,
      "logps/ref_response": -201.3350372314453,
      "logps/rejected": -383.038818359375,
      "loss": 0.6609,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.1827173233032227,
      "rewards/margins": 1.14064621925354,
      "rewards/rejected": -2.323363780975342,
      "step": 80
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -2.1371817588806152,
      "logits/rejected": -2.031243085861206,
      "logps/chosen": -288.8699951171875,
      "logps/pi_response": -221.05953979492188,
      "logps/ref_response": -203.6631317138672,
      "logps/rejected": -432.8627014160156,
      "loss": 0.6938,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.5247129201889038,
      "rewards/margins": 1.1340636014938354,
      "rewards/rejected": -2.6587765216827393,
      "step": 90
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -2.113327741622925,
      "logits/rejected": -2.029888391494751,
      "logps/chosen": -297.3309326171875,
      "logps/pi_response": -229.23782348632812,
      "logps/ref_response": -213.95852661132812,
      "logps/rejected": -410.16015625,
      "loss": 0.7029,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5872786045074463,
      "rewards/margins": 0.9572550654411316,
      "rewards/rejected": -2.5445337295532227,
      "step": 100
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -2.10136342048645,
      "logits/rejected": -2.0246944427490234,
      "logps/chosen": -301.06085205078125,
      "logps/pi_response": -213.1235809326172,
      "logps/ref_response": -197.9862823486328,
      "logps/rejected": -417.035400390625,
      "loss": 0.6701,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4077179431915283,
      "rewards/margins": 1.052346110343933,
      "rewards/rejected": -2.460064172744751,
      "step": 110
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -2.0771682262420654,
      "logits/rejected": -2.0391485691070557,
      "logps/chosen": -286.8796691894531,
      "logps/pi_response": -202.24591064453125,
      "logps/ref_response": -183.84140014648438,
      "logps/rejected": -393.9871520996094,
      "loss": 0.582,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8573909997940063,
      "rewards/margins": 1.027687668800354,
      "rewards/rejected": -2.8850789070129395,
      "step": 120
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -2.0720248222351074,
      "logits/rejected": -2.039855480194092,
      "logps/chosen": -284.1001892089844,
      "logps/pi_response": -205.81741333007812,
      "logps/ref_response": -192.61219787597656,
      "logps/rejected": -424.58856201171875,
      "loss": 0.6267,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.078873872756958,
      "rewards/margins": 1.561975121498108,
      "rewards/rejected": -2.6408488750457764,
      "step": 130
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": -2.074803590774536,
      "logits/rejected": -1.9720996618270874,
      "logps/chosen": -289.3224182128906,
      "logps/pi_response": -216.6945343017578,
      "logps/ref_response": -199.8957977294922,
      "logps/rejected": -415.46209716796875,
      "loss": 0.6077,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5722277164459229,
      "rewards/margins": 1.58167564868927,
      "rewards/rejected": -3.1539032459259033,
      "step": 140
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -2.0950675010681152,
      "logits/rejected": -2.0208332538604736,
      "logps/chosen": -298.1050720214844,
      "logps/pi_response": -232.5490264892578,
      "logps/ref_response": -216.212158203125,
      "logps/rejected": -405.6884460449219,
      "loss": 0.6254,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4602974653244019,
      "rewards/margins": 0.9969442486763,
      "rewards/rejected": -2.4572417736053467,
      "step": 150
    },
    {
      "epoch": 1.0,
      "step": 159,
      "total_flos": 0.0,
      "train_loss": 0.6964452761524128,
      "train_runtime": 4539.1381,
      "train_samples_per_second": 4.489,
      "train_steps_per_second": 0.035
    }
  ],
  "logging_steps": 10,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}