{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 6.896551724137931,
  "eval_steps": 500,
  "global_step": 200,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.3448275862068966,
      "grad_norm": 3.9107234477996826,
      "learning_rate": 8.620689655172415e-07,
      "logits/chosen": -2.363854169845581,
      "logits/rejected": -2.338671922683716,
      "logps/chosen": -76.8819351196289,
      "logps/rejected": -75.86869049072266,
      "loss": 0.6942,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.003996999468654394,
      "rewards/margins": -0.004145228303968906,
      "rewards/rejected": 0.00014822949015069753,
      "step": 10
    },
    {
      "epoch": 0.6896551724137931,
      "grad_norm": 4.328212738037109,
      "learning_rate": 1.724137931034483e-06,
      "logits/chosen": -2.3617663383483887,
      "logits/rejected": -2.3477540016174316,
      "logps/chosen": -93.76543426513672,
      "logps/rejected": -75.16656494140625,
      "loss": 0.6931,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.002653368515893817,
      "rewards/margins": 0.006352785974740982,
      "rewards/rejected": -0.003699416993185878,
      "step": 20
    },
    {
      "epoch": 1.0344827586206897,
      "grad_norm": 3.473827838897705,
      "learning_rate": 2.5862068965517246e-06,
      "logits/chosen": -2.3354058265686035,
      "logits/rejected": -2.3330740928649902,
      "logps/chosen": -138.9960479736328,
      "logps/rejected": -67.00438690185547,
      "loss": 0.6927,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.01745142787694931,
      "rewards/margins": 0.013760591857135296,
      "rewards/rejected": 0.0036908381152898073,
      "step": 30
    },
    {
      "epoch": 1.3793103448275863,
      "grad_norm": 3.9458820819854736,
      "learning_rate": 3.448275862068966e-06,
      "logits/chosen": -2.3478751182556152,
      "logits/rejected": -2.3700289726257324,
      "logps/chosen": -75.59983825683594,
      "logps/rejected": -67.20357513427734,
      "loss": 0.69,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00024204826331697404,
      "rewards/margins": 0.005866119172424078,
      "rewards/rejected": -0.005624071694910526,
      "step": 40
    },
    {
      "epoch": 1.7241379310344827,
      "grad_norm": 4.299890041351318,
      "learning_rate": 4.310344827586207e-06,
      "logits/chosen": -2.338815212249756,
      "logits/rejected": -2.3532769680023193,
      "logps/chosen": -72.74342346191406,
      "logps/rejected": -87.63409423828125,
      "loss": 0.6899,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.003249581903219223,
      "rewards/margins": 0.006888723932206631,
      "rewards/rejected": -0.010138307698071003,
      "step": 50
    },
    {
      "epoch": 2.0689655172413794,
      "grad_norm": 4.957555294036865,
      "learning_rate": 4.999818897894192e-06,
      "logits/chosen": -2.344572067260742,
      "logits/rejected": -2.345888614654541,
      "logps/chosen": -80.16046142578125,
      "logps/rejected": -73.05986022949219,
      "loss": 0.6802,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.013288321904838085,
      "rewards/margins": 0.020968889817595482,
      "rewards/rejected": -0.03425721079111099,
      "step": 60
    },
    {
      "epoch": 2.413793103448276,
      "grad_norm": 4.212078094482422,
      "learning_rate": 4.9934830787948756e-06,
      "logits/chosen": -2.3330090045928955,
      "logits/rejected": -2.328575372695923,
      "logps/chosen": -73.63365173339844,
      "logps/rejected": -74.45356750488281,
      "loss": 0.6693,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.009874681942164898,
      "rewards/margins": 0.04785541445016861,
      "rewards/rejected": -0.05773010104894638,
      "step": 70
    },
    {
      "epoch": 2.7586206896551726,
      "grad_norm": 4.446742534637451,
      "learning_rate": 4.978118375700895e-06,
      "logits/chosen": -2.3472495079040527,
      "logits/rejected": -2.3697409629821777,
      "logps/chosen": -73.44490814208984,
      "logps/rejected": -89.69837951660156,
      "loss": 0.6553,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.022817375138401985,
      "rewards/margins": 0.09045850485563278,
      "rewards/rejected": -0.11327588558197021,
      "step": 80
    },
    {
      "epoch": 3.103448275862069,
      "grad_norm": 4.435715198516846,
      "learning_rate": 4.953780424089803e-06,
      "logits/chosen": -2.346717596054077,
      "logits/rejected": -2.3626608848571777,
      "logps/chosen": -85.64796447753906,
      "logps/rejected": -80.06268310546875,
      "loss": 0.6432,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0543200746178627,
      "rewards/margins": 0.1085677295923233,
      "rewards/rejected": -0.1628878116607666,
      "step": 90
    },
    {
      "epoch": 3.4482758620689653,
      "grad_norm": 4.5200910568237305,
      "learning_rate": 4.920557351506409e-06,
      "logits/chosen": -2.358900547027588,
      "logits/rejected": -2.3591980934143066,
      "logps/chosen": -82.83473205566406,
      "logps/rejected": -80.66204071044922,
      "loss": 0.5996,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.046993009746074677,
      "rewards/margins": 0.20401433110237122,
      "rewards/rejected": -0.2510073184967041,
      "step": 100
    },
    {
      "epoch": 3.793103448275862,
      "grad_norm": 4.106562614440918,
      "learning_rate": 4.878569458453592e-06,
      "logits/chosen": -2.347762107849121,
      "logits/rejected": -2.33540415763855,
      "logps/chosen": -72.23506164550781,
      "logps/rejected": -85.77088928222656,
      "loss": 0.6003,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.051100969314575195,
      "rewards/margins": 0.24443332850933075,
      "rewards/rejected": -0.29553431272506714,
      "step": 110
    },
    {
      "epoch": 4.137931034482759,
      "grad_norm": 4.67632532119751,
      "learning_rate": 4.827968782785062e-06,
      "logits/chosen": -2.353370428085327,
      "logits/rejected": -2.378964900970459,
      "logps/chosen": -67.91919708251953,
      "logps/rejected": -86.26390838623047,
      "loss": 0.5716,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.11952020972967148,
      "rewards/margins": 0.30119410157203674,
      "rewards/rejected": -0.42071428894996643,
      "step": 120
    },
    {
      "epoch": 4.482758620689655,
      "grad_norm": 4.5131025314331055,
      "learning_rate": 4.7689385491773934e-06,
      "logits/chosen": -2.3820648193359375,
      "logits/rejected": -2.403965473175049,
      "logps/chosen": -71.66841125488281,
      "logps/rejected": -77.90869140625,
      "loss": 0.5362,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0694696456193924,
      "rewards/margins": 0.47986412048339844,
      "rewards/rejected": -0.5493337512016296,
      "step": 130
    },
    {
      "epoch": 4.827586206896552,
      "grad_norm": 4.446605205535889,
      "learning_rate": 4.70169250567482e-06,
      "logits/chosen": -2.372788906097412,
      "logits/rejected": -2.379612684249878,
      "logps/chosen": -69.44779968261719,
      "logps/rejected": -80.1120376586914,
      "loss": 0.5186,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.12896183133125305,
      "rewards/margins": 0.5137210488319397,
      "rewards/rejected": -0.6426829099655151,
      "step": 140
    },
    {
      "epoch": 5.172413793103448,
      "grad_norm": 4.605374813079834,
      "learning_rate": 4.626474149709127e-06,
      "logits/chosen": -2.3140041828155518,
      "logits/rejected": -2.336975574493408,
      "logps/chosen": -87.05641174316406,
      "logps/rejected": -94.53668212890625,
      "loss": 0.4794,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.15513856709003448,
      "rewards/margins": 0.6417296528816223,
      "rewards/rejected": -0.7968682050704956,
      "step": 150
    },
    {
      "epoch": 5.517241379310345,
      "grad_norm": 6.027809143066406,
      "learning_rate": 4.54355584639723e-06,
      "logits/chosen": -2.365192413330078,
      "logits/rejected": -2.3872745037078857,
      "logps/chosen": -73.7781753540039,
      "logps/rejected": -82.1873550415039,
      "loss": 0.4599,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.25834327936172485,
      "rewards/margins": 0.5800828337669373,
      "rewards/rejected": -0.8384261131286621,
      "step": 160
    },
    {
      "epoch": 5.862068965517241,
      "grad_norm": 5.196141242980957,
      "learning_rate": 4.45323784230908e-06,
      "logits/chosen": -2.391653299331665,
      "logits/rejected": -2.394854784011841,
      "logps/chosen": -72.33963775634766,
      "logps/rejected": -86.77699279785156,
      "loss": 0.4467,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.2543284595012665,
      "rewards/margins": 0.7806032299995422,
      "rewards/rejected": -1.0349315404891968,
      "step": 170
    },
    {
      "epoch": 6.206896551724138,
      "grad_norm": 5.285089015960693,
      "learning_rate": 4.355847178277025e-06,
      "logits/chosen": -2.3918795585632324,
      "logits/rejected": -2.4050662517547607,
      "logps/chosen": -77.20832061767578,
      "logps/rejected": -98.5592041015625,
      "loss": 0.4052,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.35020238161087036,
      "rewards/margins": 1.0169063806533813,
      "rewards/rejected": -1.367108702659607,
      "step": 180
    },
    {
      "epoch": 6.551724137931035,
      "grad_norm": 5.308840751647949,
      "learning_rate": 4.2517365051833564e-06,
      "logits/chosen": -2.3778843879699707,
      "logits/rejected": -2.3865675926208496,
      "logps/chosen": -69.90904235839844,
      "logps/rejected": -86.66047668457031,
      "loss": 0.3809,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.42647725343704224,
      "rewards/margins": 0.9949015378952026,
      "rewards/rejected": -1.4213788509368896,
      "step": 190
    },
    {
      "epoch": 6.896551724137931,
      "grad_norm": 6.095545768737793,
      "learning_rate": 4.141282807014034e-06,
      "logits/chosen": -2.407437801361084,
      "logits/rejected": -2.4296727180480957,
      "logps/chosen": -82.95701599121094,
      "logps/rejected": -83.4615478515625,
      "loss": 0.3742,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6070786714553833,
      "rewards/margins": 0.968428909778595,
      "rewards/rejected": -1.5755075216293335,
      "step": 200
    }
  ],
  "logging_steps": 10,
  "max_steps": 580,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 20,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 7.2868315071198e+17,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}