| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9968652037617555, |
| "eval_steps": 500, |
| "global_step": 159, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.01, |
| "learning_rate": 3.125e-08, |
| "logits/chosen": -2.1063718795776367, |
| "logits/rejected": -2.0172834396362305, |
| "logps/chosen": -441.2913818359375, |
| "logps/pi_response": -291.7864990234375, |
| "logps/ref_response": -291.7864990234375, |
| "logps/rejected": -407.91192626953125, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 3.1249999999999997e-07, |
| "logits/chosen": -2.116103172302246, |
| "logits/rejected": -2.0064265727996826, |
| "logps/chosen": -276.1221008300781, |
| "logps/pi_response": -191.32034301757812, |
| "logps/ref_response": -190.91871643066406, |
| "logps/rejected": -346.5228576660156, |
| "loss": 0.687, |
| "rewards/accuracies": 0.5694444179534912, |
| "rewards/chosen": -0.010301925241947174, |
| "rewards/margins": 0.014795614406466484, |
| "rewards/rejected": -0.025097541511058807, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 4.990353313429303e-07, |
| "logits/chosen": -2.0136561393737793, |
| "logits/rejected": -1.981844186782837, |
| "logps/chosen": -291.6961975097656, |
| "logps/pi_response": -205.843505859375, |
| "logps/ref_response": -189.8782196044922, |
| "logps/rejected": -444.3314514160156, |
| "loss": 0.6587, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -0.355610728263855, |
| "rewards/margins": 0.2842535376548767, |
| "rewards/rejected": -0.6398642659187317, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 4.882681251368548e-07, |
| "logits/chosen": -1.9893925189971924, |
| "logits/rejected": -1.946048378944397, |
| "logps/chosen": -291.5216979980469, |
| "logps/pi_response": -212.42111206054688, |
| "logps/ref_response": -199.7083740234375, |
| "logps/rejected": -389.95782470703125, |
| "loss": 0.673, |
| "rewards/accuracies": 0.5562499761581421, |
| "rewards/chosen": -0.2551540434360504, |
| "rewards/margins": 0.27687785029411316, |
| "rewards/rejected": -0.5320318937301636, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.25, |
| "learning_rate": 4.6604720940421207e-07, |
| "logits/chosen": -2.0515387058258057, |
| "logits/rejected": -1.974180817604065, |
| "logps/chosen": -295.8572998046875, |
| "logps/pi_response": -216.64120483398438, |
| "logps/ref_response": -197.79104614257812, |
| "logps/rejected": -470.1461486816406, |
| "loss": 0.6364, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.25649192929267883, |
| "rewards/margins": 0.36993804574012756, |
| "rewards/rejected": -0.6264299750328064, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 4.3344075855595097e-07, |
| "logits/chosen": -2.0759925842285156, |
| "logits/rejected": -1.9234635829925537, |
| "logps/chosen": -312.621826171875, |
| "logps/pi_response": -232.86474609375, |
| "logps/ref_response": -209.6178436279297, |
| "logps/rejected": -509.26318359375, |
| "loss": 0.625, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.3486739993095398, |
| "rewards/margins": 0.5309742093086243, |
| "rewards/rejected": -0.8796480894088745, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 3.920161866827889e-07, |
| "logits/chosen": -1.797320008277893, |
| "logits/rejected": -1.6790682077407837, |
| "logps/chosen": -304.28594970703125, |
| "logps/pi_response": -239.77108764648438, |
| "logps/ref_response": -206.43423461914062, |
| "logps/rejected": -470.97528076171875, |
| "loss": 0.6067, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.4016168713569641, |
| "rewards/margins": 0.475954532623291, |
| "rewards/rejected": -0.8775714635848999, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 3.4376480090239047e-07, |
| "logits/chosen": -1.44484281539917, |
| "logits/rejected": -1.1385029554367065, |
| "logps/chosen": -307.74578857421875, |
| "logps/pi_response": -231.57217407226562, |
| "logps/ref_response": -184.50869750976562, |
| "logps/rejected": -461.8819885253906, |
| "loss": 0.589, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.42910051345825195, |
| "rewards/margins": 0.6111310124397278, |
| "rewards/rejected": -1.040231466293335, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.5, |
| "learning_rate": 2.910060778827554e-07, |
| "logits/chosen": -1.2302042245864868, |
| "logits/rejected": -0.8724157214164734, |
| "logps/chosen": -384.6354064941406, |
| "logps/pi_response": -266.6485290527344, |
| "logps/ref_response": -202.98727416992188, |
| "logps/rejected": -476.89410400390625, |
| "loss": 0.5814, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.7001193165779114, |
| "rewards/margins": 0.4296953082084656, |
| "rewards/rejected": -1.1298145055770874, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.56, |
| "learning_rate": 2.3627616503391812e-07, |
| "logits/chosen": -1.0443007946014404, |
| "logits/rejected": -0.5559221506118774, |
| "logps/chosen": -355.37078857421875, |
| "logps/pi_response": -265.6754150390625, |
| "logps/ref_response": -202.15640258789062, |
| "logps/rejected": -486.13507080078125, |
| "loss": 0.5948, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.6906540989875793, |
| "rewards/margins": 0.4475804269313812, |
| "rewards/rejected": -1.1382344961166382, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 1.8220596619089573e-07, |
| "logits/chosen": -0.894809901714325, |
| "logits/rejected": -0.34124261140823364, |
| "logps/chosen": -358.05755615234375, |
| "logps/pi_response": -283.3772277832031, |
| "logps/ref_response": -207.0720672607422, |
| "logps/rejected": -521.771728515625, |
| "loss": 0.5624, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.7900195121765137, |
| "rewards/margins": 0.6714037656784058, |
| "rewards/rejected": -1.4614231586456299, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 1.3139467229135998e-07, |
| "logits/chosen": -0.8867988586425781, |
| "logits/rejected": -0.3483342230319977, |
| "logps/chosen": -380.3785095214844, |
| "logps/pi_response": -293.0396728515625, |
| "logps/ref_response": -201.5030975341797, |
| "logps/rejected": -526.4119873046875, |
| "loss": 0.5833, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.8987709283828735, |
| "rewards/margins": 0.5739961266517639, |
| "rewards/rejected": -1.4727669954299927, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 8.628481651367875e-08, |
| "logits/chosen": -0.8515814542770386, |
| "logits/rejected": -0.5452559590339661, |
| "logps/chosen": -343.3797912597656, |
| "logps/pi_response": -272.0332336425781, |
| "logps/ref_response": -198.11781311035156, |
| "logps/rejected": -536.7486572265625, |
| "loss": 0.5449, |
| "rewards/accuracies": 0.7562500238418579, |
| "rewards/chosen": -0.7609699368476868, |
| "rewards/margins": 0.6519715785980225, |
| "rewards/rejected": -1.412941575050354, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 4.904486005914027e-08, |
| "logits/chosen": -0.8249815702438354, |
| "logits/rejected": -0.48517799377441406, |
| "logps/chosen": -350.73858642578125, |
| "logps/pi_response": -274.28204345703125, |
| "logps/ref_response": -193.91168212890625, |
| "logps/rejected": -545.8704223632812, |
| "loss": 0.5521, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": -0.7474575042724609, |
| "rewards/margins": 0.7682065963745117, |
| "rewards/rejected": -1.5156641006469727, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 2.1464952759020856e-08, |
| "logits/chosen": -0.6546958088874817, |
| "logits/rejected": -0.09045759588479996, |
| "logps/chosen": -367.89312744140625, |
| "logps/pi_response": -284.0288391113281, |
| "logps/ref_response": -196.39334106445312, |
| "logps/rejected": -535.0501708984375, |
| "loss": 0.5633, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.8734307289123535, |
| "rewards/margins": 0.7547620534896851, |
| "rewards/rejected": -1.6281929016113281, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 4.8708793644441086e-09, |
| "logits/chosen": -0.8392259478569031, |
| "logits/rejected": -0.3418545424938202, |
| "logps/chosen": -361.2017517089844, |
| "logps/pi_response": -292.3738708496094, |
| "logps/ref_response": -211.84817504882812, |
| "logps/rejected": -521.71875, |
| "loss": 0.5783, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.8251138925552368, |
| "rewards/margins": 0.5152265429496765, |
| "rewards/rejected": -1.340340495109558, |
| "step": 150 |
| }, |
| { |
| "epoch": 1.0, |
| "step": 159, |
| "total_flos": 0.0, |
| "train_loss": 0.5938605962309448, |
| "train_runtime": 4590.1424, |
| "train_samples_per_second": 4.44, |
| "train_steps_per_second": 0.035 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 159, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "total_flos": 0.0, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
