| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9945, |
| "eval_steps": 500, |
| "global_step": 153, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.01, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.20925004399681482, |
| "learning_rate": 3.125e-08, |
| "logits/chosen": -1.6728180646896362, |
| "logits/rejected": -1.6728180646896362, |
| "logps/chosen": -139.26568603515625, |
| "logps/pi_response": -223.70187377929688, |
| "logps/ref_response": -223.70187377929688, |
| "logps/rejected": -139.26568603515625, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.07, |
| "eta": 0.0009999999310821295, |
| "grad_norm": 0.2664449122199502, |
| "learning_rate": 3.1249999999999997e-07, |
| "logits/chosen": -1.9198987483978271, |
| "logits/rejected": -1.9198987483978271, |
| "logps/chosen": -185.7984161376953, |
| "logps/pi_response": -284.7489929199219, |
| "logps/ref_response": -274.8498229980469, |
| "logps/rejected": -185.7984161376953, |
| "loss": 0.693, |
| "rewards/accuracies": 0.09829059988260269, |
| "rewards/chosen": -0.05739467218518257, |
| "rewards/margins": -8.19944467878031e-09, |
| "rewards/rejected": -0.057394664734601974, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.13, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.17607425906899, |
| "learning_rate": 4.989490450759331e-07, |
| "logits/chosen": -1.2133342027664185, |
| "logits/rejected": -1.2133342027664185, |
| "logps/chosen": -344.1175231933594, |
| "logps/pi_response": -420.7164001464844, |
| "logps/ref_response": -268.8954772949219, |
| "logps/rejected": -344.1175231933594, |
| "loss": 0.6916, |
| "rewards/accuracies": 0.17307692766189575, |
| "rewards/chosen": -1.5597572326660156, |
| "rewards/margins": 2.3523059056174134e-08, |
| "rewards/rejected": -1.5597573518753052, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.2, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.9959359173659126, |
| "learning_rate": 4.872270441827174e-07, |
| "logits/chosen": 0.10036426037549973, |
| "logits/rejected": 0.10036426037549973, |
| "logps/chosen": -764.1023559570312, |
| "logps/pi_response": -850.1804809570312, |
| "logps/ref_response": -272.0489807128906, |
| "logps/rejected": -764.1023559570312, |
| "loss": 0.6873, |
| "rewards/accuracies": 0.11153846234083176, |
| "rewards/chosen": -5.858819007873535, |
| "rewards/margins": -3.943076620771535e-08, |
| "rewards/rejected": -5.858819484710693, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.26, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 1.9030553381279938, |
| "learning_rate": 4.6308512113530063e-07, |
| "logits/chosen": 2.6020889282226562, |
| "logits/rejected": 2.6020889282226562, |
| "logps/chosen": -15368.376953125, |
| "logps/pi_response": -9728.86328125, |
| "logps/ref_response": -275.9498596191406, |
| "logps/rejected": -15368.376953125, |
| "loss": 0.6, |
| "rewards/accuracies": 0.042307693511247635, |
| "rewards/chosen": -151.69932556152344, |
| "rewards/margins": 1.8339891028062993e-07, |
| "rewards/rejected": -151.69932556152344, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.33, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.9022013856679871, |
| "learning_rate": 4.277872161641681e-07, |
| "logits/chosen": 5.100710391998291, |
| "logits/rejected": 5.100710391998291, |
| "logps/chosen": -25452.970703125, |
| "logps/pi_response": -16005.287109375, |
| "logps/ref_response": -266.91033935546875, |
| "logps/rejected": -25452.970703125, |
| "loss": 0.5304, |
| "rewards/accuracies": 0.023076923564076424, |
| "rewards/chosen": -252.7429656982422, |
| "rewards/margins": -4.69501202360334e-07, |
| "rewards/rejected": -252.7429656982422, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.39, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.9078413895498612, |
| "learning_rate": 3.8318133624280046e-07, |
| "logits/chosen": 4.621513843536377, |
| "logits/rejected": 4.621513843536377, |
| "logps/chosen": -29423.666015625, |
| "logps/pi_response": -17608.6015625, |
| "logps/ref_response": -265.94757080078125, |
| "logps/rejected": -29423.666015625, |
| "loss": 0.519, |
| "rewards/accuracies": 0.03076923079788685, |
| "rewards/chosen": -292.3346862792969, |
| "rewards/margins": 6.455641710090276e-07, |
| "rewards/rejected": -292.3346862792969, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.46, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.7756777568132545, |
| "learning_rate": 3.316028034595861e-07, |
| "logits/chosen": 4.356642723083496, |
| "logits/rejected": 4.356642723083496, |
| "logps/chosen": -29171.62890625, |
| "logps/pi_response": -18937.943359375, |
| "logps/ref_response": -276.5423278808594, |
| "logps/rejected": -29171.62890625, |
| "loss": 0.5212, |
| "rewards/accuracies": 0.015384615398943424, |
| "rewards/chosen": -289.7953796386719, |
| "rewards/margins": 0.0, |
| "rewards/rejected": -289.7953796386719, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.52, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.9174650913563618, |
| "learning_rate": 2.7575199021178855e-07, |
| "logits/chosen": 4.428848743438721, |
| "logits/rejected": 4.428848743438721, |
| "logps/chosen": -27704.791015625, |
| "logps/pi_response": -18176.86328125, |
| "logps/ref_response": -271.93060302734375, |
| "logps/rejected": -27704.791015625, |
| "loss": 0.5132, |
| "rewards/accuracies": 0.01923076994717121, |
| "rewards/chosen": -275.2251281738281, |
| "rewards/margins": -9.97690108306415e-07, |
| "rewards/rejected": -275.2251281738281, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.58, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.9019855937057101, |
| "learning_rate": 2.1855294234408068e-07, |
| "logits/chosen": 4.3772735595703125, |
| "logits/rejected": 4.3772735595703125, |
| "logps/chosen": -29193.177734375, |
| "logps/pi_response": -17440.9921875, |
| "logps/ref_response": -266.1241455078125, |
| "logps/rejected": -29193.177734375, |
| "loss": 0.5231, |
| "rewards/accuracies": 0.03076923079788685, |
| "rewards/chosen": -290.08026123046875, |
| "rewards/margins": -2.9343825147520874e-08, |
| "rewards/rejected": -290.0802307128906, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.65, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.8179873354250534, |
| "learning_rate": 1.6300029195778453e-07, |
| "logits/chosen": 4.375646114349365, |
| "logits/rejected": 4.375646114349365, |
| "logps/chosen": -27328.33203125, |
| "logps/pi_response": -17462.0546875, |
| "logps/ref_response": -267.173828125, |
| "logps/rejected": -27328.33203125, |
| "loss": 0.5171, |
| "rewards/accuracies": 0.026923077180981636, |
| "rewards/chosen": -271.51708984375, |
| "rewards/margins": 1.613910427522569e-07, |
| "rewards/rejected": -271.51708984375, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.71, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.9895798769556228, |
| "learning_rate": 1.1200247470632392e-07, |
| "logits/chosen": 4.529591083526611, |
| "logits/rejected": 4.529591083526611, |
| "logps/chosen": -28326.220703125, |
| "logps/pi_response": -18734.5078125, |
| "logps/ref_response": -291.685791015625, |
| "logps/rejected": -28326.220703125, |
| "loss": 0.5103, |
| "rewards/accuracies": 0.04615384712815285, |
| "rewards/chosen": -281.44744873046875, |
| "rewards/margins": 1.540550869094659e-07, |
| "rewards/rejected": -281.44744873046875, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.78, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.9157342554933109, |
| "learning_rate": 6.822945986946385e-08, |
| "logits/chosen": 4.574192523956299, |
| "logits/rejected": 4.574192523956299, |
| "logps/chosen": -29500.658203125, |
| "logps/pi_response": -18083.599609375, |
| "logps/ref_response": -266.3084716796875, |
| "logps/rejected": -29500.658203125, |
| "loss": 0.5148, |
| "rewards/accuracies": 0.05384615436196327, |
| "rewards/chosen": -293.0876159667969, |
| "rewards/margins": 1.4085036355027114e-06, |
| "rewards/rejected": -293.0876159667969, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.84, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.932891201307216, |
| "learning_rate": 3.397296523427806e-08, |
| "logits/chosen": 4.706723213195801, |
| "logits/rejected": 4.706723213195801, |
| "logps/chosen": -26521.8828125, |
| "logps/pi_response": -15752.3818359375, |
| "logps/ref_response": -260.1748046875, |
| "logps/rejected": -26521.884765625, |
| "loss": 0.5218, |
| "rewards/accuracies": 0.04615384712815285, |
| "rewards/chosen": -263.5008544921875, |
| "rewards/margins": 8.876506853994215e-07, |
| "rewards/rejected": -263.5008544921875, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.91, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.7630353244282155, |
| "learning_rate": 1.1026475173977978e-08, |
| "logits/chosen": 4.662195205688477, |
| "logits/rejected": 4.662195205688477, |
| "logps/chosen": -29146.107421875, |
| "logps/pi_response": -18217.185546875, |
| "logps/ref_response": -266.84295654296875, |
| "logps/rejected": -29146.107421875, |
| "loss": 0.5144, |
| "rewards/accuracies": 0.03846153989434242, |
| "rewards/chosen": -289.6330261230469, |
| "rewards/margins": 5.868765029504175e-08, |
| "rewards/rejected": -289.6330261230469, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.97, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 0.7944892762549526, |
| "learning_rate": 5.913435276374834e-10, |
| "logits/chosen": 4.59439754486084, |
| "logits/rejected": 4.59439754486084, |
| "logps/chosen": -28846.330078125, |
| "logps/pi_response": -18351.314453125, |
| "logps/ref_response": -276.857666015625, |
| "logps/rejected": -28846.330078125, |
| "loss": 0.5191, |
| "rewards/accuracies": 0.03846153989434242, |
| "rewards/chosen": -286.6873779296875, |
| "rewards/margins": -8.803147579783399e-07, |
| "rewards/rejected": -286.6873779296875, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.99, |
| "step": 153, |
| "total_flos": 0.0, |
| "train_loss": 0.0, |
| "train_runtime": 0.0094, |
| "train_samples_per_second": 2122892.066, |
| "train_steps_per_second": 16240.124 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 153, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 200, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
