| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.0, |
| "eval_steps": 100, |
| "global_step": 1856, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 0.40908203125, |
| "epoch": 0.0053893829156561575, |
| "grad_norm": 7.759696960449219, |
| "learning_rate": 4.8387096774193546e-08, |
| "logits/chosen": -0.44672375660898817, |
| "logits/rejected": -0.39365523942633257, |
| "logps/chosen": -107.6, |
| "logps/rejected": -109.3, |
| "loss": 0.6923, |
| "mean_token_accuracy": 0.9019625961780549, |
| "num_tokens": 365562.0, |
| "rewards/accuracies": 0.328125, |
| "rewards/chosen": 0.00010342597961425782, |
| "rewards/margins": 0.0004913330078125, |
| "rewards/rejected": -0.0003886222839355469, |
| "step": 10 |
| }, |
| { |
| "entropy": 0.41064453125, |
| "epoch": 0.010778765831312315, |
| "grad_norm": 7.587465286254883, |
| "learning_rate": 1.0215053763440861e-07, |
| "logits/chosen": -0.3728776172134608, |
| "logits/rejected": -0.332460740198616, |
| "logps/chosen": -100.825, |
| "logps/rejected": -102.975, |
| "loss": 0.693, |
| "mean_token_accuracy": 0.9035390466451645, |
| "num_tokens": 720708.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.00041942596435546876, |
| "rewards/margins": -0.00022115707397460938, |
| "rewards/rejected": -0.00019679069519042968, |
| "step": 20 |
| }, |
| { |
| "entropy": 0.4173828125, |
| "epoch": 0.016168148746968473, |
| "grad_norm": 6.577247142791748, |
| "learning_rate": 1.5591397849462365e-07, |
| "logits/chosen": -0.472471438697433, |
| "logits/rejected": -0.39264794802016467, |
| "logps/chosen": -96.45, |
| "logps/rejected": -99.825, |
| "loss": 0.6929, |
| "mean_token_accuracy": 0.9041816651821136, |
| "num_tokens": 1075168.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.00027996301651000977, |
| "rewards/margins": -0.00011727809906005859, |
| "rewards/rejected": -0.00016357898712158204, |
| "step": 30 |
| }, |
| { |
| "entropy": 0.39150390625, |
| "epoch": 0.02155753166262463, |
| "grad_norm": 6.858296871185303, |
| "learning_rate": 2.0967741935483871e-07, |
| "logits/chosen": -0.3733890259921028, |
| "logits/rejected": -0.3552958369517544, |
| "logps/chosen": -99.3375, |
| "logps/rejected": -103.175, |
| "loss": 0.6929, |
| "mean_token_accuracy": 0.9075490444898605, |
| "num_tokens": 1443719.0, |
| "rewards/accuracies": 0.35, |
| "rewards/chosen": -0.0002532958984375, |
| "rewards/margins": 0.000262451171875, |
| "rewards/rejected": -0.0005157232284545898, |
| "step": 40 |
| }, |
| { |
| "entropy": 0.39990234375, |
| "epoch": 0.026946914578280787, |
| "grad_norm": 6.983979225158691, |
| "learning_rate": 2.6344086021505376e-07, |
| "logits/chosen": -0.3704425655981937, |
| "logits/rejected": -0.3571443883083806, |
| "logps/chosen": -89.5625, |
| "logps/rejected": -92.6, |
| "loss": 0.6927, |
| "mean_token_accuracy": 0.9039196908473969, |
| "num_tokens": 1782804.0, |
| "rewards/accuracies": 0.428125, |
| "rewards/chosen": -0.0011333465576171876, |
| "rewards/margins": 0.0005399942398071289, |
| "rewards/rejected": -0.0016736984252929688, |
| "step": 50 |
| }, |
| { |
| "entropy": 0.4013671875, |
| "epoch": 0.03233629749393695, |
| "grad_norm": 6.333939552307129, |
| "learning_rate": 3.172043010752688e-07, |
| "logits/chosen": -0.34019105871641864, |
| "logits/rejected": -0.3218748638672174, |
| "logps/chosen": -94.4875, |
| "logps/rejected": -93.575, |
| "loss": 0.6919, |
| "mean_token_accuracy": 0.9046859920024872, |
| "num_tokens": 2126112.0, |
| "rewards/accuracies": 0.490625, |
| "rewards/chosen": -0.0008403778076171875, |
| "rewards/margins": 0.0018994331359863282, |
| "rewards/rejected": -0.002740812301635742, |
| "step": 60 |
| }, |
| { |
| "entropy": 0.41484375, |
| "epoch": 0.0377256804095931, |
| "grad_norm": 7.4297566413879395, |
| "learning_rate": 3.7096774193548384e-07, |
| "logits/chosen": -0.3875610308110304, |
| "logits/rejected": -0.35152331612465404, |
| "logps/chosen": -97.375, |
| "logps/rejected": -99.3625, |
| "loss": 0.6914, |
| "mean_token_accuracy": 0.9037097364664077, |
| "num_tokens": 2470754.0, |
| "rewards/accuracies": 0.540625, |
| "rewards/chosen": -0.0022185802459716796, |
| "rewards/margins": 0.002935457229614258, |
| "rewards/rejected": -0.005158233642578125, |
| "step": 70 |
| }, |
| { |
| "entropy": 0.41259765625, |
| "epoch": 0.04311506332524926, |
| "grad_norm": 7.677794456481934, |
| "learning_rate": 4.247311827956989e-07, |
| "logits/chosen": -0.35103980570551985, |
| "logits/rejected": -0.3017016560657181, |
| "logps/chosen": -97.5, |
| "logps/rejected": -99.3875, |
| "loss": 0.6909, |
| "mean_token_accuracy": 0.9047763884067536, |
| "num_tokens": 2836116.0, |
| "rewards/accuracies": 0.571875, |
| "rewards/chosen": -0.004664897918701172, |
| "rewards/margins": 0.004430198669433593, |
| "rewards/rejected": -0.009100341796875, |
| "step": 80 |
| }, |
| { |
| "entropy": 0.41826171875, |
| "epoch": 0.04850444624090541, |
| "grad_norm": 6.765298366546631, |
| "learning_rate": 4.78494623655914e-07, |
| "logits/chosen": -0.4268290587468382, |
| "logits/rejected": -0.37336222025804355, |
| "logps/chosen": -108.175, |
| "logps/rejected": -111.6, |
| "loss": 0.6898, |
| "mean_token_accuracy": 0.9004561603069305, |
| "num_tokens": 3207605.0, |
| "rewards/accuracies": 0.684375, |
| "rewards/chosen": -0.008296608785167336, |
| "rewards/margins": 0.006721109163481742, |
| "rewards/rejected": -0.01501771821640432, |
| "setc/cal_net_lr": 3.375e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.052821920812129976, |
| "setc/logratio_margin_delta": -0.0747640784829855, |
| "setc/logratio_margin_vanilla": 0.746875, |
| "setc/logratio_margin_weighted": 0.6721109215170145, |
| "step": 90 |
| }, |
| { |
| "entropy": 0.44228515625, |
| "epoch": 0.05389382915656157, |
| "grad_norm": 7.393381595611572, |
| "learning_rate": 5.322580645161289e-07, |
| "logits/chosen": -0.4361902457762258, |
| "logits/rejected": -0.3870189698671949, |
| "logps/chosen": -101.75, |
| "logps/rejected": -109.4, |
| "loss": 0.6872, |
| "mean_token_accuracy": 0.9016273647546769, |
| "num_tokens": 3548981.0, |
| "rewards/accuracies": 0.728125, |
| "rewards/chosen": -0.013736124988645316, |
| "rewards/margins": 0.01212767151882872, |
| "rewards/rejected": -0.02586379610002041, |
| "setc/cal_net_lr": 0.00011666666666666668, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.053199329786002635, |
| "setc/logratio_margin_delta": -0.07141252495348453, |
| "setc/logratio_margin_vanilla": 1.2841796875, |
| "setc/logratio_margin_weighted": 1.2127671625465155, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.05389382915656157, |
| "eval_entropy": 0.4267578125, |
| "eval_logits/chosen": -0.450571413522897, |
| "eval_logits/rejected": -0.3940343719054589, |
| "eval_logps/chosen": -108.859375, |
| "eval_logps/rejected": -108.640625, |
| "eval_loss": 0.6871142387390137, |
| "eval_mean_token_accuracy": 0.9000339470803738, |
| "eval_num_tokens": 3548981.0, |
| "eval_rewards/accuracies": 0.671875, |
| "eval_rewards/chosen": -0.021255508123431355, |
| "eval_rewards/margins": 0.012425659064319916, |
| "eval_rewards/rejected": -0.03368116734782234, |
| "eval_runtime": 14.2001, |
| "eval_samples_per_second": 35.211, |
| "eval_setc/cal_net_lr": 0.00015833333333333327, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.04981097369454801, |
| "eval_setc/logratio_margin_delta": -0.06453109718859196, |
| "eval_setc/logratio_margin_vanilla": 1.3050537109375, |
| "eval_setc/logratio_margin_weighted": 1.240522613748908, |
| "eval_steps_per_second": 2.254, |
| "step": 100 |
| }, |
| { |
| "entropy": 0.42685546875, |
| "epoch": 0.059283212072217734, |
| "grad_norm": 7.357357501983643, |
| "learning_rate": 5.860215053763441e-07, |
| "logits/chosen": -0.43970988910262926, |
| "logits/rejected": -0.40844065752963693, |
| "logps/chosen": -107.05, |
| "logps/rejected": -113.025, |
| "loss": 0.6854, |
| "mean_token_accuracy": 0.899716067314148, |
| "num_tokens": 3902446.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": -0.02834852202795446, |
| "rewards/margins": 0.015831261326093228, |
| "rewards/rejected": -0.04417978236451745, |
| "setc/cal_net_lr": 0.0002, |
| "setc/credit_mean": 0.9999999970197677, |
| "setc/credit_std": 0.047479902021586896, |
| "setc/logratio_margin_delta": -0.10251835882663726, |
| "setc/logratio_margin_vanilla": 1.68564453125, |
| "setc/logratio_margin_weighted": 1.5831261724233627, |
| "step": 110 |
| }, |
| { |
| "entropy": 0.42177734375, |
| "epoch": 0.0646725949878739, |
| "grad_norm": 7.097802639007568, |
| "learning_rate": 6.397849462365591e-07, |
| "logits/chosen": -0.41605972462656393, |
| "logits/rejected": -0.3486092219206212, |
| "logps/chosen": -97.4, |
| "logps/rejected": -101.15, |
| "loss": 0.6811, |
| "mean_token_accuracy": 0.8998741269111633, |
| "num_tokens": 4248207.0, |
| "rewards/accuracies": 0.715625, |
| "rewards/chosen": -0.0486921863630414, |
| "rewards/margins": 0.02490431647747755, |
| "rewards/rejected": -0.07359650395810605, |
| "setc/cal_net_lr": 0.0002833333333333333, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.040153346955776215, |
| "setc/logratio_margin_delta": -0.10605261325836182, |
| "setc/logratio_margin_vanilla": 2.596484375, |
| "setc/logratio_margin_weighted": 2.4904317617416383, |
| "step": 120 |
| }, |
| { |
| "entropy": 0.4609375, |
| "epoch": 0.07006197790353004, |
| "grad_norm": 8.569380760192871, |
| "learning_rate": 6.935483870967742e-07, |
| "logits/chosen": -0.5107418885495224, |
| "logits/rejected": -0.46866435782064403, |
| "logps/chosen": -125.8, |
| "logps/rejected": -125.425, |
| "loss": 0.6705, |
| "mean_token_accuracy": 0.8874441146850586, |
| "num_tokens": 4601428.0, |
| "rewards/accuracies": 0.721875, |
| "rewards/chosen": -0.09584741443395614, |
| "rewards/margins": 0.04766115248203277, |
| "rewards/rejected": -0.14350856877863408, |
| "setc/cal_net_lr": 0.0003666666666666668, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.035092563927173616, |
| "setc/logratio_margin_delta": -0.18232210874557495, |
| "setc/logratio_margin_vanilla": 4.9484375, |
| "setc/logratio_margin_weighted": 4.766115391254425, |
| "step": 130 |
| }, |
| { |
| "entropy": 0.46455078125, |
| "epoch": 0.0754513608191862, |
| "grad_norm": 7.780826091766357, |
| "learning_rate": 7.473118279569892e-07, |
| "logits/chosen": -0.4342919440785252, |
| "logits/rejected": -0.42807028323097684, |
| "logps/chosen": -100.9, |
| "logps/rejected": -110.775, |
| "loss": 0.6591, |
| "mean_token_accuracy": 0.8856117874383926, |
| "num_tokens": 4920681.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": -0.1356092609465122, |
| "rewards/margins": 0.07387373577803373, |
| "rewards/rejected": -0.2094829984009266, |
| "setc/cal_net_lr": 0.00045, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.022299187257885933, |
| "setc/logratio_margin_delta": -0.09700125455856323, |
| "setc/logratio_margin_vanilla": 7.484375, |
| "setc/logratio_margin_weighted": 7.387373745441437, |
| "step": 140 |
| }, |
| { |
| "entropy": 0.45078125, |
| "epoch": 0.08084074373484236, |
| "grad_norm": 8.610173225402832, |
| "learning_rate": 8.010752688172043e-07, |
| "logits/chosen": -0.41425887318933086, |
| "logits/rejected": -0.3917996553785628, |
| "logps/chosen": -125.75, |
| "logps/rejected": -139.075, |
| "loss": 0.6385, |
| "mean_token_accuracy": 0.88282710313797, |
| "num_tokens": 5269262.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": -0.2560404367744923, |
| "rewards/margins": 0.1263975765556097, |
| "rewards/rejected": -0.3824380189180374, |
| "setc/cal_net_lr": 0.0005333333333333333, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.012131102010607719, |
| "setc/logratio_margin_delta": 0.07022677659988404, |
| "setc/logratio_margin_vanilla": 12.56953125, |
| "setc/logratio_margin_weighted": 12.639758026599884, |
| "step": 150 |
| }, |
| { |
| "entropy": 0.46650390625, |
| "epoch": 0.08623012665049852, |
| "grad_norm": 10.757467269897461, |
| "learning_rate": 8.548387096774193e-07, |
| "logits/chosen": -0.48324375921289925, |
| "logits/rejected": -0.4760727307021881, |
| "logps/chosen": -147.85, |
| "logps/rejected": -173.725, |
| "loss": 0.6143, |
| "mean_token_accuracy": 0.8614455878734588, |
| "num_tokens": 5635034.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": -0.46175644397735593, |
| "rewards/margins": 0.20489476919174193, |
| "rewards/rejected": -0.6666512250900268, |
| "setc/cal_net_lr": 0.0006166666666666666, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.01982680819928646, |
| "setc/logratio_margin_delta": 0.5832273483276367, |
| "setc/logratio_margin_vanilla": 19.90625, |
| "setc/logratio_margin_weighted": 20.489477348327636, |
| "step": 160 |
| }, |
| { |
| "entropy": 0.4853515625, |
| "epoch": 0.09161950956615468, |
| "grad_norm": 12.929532051086426, |
| "learning_rate": 9.086021505376343e-07, |
| "logits/chosen": -0.5325447226004508, |
| "logits/rejected": -0.533857912904264, |
| "logps/chosen": -177.75, |
| "logps/rejected": -207.0, |
| "loss": 0.5938, |
| "mean_token_accuracy": 0.8355448335409165, |
| "num_tokens": 5996408.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.7701589599251747, |
| "rewards/margins": 0.2884057696908712, |
| "rewards/rejected": -1.0585647374391556, |
| "setc/cal_net_lr": 0.0007000000000000002, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.04059908324852586, |
| "setc/logratio_margin_delta": 1.1937029838562012, |
| "setc/logratio_margin_vanilla": 27.646875, |
| "setc/logratio_margin_weighted": 28.8405779838562, |
| "step": 170 |
| }, |
| { |
| "entropy": 0.4525390625, |
| "epoch": 0.09700889248181083, |
| "grad_norm": 13.670575141906738, |
| "learning_rate": 9.623655913978494e-07, |
| "logits/chosen": -0.5174588059832541, |
| "logits/rejected": -0.4903172377790385, |
| "logps/chosen": -212.5, |
| "logps/rejected": -245.725, |
| "loss": 0.5927, |
| "mean_token_accuracy": 0.818328133225441, |
| "num_tokens": 6344423.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": -1.1680178940296173, |
| "rewards/margins": 0.3832601685076952, |
| "rewards/rejected": -1.5512780517339706, |
| "setc/cal_net_lr": 0.0007833333333333333, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06093887090682983, |
| "setc/logratio_margin_delta": 2.3182060956954955, |
| "setc/logratio_margin_vanilla": 36.0078125, |
| "setc/logratio_margin_weighted": 38.326018595695494, |
| "step": 180 |
| }, |
| { |
| "entropy": 0.4759765625, |
| "epoch": 0.10239827539746699, |
| "grad_norm": 13.61806583404541, |
| "learning_rate": 9.999920375234096e-07, |
| "logits/chosen": -0.5286533092264682, |
| "logits/rejected": -0.5037254571131352, |
| "logps/chosen": -254.55, |
| "logps/rejected": -306.6, |
| "loss": 0.5476, |
| "mean_token_accuracy": 0.8002152472734452, |
| "num_tokens": 6709991.0, |
| "rewards/accuracies": 0.715625, |
| "rewards/chosen": -1.5595814138650894, |
| "rewards/margins": 0.5580403164029122, |
| "rewards/rejected": -2.1176217913627626, |
| "setc/cal_net_lr": 0.0008666666666666668, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07659839615225791, |
| "setc/logratio_margin_delta": 3.3602832794189452, |
| "setc/logratio_margin_vanilla": 52.44375, |
| "setc/logratio_margin_weighted": 55.804033279418945, |
| "step": 190 |
| }, |
| { |
| "entropy": 0.43544921875, |
| "epoch": 0.10778765831312315, |
| "grad_norm": 14.541138648986816, |
| "learning_rate": 9.998504894389128e-07, |
| "logits/chosen": -0.527879368703732, |
| "logits/rejected": -0.5018783955560024, |
| "logps/chosen": -266.7, |
| "logps/rejected": -320.55, |
| "loss": 0.5664, |
| "mean_token_accuracy": 0.800949826836586, |
| "num_tokens": 7083816.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": -1.7819188117980957, |
| "rewards/margins": 0.5936955399811268, |
| "rewards/rejected": -2.3756143033504484, |
| "setc/cal_net_lr": 0.0009499999999999998, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0854849111288786, |
| "setc/logratio_margin_delta": 3.891430473327637, |
| "setc/logratio_margin_vanilla": 55.478125, |
| "setc/logratio_margin_weighted": 59.36955547332764, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.10778765831312315, |
| "eval_entropy": 0.44390869140625, |
| "eval_logits/chosen": -0.579585194957734, |
| "eval_logits/rejected": -0.5690945474827908, |
| "eval_logps/chosen": -275.25, |
| "eval_logps/rejected": -336.9375, |
| "eval_loss": 0.5417425036430359, |
| "eval_mean_token_accuracy": 0.7958512287586927, |
| "eval_num_tokens": 7083816.0, |
| "eval_rewards/accuracies": 0.755859375, |
| "eval_rewards/chosen": -1.8367619067430496, |
| "eval_rewards/margins": 0.6809160728007555, |
| "eval_rewards/rejected": -2.5176779851317406, |
| "eval_runtime": 14.1879, |
| "eval_samples_per_second": 35.241, |
| "eval_setc/cal_net_lr": 0.0009916666666666676, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.09134258469566703, |
| "eval_setc/logratio_margin_delta": 4.674029171466827, |
| "eval_setc/logratio_margin_vanilla": 64.115234375, |
| "eval_setc/logratio_margin_weighted": 68.78926354646683, |
| "eval_steps_per_second": 2.255, |
| "step": 200 |
| }, |
| { |
| "entropy": 0.4421875, |
| "epoch": 0.11317704122877931, |
| "grad_norm": 15.038735389709473, |
| "learning_rate": 9.995320550872586e-07, |
| "logits/chosen": -0.5982554625710975, |
| "logits/rejected": -0.5934455933800111, |
| "logps/chosen": -288.05, |
| "logps/rejected": -341.35, |
| "loss": 0.5767, |
| "mean_token_accuracy": 0.7952886909246445, |
| "num_tokens": 7430056.0, |
| "rewards/accuracies": 0.740625, |
| "rewards/chosen": -1.9779862701892852, |
| "rewards/margins": 0.5607165463268757, |
| "rewards/rejected": -2.538702827692032, |
| "setc/cal_net_lr": 0.0009994884161461147, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.09023903161287308, |
| "setc/logratio_margin_delta": 3.6450933456420898, |
| "setc/logratio_margin_vanilla": 52.4265625, |
| "setc/logratio_margin_weighted": 56.07165584564209, |
| "step": 210 |
| }, |
| { |
| "entropy": 0.42705078125, |
| "epoch": 0.11856642414443547, |
| "grad_norm": 16.99634552001953, |
| "learning_rate": 9.990368471554702e-07, |
| "logits/chosen": -0.5453219890997539, |
| "logits/rejected": -0.5525475843850532, |
| "logps/chosen": -261.55, |
| "logps/rejected": -330.25, |
| "loss": 0.5191, |
| "mean_token_accuracy": 0.8015528202056885, |
| "num_tokens": 7791142.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": -1.7827031493186951, |
| "rewards/margins": 0.7491906136274338, |
| "rewards/rejected": -2.531893861293793, |
| "setc/cal_net_lr": 0.0009992394884850806, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.09283662959933281, |
| "setc/logratio_margin_delta": 5.744063377380371, |
| "setc/logratio_margin_vanilla": 69.175, |
| "setc/logratio_margin_weighted": 74.91906337738037, |
| "step": 220 |
| }, |
| { |
| "entropy": 0.426953125, |
| "epoch": 0.12395580706009161, |
| "grad_norm": 13.820030212402344, |
| "learning_rate": 9.98365040886914e-07, |
| "logits/chosen": -0.5075058959811635, |
| "logits/rejected": -0.5316922545794671, |
| "logps/chosen": -297.45, |
| "logps/rejected": -381.2, |
| "loss": 0.5132, |
| "mean_token_accuracy": 0.8004782497882843, |
| "num_tokens": 8160190.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": -2.1069876670837404, |
| "rewards/margins": 0.8769942551851273, |
| "rewards/rejected": -2.9839819371700287, |
| "setc/cal_net_lr": 0.0009978552797828366, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.10098463930189609, |
| "setc/logratio_margin_delta": 6.56817512512207, |
| "setc/logratio_margin_vanilla": 81.13125, |
| "setc/logratio_margin_weighted": 87.69942512512208, |
| "step": 230 |
| }, |
| { |
| "entropy": 0.41240234375, |
| "epoch": 0.1293451899757478, |
| "grad_norm": 20.368221282958984, |
| "learning_rate": 9.975168740192852e-07, |
| "logits/chosen": -0.5192135050793214, |
| "logits/rejected": -0.5336463823173456, |
| "logps/chosen": -274.9, |
| "logps/rejected": -356.6, |
| "loss": 0.5139, |
| "mean_token_accuracy": 0.7949420899152756, |
| "num_tokens": 8525986.0, |
| "rewards/accuracies": 0.753125, |
| "rewards/chosen": -1.9865135312080384, |
| "rewards/margins": 0.8216024398803711, |
| "rewards/rejected": -2.8081159591674805, |
| "setc/cal_net_lr": 0.00099575444916116, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.1036558359861374, |
| "setc/logratio_margin_delta": 7.082119369506836, |
| "setc/logratio_margin_vanilla": 75.078125, |
| "setc/logratio_margin_weighted": 82.16024436950684, |
| "step": 240 |
| }, |
| { |
| "entropy": 0.4126953125, |
| "epoch": 0.13473457289140395, |
| "grad_norm": 32.17527770996094, |
| "learning_rate": 9.964926467004765e-07, |
| "logits/chosen": -0.5428571750679503, |
| "logits/rejected": -0.5702547242679802, |
| "logps/chosen": -319.9, |
| "logps/rejected": -424.2, |
| "loss": 0.4687, |
| "mean_token_accuracy": 0.782842355966568, |
| "num_tokens": 8896270.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": -2.4224733889102934, |
| "rewards/margins": 1.0895347505807877, |
| "rewards/rejected": -3.5120081663131715, |
| "setc/cal_net_lr": 0.000992940020593752, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.10739997066557408, |
| "setc/logratio_margin_delta": 8.453478050231933, |
| "setc/logratio_margin_vanilla": 100.5, |
| "setc/logratio_margin_weighted": 108.95347805023194, |
| "step": 250 |
| }, |
| { |
| "entropy": 0.40068359375, |
| "epoch": 0.14012395580706008, |
| "grad_norm": 26.627729415893555, |
| "learning_rate": 9.952927213823627e-07, |
| "logits/chosen": -0.4605946438072367, |
| "logits/rejected": -0.490033485518619, |
| "logps/chosen": -351.4, |
| "logps/rejected": -466.9, |
| "loss": 0.4693, |
| "mean_token_accuracy": 0.7776162534952163, |
| "num_tokens": 9245420.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": -2.833879363536835, |
| "rewards/margins": 1.184336504340172, |
| "rewards/rejected": -4.018215835094452, |
| "setc/cal_net_lr": 0.000989416045220137, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.10721651390194893, |
| "setc/logratio_margin_delta": 7.883653450012207, |
| "setc/logratio_margin_vanilla": 110.55, |
| "setc/logratio_margin_weighted": 118.4336534500122, |
| "step": 260 |
| }, |
| { |
| "entropy": 0.412109375, |
| "epoch": 0.14551333872271624, |
| "grad_norm": 35.465599060058594, |
| "learning_rate": 9.939175226925377e-07, |
| "logits/chosen": -0.5947771680223737, |
| "logits/rejected": -0.609633545923327, |
| "logps/chosen": -345.4, |
| "logps/rejected": -476.9, |
| "loss": 0.4478, |
| "mean_token_accuracy": 0.7697073727846145, |
| "num_tokens": 9603870.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -2.724973976612091, |
| "rewards/margins": 1.3795409053564072, |
| "rewards/rejected": -4.10451488494873, |
| "setc/cal_net_lr": 0.000985187595514384, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.10481685362756252, |
| "setc/logratio_margin_delta": 10.8415922164917, |
| "setc/logratio_margin_vanilla": 127.1125, |
| "setc/logratio_margin_weighted": 137.9540922164917, |
| "step": 270 |
| }, |
| { |
| "entropy": 0.402734375, |
| "epoch": 0.1509027216383724, |
| "grad_norm": 20.82551383972168, |
| "learning_rate": 9.92367537284046e-07, |
| "logits/chosen": -0.5548631230339891, |
| "logits/rejected": -0.5871126674101008, |
| "logps/chosen": -330.9, |
| "logps/rejected": -440.2, |
| "loss": 0.4953, |
| "mean_token_accuracy": 0.7791453748941422, |
| "num_tokens": 9933926.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": -2.6326347470283507, |
| "rewards/margins": 1.1489318370819093, |
| "rewards/rejected": -3.7815666437149047, |
| "setc/cal_net_lr": 0.0009802607579836912, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.10762366987764835, |
| "setc/logratio_margin_delta": 10.655691719055175, |
| "setc/logratio_margin_vanilla": 104.2375, |
| "setc/logratio_margin_weighted": 114.89319171905518, |
| "step": 280 |
| }, |
| { |
| "entropy": 0.3955078125, |
| "epoch": 0.15629210455402856, |
| "grad_norm": 18.485855102539062, |
| "learning_rate": 9.906433136631696e-07, |
| "logits/chosen": -0.543341739516008, |
| "logits/rejected": -0.5576040736941209, |
| "logps/chosen": -321.1, |
| "logps/rejected": -424.9, |
| "loss": 0.4656, |
| "mean_token_accuracy": 0.7913450181484223, |
| "num_tokens": 10300364.0, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -2.416798382997513, |
| "rewards/margins": 1.1477631881833077, |
| "rewards/rejected": -3.5645615816116334, |
| "setc/cal_net_lr": 0.0009746426244073601, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.1039271742105484, |
| "setc/logratio_margin_delta": 9.02632179260254, |
| "setc/logratio_margin_vanilla": 105.75, |
| "setc/logratio_margin_weighted": 114.77632179260254, |
| "step": 290 |
| }, |
| { |
| "entropy": 0.37470703125, |
| "epoch": 0.16168148746968472, |
| "grad_norm": 19.445205688476562, |
| "learning_rate": 9.887454619953203e-07, |
| "logits/chosen": -0.5234257126029256, |
| "logits/rejected": -0.5395859317035523, |
| "logps/chosen": -333.7, |
| "logps/rejected": -451.1, |
| "loss": 0.4567, |
| "mean_token_accuracy": 0.7912471622228623, |
| "num_tokens": 10652272.0, |
| "rewards/accuracies": 0.81875, |
| "rewards/chosen": -2.56951225399971, |
| "rewards/margins": 1.2171583086252213, |
| "rewards/rejected": -3.7866706013679505, |
| "setc/cal_net_lr": 0.0009683412816287581, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.09890042394399642, |
| "setc/logratio_margin_delta": 9.815832328796386, |
| "setc/logratio_margin_vanilla": 111.9, |
| "setc/logratio_margin_weighted": 121.71583232879638, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.16168148746968472, |
| "eval_entropy": 0.3961181640625, |
| "eval_logits/chosen": -0.5485536702445084, |
| "eval_logits/rejected": -0.5521437157571957, |
| "eval_logps/chosen": -344.875, |
| "eval_logps/rejected": -457.9375, |
| "eval_loss": 0.45288482308387756, |
| "eval_mean_token_accuracy": 0.7810891252011061, |
| "eval_num_tokens": 10652272.0, |
| "eval_rewards/accuracies": 0.810546875, |
| "eval_rewards/chosen": -2.6322504356503487, |
| "eval_rewards/margins": 1.2452996505890042, |
| "eval_rewards/rejected": -3.877550110220909, |
| "eval_runtime": 14.1906, |
| "eval_samples_per_second": 35.235, |
| "eval_setc/cal_net_lr": 0.000964790405843831, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.09871792490594089, |
| "eval_setc/logratio_margin_delta": 9.83822700381279, |
| "eval_setc/logratio_margin_vanilla": 115.9619140625, |
| "eval_setc/logratio_margin_weighted": 125.80014106631279, |
| "eval_steps_per_second": 2.255, |
| "step": 300 |
| }, |
| { |
| "entropy": 0.37734375, |
| "epoch": 0.16707087038534088, |
| "grad_norm": 25.576738357543945, |
| "learning_rate": 9.866746538891172e-07, |
| "logits/chosen": -0.5276412640451731, |
| "logits/rejected": -0.5328882291269553, |
| "logps/chosen": -361.3, |
| "logps/rejected": -468.8, |
| "loss": 0.5015, |
| "mean_token_accuracy": 0.7804528266191483, |
| "num_tokens": 11025201.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.8542094469070434, |
| "rewards/margins": 1.1937554821372032, |
| "rewards/rejected": -4.047964870929718, |
| "setc/cal_net_lr": 0.0009613657999149686, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.09489493407309055, |
| "setc/logratio_margin_delta": 11.188052368164062, |
| "setc/logratio_margin_vanilla": 108.1875, |
| "setc/logratio_margin_weighted": 119.37555236816407, |
| "step": 310 |
| }, |
| { |
| "entropy": 0.39609375, |
| "epoch": 0.17246025330099704, |
| "grad_norm": 25.457834243774414, |
| "learning_rate": 9.844316221587202e-07, |
| "logits/chosen": -0.547783353678369, |
| "logits/rejected": -0.5447233904757801, |
| "logps/chosen": -354.2, |
| "logps/rejected": -475.9, |
| "loss": 0.4576, |
| "mean_token_accuracy": 0.7729141265153885, |
| "num_tokens": 11378241.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": -2.7483198761940004, |
| "rewards/margins": 1.3043047487735748, |
| "rewards/rejected": -4.052624702453613, |
| "setc/cal_net_lr": 0.0009537262199008873, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.09067346304655075, |
| "setc/logratio_margin_delta": 9.902355837821961, |
| "setc/logratio_margin_vanilla": 120.528125, |
| "setc/logratio_margin_weighted": 130.43048083782196, |
| "step": 320 |
| }, |
| { |
| "entropy": 0.3943359375, |
| "epoch": 0.1778496362166532, |
| "grad_norm": 26.59535789489746, |
| "learning_rate": 9.82017160564499e-07, |
| "logits/chosen": -0.5416634787030847, |
| "logits/rejected": -0.5507818676121325, |
| "logps/chosen": -370.9, |
| "logps/rejected": -523.6, |
| "loss": 0.4163, |
| "mean_token_accuracy": 0.7765046328306198, |
| "num_tokens": 11731948.0, |
| "rewards/accuracies": 0.809375, |
| "rewards/chosen": -2.9148443579673766, |
| "rewards/margins": 1.6325711846351623, |
| "rewards/rejected": -4.547415578365326, |
| "setc/cal_net_lr": 0.0009454335381365463, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.09494004286825657, |
| "setc/logratio_margin_delta": 11.8196231842041, |
| "setc/logratio_margin_vanilla": 151.4375, |
| "setc/logratio_margin_weighted": 163.2571231842041, |
| "step": 330 |
| }, |
| { |
| "entropy": 0.35849609375, |
| "epoch": 0.18323901913230936, |
| "grad_norm": 32.58420944213867, |
| "learning_rate": 9.79432123532143e-07, |
| "logits/chosen": -0.49375742239443066, |
| "logits/rejected": -0.5324902808324306, |
| "logps/chosen": -475.7, |
| "logps/rejected": -628.1, |
| "loss": 0.4615, |
| "mean_token_accuracy": 0.7430515229701996, |
| "num_tokens": 12055728.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": -4.24499009847641, |
| "rewards/margins": 1.6139036893844605, |
| "rewards/rejected": -5.858893728256225, |
| "setc/cal_net_lr": 0.0009364996912584834, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.09690484963357449, |
| "setc/logratio_margin_delta": 11.415373611450196, |
| "setc/logratio_margin_vanilla": 149.975, |
| "setc/logratio_margin_weighted": 161.3903736114502, |
| "step": 340 |
| }, |
| { |
| "entropy": 0.3806640625, |
| "epoch": 0.18862840204796552, |
| "grad_norm": 27.812801361083984, |
| "learning_rate": 9.76677425850295e-07, |
| "logits/chosen": -0.6376107562818031, |
| "logits/rejected": -0.6418548650284829, |
| "logps/chosen": -465.6, |
| "logps/rejected": -605.1, |
| "loss": 0.4574, |
| "mean_token_accuracy": 0.7482803016901016, |
| "num_tokens": 12415175.0, |
| "rewards/accuracies": 0.778125, |
| "rewards/chosen": -3.9335139870643614, |
| "rewards/margins": 1.5329814106225967, |
| "rewards/rejected": -5.466495335102081, |
| "setc/cal_net_lr": 0.000926937538807931, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08994356840848923, |
| "setc/logratio_margin_delta": 11.082522773742676, |
| "setc/logratio_margin_vanilla": 142.215625, |
| "setc/logratio_margin_weighted": 153.29814777374267, |
| "step": 350 |
| }, |
| { |
| "entropy": 0.4140625, |
| "epoch": 0.19401778496362165, |
| "grad_norm": 21.86480712890625, |
| "learning_rate": 9.737540423468298e-07, |
| "logits/chosen": -0.6784440598334419, |
| "logits/rejected": -0.7089506769710789, |
| "logps/chosen": -366.3, |
| "logps/rejected": -501.6, |
| "loss": 0.4309, |
| "mean_token_accuracy": 0.7626632928848267, |
| "num_tokens": 12762716.0, |
| "rewards/accuracies": 0.834375, |
| "rewards/chosen": -2.877052891254425, |
| "rewards/margins": 1.4415714114904403, |
| "rewards/rejected": -4.318624341487885, |
| "setc/cal_net_lr": 0.0009167608447205565, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08553993590176105, |
| "setc/logratio_margin_delta": 10.219639015197753, |
| "setc/logratio_margin_vanilla": 133.9375, |
| "setc/logratio_margin_weighted": 144.15713901519774, |
| "step": 360 |
| }, |
| { |
| "entropy": 0.43505859375, |
| "epoch": 0.1994071678792778, |
| "grad_norm": 38.346282958984375, |
| "learning_rate": 9.706630075438816e-07, |
| "logits/chosen": -0.641167718623423, |
| "logits/rejected": -0.6645962228342419, |
| "logps/chosen": -407.6, |
| "logps/rejected": -551.5, |
| "loss": 0.4323, |
| "mean_token_accuracy": 0.7579832553863526, |
| "num_tokens": 13126907.0, |
| "rewards/accuracies": 0.815625, |
| "rewards/chosen": -3.3240499973297117, |
| "rewards/margins": 1.5297267407178878, |
| "rewards/rejected": -4.8537767171859745, |
| "setc/cal_net_lr": 0.0009059842575144065, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08733033128082753, |
| "setc/logratio_margin_delta": 11.610177040100098, |
| "setc/logratio_margin_vanilla": 141.3625, |
| "setc/logratio_margin_weighted": 152.9726770401001, |
| "step": 370 |
| }, |
| { |
| "entropy": 0.44599609375, |
| "epoch": 0.20479655079493397, |
| "grad_norm": 23.460803985595703, |
| "learning_rate": 9.67405415291751e-07, |
| "logits/chosen": -0.7566490767380819, |
| "logits/rejected": -0.7921768754823747, |
| "logps/chosen": -448.3, |
| "logps/rejected": -605.6, |
| "loss": 0.4256, |
| "mean_token_accuracy": 0.7468535989522934, |
| "num_tokens": 13476130.0, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -3.7169021725654603, |
| "rewards/margins": 1.668231150507927, |
| "rewards/rejected": -5.385133290290833, |
| "setc/cal_net_lr": 0.0008946232892045628, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08953866809606552, |
| "setc/logratio_margin_delta": 11.298118209838867, |
| "setc/logratio_margin_vanilla": 155.525, |
| "setc/logratio_margin_weighted": 166.82311820983887, |
| "step": 380 |
| }, |
| { |
| "entropy": 0.4015625, |
| "epoch": 0.21018593371059013, |
| "grad_norm": 37.17218780517578, |
| "learning_rate": 9.639824183818136e-07, |
| "logits/chosen": -0.6983920089322826, |
| "logits/rejected": -0.7127737130064461, |
| "logps/chosen": -405.7, |
| "logps/rejected": -581.9, |
| "loss": 0.3994, |
| "mean_token_accuracy": 0.7641981273889542, |
| "num_tokens": 13824865.0, |
| "rewards/accuracies": 0.8375, |
| "rewards/chosen": -3.3119530200958254, |
| "rewards/margins": 1.9102674454450608, |
| "rewards/rejected": -5.222220623493195, |
| "setc/cal_net_lr": 0.0008826942929748675, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08975566849112511, |
| "setc/logratio_margin_delta": 13.101746749877929, |
| "setc/logratio_margin_vanilla": 177.925, |
| "setc/logratio_margin_weighted": 191.02674674987793, |
| "step": 390 |
| }, |
| { |
| "entropy": 0.39140625, |
| "epoch": 0.2155753166262463, |
| "grad_norm": 31.108200073242188, |
| "learning_rate": 9.603952281385731e-07, |
| "logits/chosen": -0.6745283746311073, |
| "logits/rejected": -0.7180434918542554, |
| "logps/chosen": -434.1, |
| "logps/rejected": -567.2, |
| "loss": 0.4993, |
| "mean_token_accuracy": 0.7623392403125763, |
| "num_tokens": 14186289.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -3.6153177618980408, |
| "rewards/margins": 1.431649774312973, |
| "rewards/rejected": -5.046967601776123, |
| "setc/cal_net_lr": 0.0008702144396388513, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0924822997301817, |
| "setc/logratio_margin_delta": 9.014981460571288, |
| "setc/logratio_margin_vanilla": 134.15, |
| "setc/logratio_margin_weighted": 143.16498146057128, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.2155753166262463, |
| "eval_entropy": 0.39117431640625, |
| "eval_logits/chosen": -0.7596684509609115, |
| "eval_logits/rejected": -0.7672941152249027, |
| "eval_logps/chosen": -379.0625, |
| "eval_logps/rejected": -529.625, |
| "eval_loss": 0.403584361076355, |
| "eval_mean_token_accuracy": 0.7722007241100073, |
| "eval_num_tokens": 14186289.0, |
| "eval_rewards/accuracies": 0.833984375, |
| "eval_rewards/chosen": -2.980880632996559, |
| "eval_rewards/margins": 1.6366331987082958, |
| "eval_rewards/rejected": -4.617513798177242, |
| "eval_runtime": 14.2806, |
| "eval_samples_per_second": 35.013, |
| "eval_setc/cal_net_lr": 0.0008634697702573876, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.08779020188376307, |
| "eval_setc/logratio_margin_delta": 11.151197791099548, |
| "eval_setc/logratio_margin_vanilla": 153.91796875, |
| "eval_setc/logratio_margin_weighted": 165.06916654109955, |
| "eval_steps_per_second": 2.241, |
| "step": 400 |
| }, |
| { |
| "entropy": 0.3703125, |
| "epoch": 0.22096469954190245, |
| "grad_norm": 24.23314094543457, |
| "learning_rate": 9.566451139909988e-07, |
| "logits/chosen": -0.6472189487288608, |
| "logits/rejected": -0.6706296399402472, |
| "logps/chosen": -380.5, |
| "logps/rejected": -509.3, |
| "loss": 0.4374, |
| "mean_token_accuracy": 0.7814467757940292, |
| "num_tokens": 14563626.0, |
| "rewards/accuracies": 0.809375, |
| "rewards/chosen": -3.0536776065826414, |
| "rewards/margins": 1.3719112485647202, |
| "rewards/rejected": -4.425588870048523, |
| "setc/cal_net_lr": 0.0008572016929237579, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08474965952336788, |
| "setc/logratio_margin_delta": 9.878629875183105, |
| "setc/logratio_margin_vanilla": 127.3125, |
| "setc/logratio_margin_weighted": 137.1911298751831, |
| "step": 410 |
| }, |
| { |
| "entropy": 0.37265625, |
| "epoch": 0.22635408245755861, |
| "grad_norm": 28.370731353759766, |
| "learning_rate": 9.527334030233028e-07, |
| "logits/chosen": -0.6874033644949044, |
| "logits/rejected": -0.6702958053090058, |
| "logps/chosen": -365.85, |
| "logps/rejected": -494.9, |
| "loss": 0.486, |
| "mean_token_accuracy": 0.7773175150156021, |
| "num_tokens": 14919543.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": -2.9779480576515196, |
| "rewards/margins": 1.3624024033546447, |
| "rewards/rejected": -4.340350472927094, |
| "setc/cal_net_lr": 0.0008436747836132224, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08348823450505734, |
| "setc/logratio_margin_delta": 11.090241241455079, |
| "setc/logratio_margin_vanilla": 125.15, |
| "setc/logratio_margin_weighted": 136.24024124145507, |
| "step": 420 |
| }, |
| { |
| "entropy": 0.36806640625, |
| "epoch": 0.23174346537321477, |
| "grad_norm": 20.790348052978516, |
| "learning_rate": 9.486614795053136e-07, |
| "logits/chosen": -0.687221071504475, |
| "logits/rejected": -0.6840851017343192, |
| "logps/chosen": -355.1, |
| "logps/rejected": -485.7, |
| "loss": 0.4321, |
| "mean_token_accuracy": 0.785921522974968, |
| "num_tokens": 15277577.0, |
| "rewards/accuracies": 0.825, |
| "rewards/chosen": -2.7120434999465943, |
| "rewards/margins": 1.4238930448889733, |
| "rewards/rejected": -4.135936522483826, |
| "setc/cal_net_lr": 0.0008296531825858478, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08098721131682396, |
| "setc/logratio_margin_delta": 11.239310264587402, |
| "setc/logratio_margin_vanilla": 131.15, |
| "setc/logratio_margin_weighted": 142.3893102645874, |
| "step": 430 |
| }, |
| { |
| "entropy": 0.36962890625, |
| "epoch": 0.23713284828887093, |
| "grad_norm": 29.099376678466797, |
| "learning_rate": 9.444307844026128e-07, |
| "logits/chosen": -0.7001576273000311, |
| "logits/rejected": -0.7564168572512255, |
| "logps/chosen": -394.0, |
| "logps/rejected": -555.2, |
| "loss": 0.4012, |
| "mean_token_accuracy": 0.777391391992569, |
| "num_tokens": 15614929.0, |
| "rewards/accuracies": 0.834375, |
| "rewards/chosen": -3.16938259601593, |
| "rewards/margins": 1.7262235552072525, |
| "rewards/rejected": -4.8956061720848085, |
| "setc/cal_net_lr": 0.0008151570727884652, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08620954863727093, |
| "setc/logratio_margin_delta": 14.22236442565918, |
| "setc/logratio_margin_vanilla": 158.4, |
| "setc/logratio_margin_weighted": 172.62236442565919, |
| "step": 440 |
| }, |
| { |
| "entropy": 0.35576171875, |
| "epoch": 0.2425222312045271, |
| "grad_norm": 36.5993537902832, |
| "learning_rate": 9.400428148666088e-07, |
| "logits/chosen": -0.7156464373663424, |
| "logits/rejected": -0.7045427588458122, |
| "logps/chosen": -444.4, |
| "logps/rejected": -642.3, |
| "loss": 0.4123, |
| "mean_token_accuracy": 0.7572789788246155, |
| "num_tokens": 15977200.0, |
| "rewards/accuracies": 0.821875, |
| "rewards/chosen": -3.769306206703186, |
| "rewards/margins": 2.0929836332798004, |
| "rewards/rejected": -5.86228985786438, |
| "setc/cal_net_lr": 0.0008002073201844367, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08339687958359718, |
| "setc/logratio_margin_delta": 14.948371696472169, |
| "setc/logratio_margin_vanilla": 194.35, |
| "setc/logratio_margin_weighted": 209.29837169647217, |
| "step": 450 |
| }, |
| { |
| "entropy": 0.34697265625, |
| "epoch": 0.24791161412018323, |
| "grad_norm": 21.59111785888672, |
| "learning_rate": 9.354991237047272e-07, |
| "logits/chosen": -0.6320623481408233, |
| "logits/rejected": -0.6675801158708582, |
| "logps/chosen": -436.2, |
| "logps/rejected": -619.0, |
| "loss": 0.3781, |
| "mean_token_accuracy": 0.7620709240436554, |
| "num_tokens": 16318118.0, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -3.739296352863312, |
| "rewards/margins": 1.971435186266899, |
| "rewards/rejected": -5.710731565952301, |
| "setc/cal_net_lr": 0.0007848254437188073, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08574292548000813, |
| "setc/logratio_margin_delta": 13.068524169921876, |
| "setc/logratio_margin_vanilla": 184.075, |
| "setc/logratio_margin_weighted": 197.14352416992188, |
| "step": 460 |
| }, |
| { |
| "entropy": 0.35234375, |
| "epoch": 0.2533009970358394, |
| "grad_norm": 39.83766555786133, |
| "learning_rate": 9.308013188309052e-07, |
| "logits/chosen": -0.5782339558042321, |
| "logits/rejected": -0.5778674119896376, |
| "logps/chosen": -449.1, |
| "logps/rejected": -629.1, |
| "loss": 0.4338, |
| "mean_token_accuracy": 0.7699876993894577, |
| "num_tokens": 16695798.0, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -3.8279688358306885, |
| "rewards/margins": 1.9190792471170426, |
| "rewards/rejected": -5.747048032283783, |
| "setc/cal_net_lr": 0.0007690335843435461, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08457843437790871, |
| "setc/logratio_margin_delta": 15.239182376861573, |
| "setc/logratio_margin_vanilla": 176.66875, |
| "setc/logratio_margin_weighted": 191.90793237686157, |
| "step": 470 |
| }, |
| { |
| "entropy": 0.37353515625, |
| "epoch": 0.2586903799514956, |
| "grad_norm": 61.235111236572266, |
| "learning_rate": 9.259510626965874e-07, |
| "logits/chosen": -0.6445287935159719, |
| "logits/rejected": -0.6687137779580832, |
| "logps/chosen": -441.0, |
| "logps/rejected": -700.9, |
| "loss": 0.3949, |
| "mean_token_accuracy": 0.7573456019163132, |
| "num_tokens": 17051928.0, |
| "rewards/accuracies": 0.846875, |
| "rewards/chosen": -3.6645211458206175, |
| "rewards/margins": 2.796916735172272, |
| "rewards/rejected": -6.46143786907196, |
| "setc/cal_net_lr": 0.0007528544731474588, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08809281550347806, |
| "setc/logratio_margin_delta": 17.716680145263673, |
| "setc/logratio_margin_vanilla": 261.975, |
| "setc/logratio_margin_weighted": 279.69168014526366, |
| "step": 480 |
| }, |
| { |
| "entropy": 0.3984375, |
| "epoch": 0.26407976286715173, |
| "grad_norm": 51.56365966796875, |
| "learning_rate": 9.209500717024184e-07, |
| "logits/chosen": -0.5772588425384099, |
| "logits/rejected": -0.5829906453335669, |
| "logps/chosen": -433.8, |
| "logps/rejected": -669.7, |
| "loss": 0.3728, |
| "mean_token_accuracy": 0.7556526750326157, |
| "num_tokens": 17399473.0, |
| "rewards/accuracies": 0.840625, |
| "rewards/chosen": -3.6556755781173704, |
| "rewards/margins": 2.525260826945305, |
| "rewards/rejected": -6.180936527252197, |
| "setc/cal_net_lr": 0.0007363113986366443, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08914664313197136, |
| "setc/logratio_margin_delta": 16.12609519958496, |
| "setc/logratio_margin_vanilla": 236.4, |
| "setc/logratio_margin_weighted": 252.52609519958497, |
| "step": 490 |
| }, |
| { |
| "entropy": 0.3970703125, |
| "epoch": 0.2694691457828079, |
| "grad_norm": 41.8188591003418, |
| "learning_rate": 9.158001155908463e-07, |
| "logits/chosen": -0.5640562576190764, |
| "logits/rejected": -0.5751849485254861, |
| "logps/chosen": -457.0, |
| "logps/rejected": -668.7, |
| "loss": 0.4275, |
| "mean_token_accuracy": 0.7455924898386002, |
| "num_tokens": 17747161.0, |
| "rewards/accuracies": 0.83125, |
| "rewards/chosen": -3.902832806110382, |
| "rewards/margins": 2.2202740639448164, |
| "rewards/rejected": -6.123106646537781, |
| "setc/cal_net_lr": 0.0007194281732126001, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0880829505622387, |
| "setc/logratio_margin_delta": 13.539912796020507, |
| "setc/logratio_margin_vanilla": 208.4875, |
| "setc/logratio_margin_weighted": 222.0274127960205, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.2694691457828079, |
| "eval_entropy": 0.40399169921875, |
| "eval_logits/chosen": -0.592532312686467, |
| "eval_logits/rejected": -0.605988185624285, |
| "eval_logps/chosen": -462.9375, |
| "eval_logps/rejected": -684.0, |
| "eval_loss": 0.35599419474601746, |
| "eval_mean_token_accuracy": 0.7497607115656137, |
| "eval_num_tokens": 17747161.0, |
| "eval_rewards/accuracies": 0.849609375, |
| "eval_rewards/chosen": -3.872654564678669, |
| "eval_rewards/margins": 2.3638561107218266, |
| "eval_rewards/rejected": -6.236510649323463, |
| "eval_runtime": 14.2091, |
| "eval_samples_per_second": 35.189, |
| "eval_setc/cal_net_lr": 0.000710449074239924, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.08703692350536585, |
| "eval_setc/logratio_margin_delta": 14.291290760040283, |
| "eval_setc/logratio_margin_vanilla": 222.796875, |
| "eval_setc/logratio_margin_weighted": 237.08816576004028, |
| "eval_steps_per_second": 2.252, |
| "step": 500 |
| }, |
| { |
| "entropy": 0.3931640625, |
| "epoch": 0.274858528698464, |
| "grad_norm": 24.2525577545166, |
| "learning_rate": 9.105030168198502e-07, |
| "logits/chosen": -0.6082334317471639, |
| "logits/rejected": -0.6164790579325305, |
| "logps/chosen": -376.2, |
| "logps/rejected": -538.2, |
| "loss": 0.4458, |
| "mean_token_accuracy": 0.7635719299316406, |
| "num_tokens": 18074898.0, |
| "rewards/accuracies": 0.809375, |
| "rewards/chosen": -3.032584583759308, |
| "rewards/margins": 1.7272971540689468, |
| "rewards/rejected": -4.7598817348480225, |
| "setc/cal_net_lr": 0.0007022290988962156, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08350460566580295, |
| "setc/logratio_margin_delta": 12.80472412109375, |
| "setc/logratio_margin_vanilla": 159.925, |
| "setc/logratio_margin_weighted": 172.72972412109374, |
| "step": 510 |
| }, |
| { |
| "entropy": 0.418359375, |
| "epoch": 0.28024791161412016, |
| "grad_norm": 49.45343017578125, |
| "learning_rate": 9.050606499180102e-07, |
| "logits/chosen": -0.6519357377370888, |
| "logits/rejected": -0.6423741925659583, |
| "logps/chosen": -402.5, |
| "logps/rejected": -578.4, |
| "loss": 0.4126, |
| "mean_token_accuracy": 0.7656619876623154, |
| "num_tokens": 18445858.0, |
| "rewards/accuracies": 0.834375, |
| "rewards/chosen": -3.202549707889557, |
| "rewards/margins": 1.860961028933525, |
| "rewards/rejected": -5.063510704040527, |
| "setc/cal_net_lr": 0.0006847389323470055, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0847404919564724, |
| "setc/logratio_margin_delta": 11.646109008789063, |
| "setc/logratio_margin_vanilla": 174.45, |
| "setc/logratio_margin_weighted": 186.09610900878906, |
| "step": 520 |
| }, |
| { |
| "entropy": 0.43330078125, |
| "epoch": 0.2856372945297763, |
| "grad_norm": 29.773883819580078, |
| "learning_rate": 8.994749408211536e-07, |
| "logits/chosen": -0.7015092396795138, |
| "logits/rejected": -0.7123617485729619, |
| "logps/chosen": -520.0, |
| "logps/rejected": -806.7, |
| "loss": 0.3962, |
| "mean_token_accuracy": 0.729384770989418, |
| "num_tokens": 18791922.0, |
| "rewards/accuracies": 0.815625, |
| "rewards/chosen": -4.475372099876404, |
| "rewards/margins": 3.0225441813468934, |
| "rewards/rejected": -7.497916245460511, |
| "setc/cal_net_lr": 0.0006669828492279234, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08948785215616226, |
| "setc/logratio_margin_delta": 17.254426956176758, |
| "setc/logratio_margin_vanilla": 285.0, |
| "setc/logratio_margin_weighted": 302.25442695617676, |
| "step": 530 |
| }, |
| { |
| "entropy": 0.4228515625, |
| "epoch": 0.2910266774454325, |
| "grad_norm": 21.04458236694336, |
| "learning_rate": 8.937478661908069e-07, |
| "logits/chosen": -0.7061397320826232, |
| "logits/rejected": -0.7431562334397835, |
| "logps/chosen": -449.0, |
| "logps/rejected": -646.0, |
| "loss": 0.3719, |
| "mean_token_accuracy": 0.7576641947031021, |
| "num_tokens": 19156930.0, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -3.6808414697647094, |
| "rewards/margins": 2.149125117063522, |
| "rewards/rejected": -5.829966616630554, |
| "setc/cal_net_lr": 0.0006489864079670575, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08795219510793686, |
| "setc/logratio_margin_delta": 14.512511444091796, |
| "setc/logratio_margin_vanilla": 200.4, |
| "setc/logratio_margin_weighted": 214.9125114440918, |
| "step": 540 |
| }, |
| { |
| "entropy": 0.424609375, |
| "epoch": 0.29641606036108864, |
| "grad_norm": 67.26245880126953, |
| "learning_rate": 8.878814527146985e-07, |
| "logits/chosen": -0.6747327955549893, |
| "logits/rejected": -0.729661536184305, |
| "logps/chosen": -396.5, |
| "logps/rejected": -612.5, |
| "loss": 0.3854, |
| "mean_token_accuracy": 0.7647815823554993, |
| "num_tokens": 19505372.0, |
| "rewards/accuracies": 0.8375, |
| "rewards/chosen": -3.2515084028244017, |
| "rewards/margins": 2.2733805954456328, |
| "rewards/rejected": -5.52488911151886, |
| "setc/cal_net_lr": 0.0006307755129683639, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.09044467471539974, |
| "setc/logratio_margin_delta": 15.188063812255859, |
| "setc/logratio_margin_vanilla": 212.15, |
| "setc/logratio_margin_weighted": 227.33806381225585, |
| "step": 550 |
| }, |
| { |
| "entropy": 0.410546875, |
| "epoch": 0.3018054432767448, |
| "grad_norm": 56.778804779052734, |
| "learning_rate": 8.818777763895585e-07, |
| "logits/chosen": -0.6860816122826272, |
| "logits/rejected": -0.6895918590666351, |
| "logps/chosen": -430.6, |
| "logps/rejected": -624.7, |
| "loss": 0.3921, |
| "mean_token_accuracy": 0.7563153892755509, |
| "num_tokens": 19863864.0, |
| "rewards/accuracies": 0.83125, |
| "rewards/chosen": -3.628837537765503, |
| "rewards/margins": 2.037660950422287, |
| "rewards/rejected": -5.666498494148255, |
| "setc/cal_net_lr": 0.0006123763773243994, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08885525353252888, |
| "setc/logratio_margin_delta": 13.316099929809571, |
| "setc/logratio_margin_vanilla": 190.45, |
| "setc/logratio_margin_weighted": 203.76609992980957, |
| "step": 560 |
| }, |
| { |
| "entropy": 0.3953125, |
| "epoch": 0.30719482619240096, |
| "grad_norm": 42.43838119506836, |
| "learning_rate": 8.75738961786467e-07, |
| "logits/chosen": -0.688892356215907, |
| "logits/rejected": -0.7157972843996309, |
| "logps/chosen": -461.9, |
| "logps/rejected": -660.4, |
| "loss": 0.4452, |
| "mean_token_accuracy": 0.7513888716697693, |
| "num_tokens": 20215920.0, |
| "rewards/accuracies": 0.803125, |
| "rewards/chosen": -3.9202752113342285, |
| "rewards/margins": 2.134818637371063, |
| "rewards/rejected": -6.055093717575073, |
| "setc/cal_net_lr": 0.0005938154850847184, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08897339478135109, |
| "setc/logratio_margin_delta": 14.231868362426757, |
| "setc/logratio_margin_vanilla": 199.25, |
| "setc/logratio_margin_weighted": 213.48186836242675, |
| "step": 570 |
| }, |
| { |
| "entropy": 0.3853515625, |
| "epoch": 0.3125842091080571, |
| "grad_norm": 39.56584167480469, |
| "learning_rate": 8.694671812990155e-07, |
| "logits/chosen": -0.643150039587863, |
| "logits/rejected": -0.6802687248343776, |
| "logps/chosen": -361.9, |
| "logps/rejected": -505.1, |
| "loss": 0.4191, |
| "mean_token_accuracy": 0.7855405032634735, |
| "num_tokens": 20574156.0, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -2.867064654827118, |
| "rewards/margins": 1.4799500912427903, |
| "rewards/rejected": -4.347014737129212, |
| "setc/cal_net_lr": 0.0005751195531342541, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08569788709282875, |
| "setc/logratio_margin_delta": 10.945015716552735, |
| "setc/logratio_margin_vanilla": 137.05, |
| "setc/logratio_margin_weighted": 147.99501571655273, |
| "step": 580 |
| }, |
| { |
| "entropy": 0.41494140625, |
| "epoch": 0.3179735920237133, |
| "grad_norm": 18.509885787963867, |
| "learning_rate": 8.630646543745433e-07, |
| "logits/chosen": -0.7564691341004269, |
| "logits/rejected": -0.7964878528594659, |
| "logps/chosen": -466.7, |
| "logps/rejected": -751.8, |
| "loss": 0.3255, |
| "mean_token_accuracy": 0.7475934147834777, |
| "num_tokens": 20942542.0, |
| "rewards/accuracies": 0.85, |
| "rewards/chosen": -3.9546597361564637, |
| "rewards/margins": 3.00535169839859, |
| "rewards/rejected": -6.960011458396911, |
| "setc/cal_net_lr": 0.000556315492736548, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08982903957366943, |
| "setc/logratio_margin_delta": 20.885171508789064, |
| "setc/logratio_margin_vanilla": 279.65, |
| "setc/logratio_margin_weighted": 300.53517150878906, |
| "step": 590 |
| }, |
| { |
| "entropy": 0.405859375, |
| "epoch": 0.32336297493936944, |
| "grad_norm": 26.894384384155273, |
| "learning_rate": 8.565336467287235e-07, |
| "logits/chosen": -0.7470103611470064, |
| "logits/rejected": -0.7722877603949199, |
| "logps/chosen": -531.8, |
| "logps/rejected": -820.6, |
| "loss": 0.3808, |
| "mean_token_accuracy": 0.7415017306804657, |
| "num_tokens": 21291866.0, |
| "rewards/accuracies": 0.834375, |
| "rewards/chosen": -4.668173170089721, |
| "rewards/margins": 3.097328555583954, |
| "rewards/rejected": -7.765501689910889, |
| "setc/cal_net_lr": 0.0005374303707971904, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08936081901192665, |
| "setc/logratio_margin_delta": 19.732860565185547, |
| "setc/logratio_margin_vanilla": 290.0, |
| "setc/logratio_margin_weighted": 309.73286056518555, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.32336297493936944, |
| "eval_entropy": 0.4246826171875, |
| "eval_logits/chosen": -0.7950303357768591, |
| "eval_logits/rejected": -0.809009124352729, |
| "eval_logps/chosen": -570.9375, |
| "eval_logps/rejected": -884.625, |
| "eval_loss": 0.3066518008708954, |
| "eval_mean_token_accuracy": 0.726261779665947, |
| "eval_num_tokens": 21291866.0, |
| "eval_rewards/accuracies": 0.875, |
| "eval_rewards/chosen": -5.0408351719379425, |
| "eval_rewards/margins": 3.342687487602234, |
| "eval_rewards/rejected": -8.383522659540176, |
| "eval_runtime": 14.2973, |
| "eval_samples_per_second": 34.972, |
| "eval_setc/cal_net_lr": 0.0005274940322432844, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.09106356324627995, |
| "eval_setc/logratio_margin_delta": 19.50989866256714, |
| "eval_setc/logratio_margin_vanilla": 315.8125, |
| "eval_setc/logratio_margin_weighted": 335.32239866256714, |
| "eval_steps_per_second": 2.238, |
| "step": 600 |
| }, |
| { |
| "entropy": 0.43974609375, |
| "epoch": 0.3287523578550256, |
| "grad_norm": 41.537105560302734, |
| "learning_rate": 8.498764695437746e-07, |
| "logits/chosen": -0.7988740761782873, |
| "logits/rejected": -0.7901971178299118, |
| "logps/chosen": -550.8, |
| "logps/rejected": -910.0, |
| "loss": 0.3901, |
| "mean_token_accuracy": 0.7285310417413712, |
| "num_tokens": 21645741.0, |
| "rewards/accuracies": 0.8375, |
| "rewards/chosen": -4.841351461410523, |
| "rewards/margins": 3.7519346356391905, |
| "rewards/rejected": -8.593285942077637, |
| "setc/cal_net_lr": 0.0005184913709032242, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08994946219027042, |
| "setc/logratio_margin_delta": 18.493466186523438, |
| "setc/logratio_margin_vanilla": 356.7, |
| "setc/logratio_margin_weighted": 375.19346618652344, |
| "step": 610 |
| }, |
| { |
| "entropy": 0.43916015625, |
| "epoch": 0.33414174077068176, |
| "grad_norm": 42.426490783691406, |
| "learning_rate": 8.430954786505839e-07, |
| "logits/chosen": -0.8440707111321254, |
| "logits/rejected": -0.8533261529276125, |
| "logps/chosen": -431.9, |
| "logps/rejected": -656.1, |
| "loss": 0.3473, |
| "mean_token_accuracy": 0.7551120847463608, |
| "num_tokens": 21998359.0, |
| "rewards/accuracies": 0.878125, |
| "rewards/chosen": -3.5407957673072814, |
| "rewards/margins": 2.3734045028686523, |
| "rewards/rejected": -5.914200258255005, |
| "setc/cal_net_lr": 0.0004995257541945978, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08794644176959991, |
| "setc/logratio_margin_delta": 15.790457153320313, |
| "setc/logratio_margin_vanilla": 221.55, |
| "setc/logratio_margin_weighted": 237.3404571533203, |
| "step": 620 |
| }, |
| { |
| "entropy": 0.4126953125, |
| "epoch": 0.3395311236863379, |
| "grad_norm": 24.879779815673828, |
| "learning_rate": 8.361930736950299e-07, |
| "logits/chosen": -0.8090851961587198, |
| "logits/rejected": -0.8543913061734949, |
| "logps/chosen": -432.2, |
| "logps/rejected": -689.9, |
| "loss": 0.3312, |
| "mean_token_accuracy": 0.7620727181434631, |
| "num_tokens": 22367339.0, |
| "rewards/accuracies": 0.884375, |
| "rewards/chosen": -3.6054784178733827, |
| "rewards/margins": 2.696714848279953, |
| "rewards/rejected": -6.302193331718445, |
| "setc/cal_net_lr": 0.00048056082012398516, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0865101970732212, |
| "setc/logratio_margin_delta": 17.24649543762207, |
| "setc/logratio_margin_vanilla": 252.425, |
| "setc/logratio_margin_weighted": 269.67149543762207, |
| "step": 630 |
| }, |
| { |
| "entropy": 0.402734375, |
| "epoch": 0.3449205066019941, |
| "grad_norm": 29.819805145263672, |
| "learning_rate": 8.29171697288799e-07, |
| "logits/chosen": -0.7799408370558261, |
| "logits/rejected": -0.7945290786054311, |
| "logps/chosen": -520.6, |
| "logps/rejected": -788.4, |
| "loss": 0.4063, |
| "mean_token_accuracy": 0.7504911392927169, |
| "num_tokens": 22746633.0, |
| "rewards/accuracies": 0.825, |
| "rewards/chosen": -4.477042305469513, |
| "rewards/margins": 2.874571180343628, |
| "rewards/rejected": -7.351613521575928, |
| "setc/cal_net_lr": 0.00046162386716145867, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08807485699653625, |
| "setc/logratio_margin_delta": 20.70712242126465, |
| "setc/logratio_margin_vanilla": 266.75, |
| "setc/logratio_margin_weighted": 287.45712242126467, |
| "step": 640 |
| }, |
| { |
| "entropy": 0.411328125, |
| "epoch": 0.35030988951765024, |
| "grad_norm": 45.361915588378906, |
| "learning_rate": 8.220338341449986e-07, |
| "logits/chosen": -0.7536949779449323, |
| "logits/rejected": -0.797799150332984, |
| "logps/chosen": -483.3, |
| "logps/rejected": -779.8, |
| "loss": 0.3723, |
| "mean_token_accuracy": 0.7493287652730942, |
| "num_tokens": 23123993.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -4.141227328777314, |
| "rewards/margins": 3.1743425011634825, |
| "rewards/rejected": -7.315569949150086, |
| "setc/cal_net_lr": 0.00044274215350057644, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08726362958550453, |
| "setc/logratio_margin_delta": 19.73426208496094, |
| "setc/logratio_margin_vanilla": 297.7, |
| "setc/logratio_margin_weighted": 317.43426208496095, |
| "step": 650 |
| }, |
| { |
| "entropy": 0.4083984375, |
| "epoch": 0.3556992724333064, |
| "grad_norm": 37.76664352416992, |
| "learning_rate": 8.147820101988704e-07, |
| "logits/chosen": -0.8120619076794536, |
| "logits/rejected": -0.8494678070244117, |
| "logps/chosen": -361.7, |
| "logps/rejected": -565.3, |
| "loss": 0.3474, |
| "mean_token_accuracy": 0.766129943728447, |
| "num_tokens": 23454227.0, |
| "rewards/accuracies": 0.8375, |
| "rewards/chosen": -2.9051371812820435, |
| "rewards/margins": 2.1819508969783783, |
| "rewards/rejected": -5.087088012695313, |
| "setc/cal_net_lr": 0.00042394285782244395, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08217526227235794, |
| "setc/logratio_margin_delta": 14.345091247558594, |
| "setc/logratio_margin_vanilla": 203.85, |
| "setc/logratio_margin_weighted": 218.1950912475586, |
| "step": 660 |
| }, |
| { |
| "entropy": 0.421875, |
| "epoch": 0.36108865534896256, |
| "grad_norm": 39.73728942871094, |
| "learning_rate": 8.074187917139175e-07, |
| "logits/chosen": -0.7319666812006074, |
| "logits/rejected": -0.7562072698077329, |
| "logps/chosen": -492.1, |
| "logps/rejected": -771.8, |
| "loss": 0.37, |
| "mean_token_accuracy": 0.7497668653726578, |
| "num_tokens": 23818594.0, |
| "rewards/accuracies": 0.85, |
| "rewards/chosen": -4.2121153473854065, |
| "rewards/margins": 2.9903420001268386, |
| "rewards/rejected": -7.202457308769226, |
| "setc/cal_net_lr": 0.00040525304017422745, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08255718871951104, |
| "setc/logratio_margin_delta": 18.634216690063475, |
| "setc/logratio_margin_vanilla": 280.4, |
| "setc/logratio_margin_weighted": 299.0342166900635, |
| "step": 670 |
| }, |
| { |
| "entropy": 0.42548828125, |
| "epoch": 0.3664780382646187, |
| "grad_norm": 40.21190643310547, |
| "learning_rate": 7.999467843737582e-07, |
| "logits/chosen": -0.8038282930095126, |
| "logits/rejected": -0.8421265761188057, |
| "logps/chosen": -460.9, |
| "logps/rejected": -701.0, |
| "loss": 0.3708, |
| "mean_token_accuracy": 0.7515695452690124, |
| "num_tokens": 24157407.0, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -3.902362608909607, |
| "rewards/margins": 2.5381152600049974, |
| "rewards/rejected": -6.440477788448334, |
| "setc/cal_net_lr": 0.00038669960301843077, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0827824518084526, |
| "setc/logratio_margin_delta": 16.236531066894532, |
| "setc/logratio_margin_vanilla": 237.575, |
| "setc/logratio_margin_weighted": 253.81153106689453, |
| "step": 680 |
| }, |
| { |
| "entropy": 0.39599609375, |
| "epoch": 0.3718674211802749, |
| "grad_norm": 20.98696517944336, |
| "learning_rate": 7.92368632360032e-07, |
| "logits/chosen": -0.8042416360177865, |
| "logits/rejected": -0.8270914788970604, |
| "logps/chosen": -469.8, |
| "logps/rejected": -730.8, |
| "loss": 0.344, |
| "mean_token_accuracy": 0.7479289263486862, |
| "num_tokens": 24493771.0, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -4.073775279521942, |
| "rewards/margins": 2.744447809457779, |
| "rewards/rejected": -6.8182231187820435, |
| "setc/cal_net_lr": 0.000368309252509004, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.08153500594198704, |
| "setc/logratio_margin_delta": 16.7447868347168, |
| "setc/logratio_margin_vanilla": 257.7, |
| "setc/logratio_margin_weighted": 274.4447868347168, |
| "step": 690 |
| }, |
| { |
| "entropy": 0.3873046875, |
| "epoch": 0.37725680409593104, |
| "grad_norm": 58.19563674926758, |
| "learning_rate": 7.846870174166803e-07, |
| "logits/chosen": -0.751562047809603, |
| "logits/rejected": -0.7788989722070948, |
| "logps/chosen": -553.7, |
| "logps/rejected": -904.5, |
| "loss": 0.3499, |
| "mean_token_accuracy": 0.7495994985103607, |
| "num_tokens": 24861785.0, |
| "rewards/accuracies": 0.8375, |
| "rewards/chosen": -4.865620005130768, |
| "rewards/margins": 3.707144260406494, |
| "rewards/rejected": -8.57276439666748, |
| "setc/cal_net_lr": 0.00035010846005002045, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0810060478746891, |
| "setc/logratio_margin_delta": 21.064439392089845, |
| "setc/logratio_margin_vanilla": 349.65, |
| "setc/logratio_margin_weighted": 370.71443939208984, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.37725680409593104, |
| "eval_entropy": 0.4024658203125, |
| "eval_logits/chosen": -0.8513818849761876, |
| "eval_logits/rejected": -0.8638025440627952, |
| "eval_logps/chosen": -570.4375, |
| "eval_logps/rejected": -930.875, |
| "eval_loss": 0.3007389008998871, |
| "eval_mean_token_accuracy": 0.7318701520562172, |
| "eval_num_tokens": 24861785.0, |
| "eval_rewards/accuracies": 0.86328125, |
| "eval_rewards/chosen": -5.0088541358709335, |
| "eval_rewards/margins": 3.838240258395672, |
| "eval_rewards/rejected": -8.847094357013702, |
| "eval_runtime": 14.2035, |
| "eval_samples_per_second": 35.202, |
| "eval_setc/cal_net_lr": 0.00034062822999975144, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.08001079317182302, |
| "eval_setc/logratio_margin_delta": 21.063920974731445, |
| "eval_setc/logratio_margin_vanilla": 363.875, |
| "eval_setc/logratio_margin_weighted": 384.93892097473145, |
| "eval_steps_per_second": 2.253, |
| "step": 700 |
| }, |
| { |
| "entropy": 0.4060546875, |
| "epoch": 0.38264618701158715, |
| "grad_norm": 86.25070190429688, |
| "learning_rate": 7.769046579009356e-07, |
| "logits/chosen": -0.8918346681389394, |
| "logits/rejected": -0.9074910639956075, |
| "logps/chosen": -639.5, |
| "logps/rejected": -989.4, |
| "loss": 0.4045, |
| "mean_token_accuracy": 0.7125069379806519, |
| "num_tokens": 25201876.0, |
| "rewards/accuracies": 0.821875, |
| "rewards/chosen": -5.8089584589004515, |
| "rewards/margins": 3.719921666383743, |
| "rewards/rejected": -9.52888035774231, |
| "setc/cal_net_lr": 0.0003321234241922574, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07928324565291404, |
| "setc/logratio_margin_delta": 21.29217185974121, |
| "setc/logratio_margin_vanilla": 350.7, |
| "setc/logratio_margin_weighted": 371.99217185974123, |
| "step": 710 |
| }, |
| { |
| "entropy": 0.4359375, |
| "epoch": 0.3880355699272433, |
| "grad_norm": 45.142372131347656, |
| "learning_rate": 7.690243078213531e-07, |
| "logits/chosen": -0.9740272973330708, |
| "logits/rejected": -1.0063528667573212, |
| "logps/chosen": -689.2, |
| "logps/rejected": -1139.4, |
| "loss": 0.3136, |
| "mean_token_accuracy": 0.6987293243408204, |
| "num_tokens": 25567728.0, |
| "rewards/accuracies": 0.88125, |
| "rewards/chosen": -6.253266739845276, |
| "rewards/margins": 4.749369788169861, |
| "rewards/rejected": -11.002636575698853, |
| "setc/cal_net_lr": 0.0003143800329225269, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0756493654102087, |
| "setc/logratio_margin_delta": 25.636982727050782, |
| "setc/logratio_margin_vanilla": 449.3, |
| "setc/logratio_margin_weighted": 474.9369827270508, |
| "step": 720 |
| }, |
| { |
| "entropy": 0.45234375, |
| "epoch": 0.39342495284289947, |
| "grad_norm": 13.179286003112793, |
| "learning_rate": 7.610487558632278e-07, |
| "logits/chosen": -1.0633192171048695, |
| "logits/rejected": -1.1078102349589132, |
| "logps/chosen": -656.4, |
| "logps/rejected": -1050.0, |
| "loss": 0.4255, |
| "mean_token_accuracy": 0.693815928697586, |
| "num_tokens": 25901754.0, |
| "rewards/accuracies": 0.85625, |
| "rewards/chosen": -5.947020316123963, |
| "rewards/margins": 4.109345865249634, |
| "rewards/rejected": -10.056366229057312, |
| "setc/cal_net_lr": 0.00029690382640003857, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07734486311674119, |
| "setc/logratio_margin_delta": 20.284591674804688, |
| "setc/logratio_margin_vanilla": 390.65, |
| "setc/logratio_margin_weighted": 410.93459167480466, |
| "step": 730 |
| }, |
| { |
| "entropy": 0.44501953125, |
| "epoch": 0.3988143357585556, |
| "grad_norm": 73.25760650634766, |
| "learning_rate": 7.529808244017382e-07, |
| "logits/chosen": -1.0208730154549697, |
| "logits/rejected": -1.055008389785436, |
| "logps/chosen": -627.0, |
| "logps/rejected": -961.0, |
| "loss": 0.4673, |
| "mean_token_accuracy": 0.70859514772892, |
| "num_tokens": 26254733.0, |
| "rewards/accuracies": 0.865625, |
| "rewards/chosen": -5.5865224480628966, |
| "rewards/margins": 3.537639796733856, |
| "rewards/rejected": -9.124162173271179, |
| "setc/cal_net_lr": 0.00027971996019343084, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07454265281558037, |
| "setc/logratio_margin_delta": 17.0139892578125, |
| "setc/logratio_margin_vanilla": 336.75, |
| "setc/logratio_margin_weighted": 353.7639892578125, |
| "step": 740 |
| }, |
| { |
| "entropy": 0.45361328125, |
| "epoch": 0.4042037186742118, |
| "grad_norm": 94.29061126708984, |
| "learning_rate": 7.448233685031693e-07, |
| "logits/chosen": -1.0630623814562947, |
| "logits/rejected": -1.1073087245931403, |
| "logps/chosen": -550.3, |
| "logps/rejected": -882.2, |
| "loss": 0.3111, |
| "mean_token_accuracy": 0.7219261229038239, |
| "num_tokens": 26587461.0, |
| "rewards/accuracies": 0.89375, |
| "rewards/chosen": -4.7769605159759525, |
| "rewards/margins": 3.535008490085602, |
| "rewards/rejected": -8.311969017982483, |
| "setc/cal_net_lr": 0.00026285316907138917, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07642085328698159, |
| "setc/logratio_margin_delta": 19.450860595703126, |
| "setc/logratio_margin_vanilla": 334.05, |
| "setc/logratio_margin_weighted": 353.5008605957031, |
| "step": 750 |
| }, |
| { |
| "entropy": 0.40673828125, |
| "epoch": 0.40959310158986795, |
| "grad_norm": 35.175498962402344, |
| "learning_rate": 7.365792749145662e-07, |
| "logits/chosen": -1.0218951494827677, |
| "logits/rejected": -1.048595356950924, |
| "logps/chosen": -488.2, |
| "logps/rejected": -758.2, |
| "loss": 0.37, |
| "mean_token_accuracy": 0.73828344643116, |
| "num_tokens": 26953136.0, |
| "rewards/accuracies": 0.871875, |
| "rewards/chosen": -4.2265440940856935, |
| "rewards/margins": 2.8608301758766173, |
| "rewards/rejected": -7.08737428188324, |
| "setc/cal_net_lr": 0.00024632773139897124, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07358978651463985, |
| "setc/logratio_margin_delta": 16.033021926879883, |
| "setc/logratio_margin_vanilla": 270.05, |
| "setc/logratio_margin_weighted": 286.0830219268799, |
| "step": 760 |
| }, |
| { |
| "entropy": 0.40654296875, |
| "epoch": 0.4149824845055241, |
| "grad_norm": 37.247169494628906, |
| "learning_rate": 7.28251461042177e-07, |
| "logits/chosen": -1.0070175249471507, |
| "logits/rejected": -1.0310765859216962, |
| "logps/chosen": -496.5, |
| "logps/rejected": -771.8, |
| "loss": 0.37, |
| "mean_token_accuracy": 0.7500552415847779, |
| "num_tokens": 27312799.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -4.265070939064026, |
| "rewards/margins": 2.9076979637145994, |
| "rewards/rejected": -7.172768950462341, |
| "setc/cal_net_lr": 0.00023016743419088836, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07328317761421203, |
| "setc/logratio_margin_delta": 18.319805145263672, |
| "setc/logratio_margin_vanilla": 272.45, |
| "setc/logratio_margin_weighted": 290.76980514526366, |
| "step": 770 |
| }, |
| { |
| "entropy": 0.43095703125, |
| "epoch": 0.42037186742118027, |
| "grad_norm": 19.564472198486328, |
| "learning_rate": 7.198428739190457e-07, |
| "logits/chosen": -1.082859984019952, |
| "logits/rejected": -1.0929271185391367, |
| "logps/chosen": -557.8, |
| "logps/rejected": -832.2, |
| "loss": 0.3455, |
| "mean_token_accuracy": 0.7363463073968888, |
| "num_tokens": 27667656.0, |
| "rewards/accuracies": 0.86875, |
| "rewards/chosen": -4.847213423252105, |
| "rewards/margins": 2.919535148143768, |
| "rewards/rejected": -7.766748380661011, |
| "setc/cal_net_lr": 0.00021439553887204476, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07149026319384574, |
| "setc/logratio_margin_delta": 16.803517150878907, |
| "setc/logratio_margin_vanilla": 275.15, |
| "setc/logratio_margin_weighted": 291.9535171508789, |
| "step": 780 |
| }, |
| { |
| "entropy": 0.428515625, |
| "epoch": 0.4257612503368364, |
| "grad_norm": 24.025815963745117, |
| "learning_rate": 7.113564891621212e-07, |
| "logits/chosen": -0.9810840312218222, |
| "logits/rejected": -1.0128754190678195, |
| "logps/chosen": -626.1, |
| "logps/rejected": -965.2, |
| "loss": 0.3468, |
| "mean_token_accuracy": 0.729249706864357, |
| "num_tokens": 28037217.0, |
| "rewards/accuracies": 0.86875, |
| "rewards/chosen": -5.643828749656677, |
| "rewards/margins": 3.574371063709259, |
| "rewards/rejected": -9.218199872970581, |
| "setc/cal_net_lr": 0.00019903474779462064, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07268696911633014, |
| "setc/logratio_margin_delta": 19.687115478515626, |
| "setc/logratio_margin_vanilla": 337.75, |
| "setc/logratio_margin_weighted": 357.43711547851564, |
| "step": 790 |
| }, |
| { |
| "entropy": 0.4359375, |
| "epoch": 0.4311506332524926, |
| "grad_norm": 26.062593460083008, |
| "learning_rate": 7.027953099192509e-07, |
| "logits/chosen": -1.037471741150463, |
| "logits/rejected": -1.0548466740521376, |
| "logps/chosen": -552.5, |
| "logps/rejected": -859.8, |
| "loss": 0.2958, |
| "mean_token_accuracy": 0.7349712908267975, |
| "num_tokens": 28389736.0, |
| "rewards/accuracies": 0.896875, |
| "rewards/chosen": -4.908501935005188, |
| "rewards/margins": 3.250082492828369, |
| "rewards/rejected": -8.15858452320099, |
| "setc/cal_net_lr": 0.0001841071715598952, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0715821348130703, |
| "setc/logratio_margin_delta": 19.408258819580077, |
| "setc/logratio_margin_vanilla": 305.6, |
| "setc/logratio_margin_weighted": 325.0082588195801, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.4311506332524926, |
| "eval_entropy": 0.437255859375, |
| "eval_logits/chosen": -1.0900919600946075, |
| "eval_logits/rejected": -1.1059430813034792, |
| "eval_logps/chosen": -572.875, |
| "eval_logps/rejected": -897.125, |
| "eval_loss": 0.29659560322761536, |
| "eval_mean_token_accuracy": 0.7379258405417204, |
| "eval_num_tokens": 28389736.0, |
| "eval_rewards/accuracies": 0.890625, |
| "eval_rewards/chosen": -5.015224754810333, |
| "eval_rewards/margins": 3.451749622821808, |
| "eval_rewards/rejected": -8.466974467039108, |
| "eval_runtime": 14.2077, |
| "eval_samples_per_second": 35.192, |
| "eval_setc/cal_net_lr": 0.00017643152107359864, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.07336517237126827, |
| "eval_setc/logratio_margin_delta": 19.209267139434814, |
| "eval_setc/logratio_margin_vanilla": 328.59375, |
| "eval_setc/logratio_margin_weighted": 347.8030171394348, |
| "eval_steps_per_second": 2.252, |
| "step": 800 |
| }, |
| { |
| "entropy": 0.43359375, |
| "epoch": 0.43654001616814875, |
| "grad_norm": 70.45535278320312, |
| "learning_rate": 6.941623658064312e-07, |
| "logits/chosen": -1.0733893563106265, |
| "logits/rejected": -1.0982346659335882, |
| "logps/chosen": -633.1, |
| "logps/rejected": -941.0, |
| "loss": 0.3849, |
| "mean_token_accuracy": 0.729145985841751, |
| "num_tokens": 28733805.0, |
| "rewards/accuracies": 0.834375, |
| "rewards/chosen": -5.708724451065064, |
| "rewards/margins": 3.248158019781113, |
| "rewards/rejected": -8.956882286071778, |
| "setc/cal_net_lr": 0.00016963429719184532, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07415341325104237, |
| "setc/logratio_margin_delta": 17.66580581665039, |
| "setc/logratio_margin_vanilla": 307.15, |
| "setc/logratio_margin_weighted": 324.81580581665037, |
| "step": 810 |
| }, |
| { |
| "entropy": 0.4302734375, |
| "epoch": 0.4419293990838049, |
| "grad_norm": 40.56505584716797, |
| "learning_rate": 6.854607118356928e-07, |
| "logits/chosen": -1.1385733399006264, |
| "logits/rejected": -1.183113374768871, |
| "logps/chosen": -566.1, |
| "logps/rejected": -896.6, |
| "loss": 0.3152, |
| "mean_token_accuracy": 0.7246308416128159, |
| "num_tokens": 29076104.0, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -5.08197500705719, |
| "rewards/margins": 3.4515433311462402, |
| "rewards/rejected": -8.53351833820343, |
| "setc/cal_net_lr": 0.0001556369572083343, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07320533730089665, |
| "setc/logratio_margin_delta": 18.154342651367188, |
| "setc/logratio_margin_vanilla": 327.0, |
| "setc/logratio_margin_weighted": 345.1543426513672, |
| "step": 820 |
| }, |
| { |
| "entropy": 0.43427734375, |
| "epoch": 0.44731878199946107, |
| "grad_norm": 56.18523025512695, |
| "learning_rate": 6.766934273339973e-07, |
| "logits/chosen": -1.1405602822344687, |
| "logits/rejected": -1.1867411468308247, |
| "logps/chosen": -557.2, |
| "logps/rejected": -863.8, |
| "loss": 0.2993, |
| "mean_token_accuracy": 0.7315610319375991, |
| "num_tokens": 29438936.0, |
| "rewards/accuracies": 0.88125, |
| "rewards/chosen": -4.800099658966064, |
| "rewards/margins": 3.2655778348445894, |
| "rewards/rejected": -8.065677428245545, |
| "setc/cal_net_lr": 0.00014213529963440723, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07172517664730549, |
| "setc/logratio_margin_delta": 18.5077823638916, |
| "setc/logratio_margin_vanilla": 308.05, |
| "setc/logratio_margin_weighted": 326.5577823638916, |
| "step": 830 |
| }, |
| { |
| "entropy": 0.4123046875, |
| "epoch": 0.45270816491511723, |
| "grad_norm": 27.114397048950195, |
| "learning_rate": 6.678636148535307e-07, |
| "logits/chosen": -1.1366168963090462, |
| "logits/rejected": -1.173395336965087, |
| "logps/chosen": -496.6, |
| "logps/rejected": -803.6, |
| "loss": 0.3137, |
| "mean_token_accuracy": 0.7537930309772491, |
| "num_tokens": 29818328.0, |
| "rewards/accuracies": 0.871875, |
| "rewards/chosen": -4.169769895076752, |
| "rewards/margins": 3.23821382522583, |
| "rewards/rejected": -7.407983732223511, |
| "setc/cal_net_lr": 0.0001291487590008584, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07007276304066182, |
| "setc/logratio_margin_delta": 19.77139205932617, |
| "setc/logratio_margin_vanilla": 304.05, |
| "setc/logratio_margin_weighted": 323.8213920593262, |
| "step": 840 |
| }, |
| { |
| "entropy": 0.4216796875, |
| "epoch": 0.4580975478307734, |
| "grad_norm": 41.05332946777344, |
| "learning_rate": 6.58974399073777e-07, |
| "logits/chosen": -1.1544986734058926, |
| "logits/rejected": -1.1544554202476698, |
| "logps/chosen": -538.0, |
| "logps/rejected": -871.0, |
| "loss": 0.2942, |
| "mean_token_accuracy": 0.736896401643753, |
| "num_tokens": 30188501.0, |
| "rewards/accuracies": 0.8875, |
| "rewards/chosen": -4.634168601036071, |
| "rewards/margins": 3.525930380821228, |
| "rewards/rejected": -8.160099053382874, |
| "setc/cal_net_lr": 0.00011669602836981566, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07121242731809616, |
| "setc/logratio_margin_delta": 21.643047332763672, |
| "setc/logratio_margin_vanilla": 330.95, |
| "setc/logratio_margin_weighted": 352.59304733276366, |
| "step": 850 |
| }, |
| { |
| "entropy": 0.4259765625, |
| "epoch": 0.46348693074642955, |
| "grad_norm": 64.55622100830078, |
| "learning_rate": 6.500289256957615e-07, |
| "logits/chosen": -1.1011358195249479, |
| "logits/rejected": -1.1499861979740114, |
| "logps/chosen": -573.8, |
| "logps/rejected": -874.0, |
| "loss": 0.4537, |
| "mean_token_accuracy": 0.7331288278102874, |
| "num_tokens": 30547224.0, |
| "rewards/accuracies": 0.803125, |
| "rewards/chosen": -5.030296123027801, |
| "rewards/margins": 3.2068679571151733, |
| "rewards/rejected": -8.237163949012757, |
| "setc/cal_net_lr": 0.00010479503242760742, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0720590002834797, |
| "setc/logratio_margin_delta": 18.836805725097655, |
| "setc/logratio_margin_vanilla": 301.85, |
| "setc/logratio_margin_weighted": 320.68680572509766, |
| "step": 860 |
| }, |
| { |
| "entropy": 0.43505859375, |
| "epoch": 0.4688763136620857, |
| "grad_norm": 39.62507247924805, |
| "learning_rate": 6.410303603288561e-07, |
| "logits/chosen": -1.088289906345851, |
| "logits/rejected": -1.0841832767082082, |
| "logps/chosen": -404.8, |
| "logps/rejected": -620.1, |
| "loss": 0.3673, |
| "mean_token_accuracy": 0.7633604764938354, |
| "num_tokens": 30902393.0, |
| "rewards/accuracies": 0.8625, |
| "rewards/chosen": -3.1859230756759644, |
| "rewards/margins": 2.2959958791732786, |
| "rewards/rejected": -5.481918931007385, |
| "setc/cal_net_lr": 9.346290168364385e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06972141042351723, |
| "setc/logratio_margin_delta": 14.549591827392579, |
| "setc/logratio_margin_vanilla": 215.05, |
| "setc/logratio_margin_weighted": 229.59959182739257, |
| "step": 870 |
| }, |
| { |
| "entropy": 0.41787109375, |
| "epoch": 0.47426569657774187, |
| "grad_norm": 33.14332962036133, |
| "learning_rate": 6.319818873705377e-07, |
| "logits/chosen": -1.089281081386084, |
| "logits/rejected": -1.114244560013979, |
| "logps/chosen": -462.6, |
| "logps/rejected": -784.6, |
| "loss": 0.3062, |
| "mean_token_accuracy": 0.7454081952571869, |
| "num_tokens": 31235617.0, |
| "rewards/accuracies": 0.896875, |
| "rewards/chosen": -3.9631998538970947, |
| "rewards/margins": 3.428285652399063, |
| "rewards/rejected": -7.391485595703125, |
| "setc/cal_net_lr": 8.271594781245119e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07049608752131462, |
| "setc/logratio_margin_delta": 20.25357093811035, |
| "setc/logratio_margin_vanilla": 322.575, |
| "setc/logratio_margin_weighted": 342.8285709381104, |
| "step": 880 |
| }, |
| { |
| "entropy": 0.4427734375, |
| "epoch": 0.47965507949339803, |
| "grad_norm": 25.35371208190918, |
| "learning_rate": 6.228867088794997e-07, |
| "logits/chosen": -1.117235813203115, |
| "logits/rejected": -1.1379867493894777, |
| "logps/chosen": -611.3, |
| "logps/rejected": -992.4, |
| "loss": 0.3308, |
| "mean_token_accuracy": 0.7231000691652298, |
| "num_tokens": 31611665.0, |
| "rewards/accuracies": 0.85625, |
| "rewards/chosen": -5.4193216323852536, |
| "rewards/margins": 4.017894721031189, |
| "rewards/rejected": -9.437216472625732, |
| "setc/cal_net_lr": 7.256964017435153e-05, |
| "setc/credit_mean": 0.9999999970197677, |
| "setc/credit_std": 0.0703140676021576, |
| "setc/logratio_margin_delta": 22.489488220214845, |
| "setc/logratio_margin_vanilla": 379.3, |
| "setc/logratio_margin_weighted": 401.7894882202148, |
| "step": 890 |
| }, |
| { |
| "entropy": 0.425390625, |
| "epoch": 0.4850444624090542, |
| "grad_norm": 21.312118530273438, |
| "learning_rate": 6.137480434425124e-07, |
| "logits/chosen": -1.063890301715667, |
| "logits/rejected": -1.1046153500158946, |
| "logps/chosen": -502.7, |
| "logps/rejected": -788.2, |
| "loss": 0.3396, |
| "mean_token_accuracy": 0.7333939164876938, |
| "num_tokens": 31947911.0, |
| "rewards/accuracies": 0.853125, |
| "rewards/chosen": -4.411977684497833, |
| "rewards/margins": 2.9957467019557953, |
| "rewards/rejected": -7.407724452018738, |
| "setc/cal_net_lr": 6.303858354858501e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06882553435862064, |
| "setc/logratio_margin_delta": 16.82468185424805, |
| "setc/logratio_margin_vanilla": 282.75, |
| "setc/logratio_margin_weighted": 299.57468185424807, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.4850444624090542, |
| "eval_entropy": 0.458984375, |
| "eval_logits/chosen": -1.136106022790833, |
| "eval_logits/rejected": -1.1553218335773052, |
| "eval_logps/chosen": -567.9375, |
| "eval_logps/rejected": -919.125, |
| "eval_loss": 0.2798229157924652, |
| "eval_mean_token_accuracy": 0.7211284190416336, |
| "eval_num_tokens": 31947911.0, |
| "eval_rewards/accuracies": 0.888671875, |
| "eval_rewards/chosen": -4.92750097066164, |
| "eval_rewards/margins": 3.7332901805639267, |
| "eval_rewards/rejected": -8.660791024565697, |
| "eval_runtime": 14.2045, |
| "eval_samples_per_second": 35.2, |
| "eval_setc/cal_net_lr": 5.825929969209536e-05, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.07130194280762225, |
| "eval_setc/logratio_margin_delta": 20.13284683227539, |
| "eval_setc/logratio_margin_vanilla": 354.25, |
| "eval_setc/logratio_margin_weighted": 374.3828468322754, |
| "eval_steps_per_second": 2.253, |
| "step": 900 |
| }, |
| { |
| "entropy": 0.4380859375, |
| "epoch": 0.4904338453247103, |
| "grad_norm": 30.84346580505371, |
| "learning_rate": 6.045691250354349e-07, |
| "logits/chosen": -1.0498209270589363, |
| "logits/rejected": -1.07947071859647, |
| "logps/chosen": -607.5, |
| "logps/rejected": -951.4, |
| "loss": 0.3157, |
| "mean_token_accuracy": 0.7191443383693695, |
| "num_tokens": 32308377.0, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -5.468822598457336, |
| "rewards/margins": 3.641323584318161, |
| "rewards/rejected": -9.110146307945252, |
| "setc/cal_net_lr": 5.413649711092565e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07014792338013649, |
| "setc/logratio_margin_delta": 21.932370758056642, |
| "setc/logratio_margin_vanilla": 342.2, |
| "setc/logratio_margin_weighted": 364.13237075805665, |
| "step": 910 |
| }, |
| { |
| "entropy": 0.43798828125, |
| "epoch": 0.49582322824036645, |
| "grad_norm": 69.57173919677734, |
| "learning_rate": 5.953532018787807e-07, |
| "logits/chosen": -1.0614989872760174, |
| "logits/rejected": -1.1125016970713177, |
| "logps/chosen": -562.6, |
| "logps/rejected": -910.6, |
| "loss": 0.3801, |
| "mean_token_accuracy": 0.7194907575845718, |
| "num_tokens": 32667657.0, |
| "rewards/accuracies": 0.834375, |
| "rewards/chosen": -5.013018798828125, |
| "rewards/margins": 3.655552077293396, |
| "rewards/rejected": -8.668570947647094, |
| "setc/cal_net_lr": 4.587619468605094e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07026223614811897, |
| "setc/logratio_margin_delta": 19.905216217041016, |
| "setc/logratio_margin_vanilla": 345.65, |
| "setc/logratio_margin_weighted": 365.555216217041, |
| "step": 920 |
| }, |
| { |
| "entropy": 0.43251953125, |
| "epoch": 0.5012126111560227, |
| "grad_norm": 37.31387710571289, |
| "learning_rate": 5.861035352882434e-07, |
| "logits/chosen": -1.1028243420067865, |
| "logits/rejected": -1.1239636597884728, |
| "logps/chosen": -668.2, |
| "logps/rejected": -1051.8, |
| "loss": 0.3082, |
| "mean_token_accuracy": 0.7101206243038177, |
| "num_tokens": 33038205.0, |
| "rewards/accuracies": 0.871875, |
| "rewards/chosen": -6.0299333214759825, |
| "rewards/margins": 4.101047283411026, |
| "rewards/rejected": -10.130980563163757, |
| "setc/cal_net_lr": 3.826956630309029e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07096938453614712, |
| "setc/logratio_margin_delta": 25.654744720458986, |
| "setc/logratio_margin_vanilla": 384.45, |
| "setc/logratio_margin_weighted": 410.104744720459, |
| "step": 930 |
| }, |
| { |
| "entropy": 0.43154296875, |
| "epoch": 0.5066019940716788, |
| "grad_norm": 31.79937744140625, |
| "learning_rate": 5.768233985205871e-07, |
| "logits/chosen": -1.0400086846324763, |
| "logits/rejected": -1.0826617979469901, |
| "logps/chosen": -603.6, |
| "logps/rejected": -1010.8, |
| "loss": 0.3537, |
| "mean_token_accuracy": 0.7205777823925018, |
| "num_tokens": 33412806.0, |
| "rewards/accuracies": 0.8375, |
| "rewards/chosen": -5.440573036670685, |
| "rewards/margins": 4.329031145572662, |
| "rewards/rejected": -9.769603991508484, |
| "setc/cal_net_lr": 3.1327561080901155e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07101482562720776, |
| "setc/logratio_margin_delta": 24.953127288818358, |
| "setc/logratio_margin_vanilla": 407.95, |
| "setc/logratio_margin_weighted": 432.9031272888184, |
| "step": 940 |
| }, |
| { |
| "entropy": 0.44111328125, |
| "epoch": 0.511991376987335, |
| "grad_norm": 41.023216247558594, |
| "learning_rate": 5.675160756153119e-07, |
| "logits/chosen": -1.063226666319973, |
| "logits/rejected": -1.103878181164049, |
| "logps/chosen": -572.8, |
| "logps/rejected": -937.8, |
| "loss": 0.2939, |
| "mean_token_accuracy": 0.7170851469039917, |
| "num_tokens": 33771494.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -5.1082786321640015, |
| "rewards/margins": 3.9003884315490724, |
| "rewards/rejected": -9.0086669921875, |
| "setc/cal_net_lr": 2.5060171467709125e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0710456695407629, |
| "setc/logratio_margin_delta": 24.788859558105468, |
| "setc/logratio_margin_vanilla": 365.25, |
| "setc/logratio_margin_weighted": 390.03885955810546, |
| "step": 950 |
| }, |
| { |
| "entropy": 0.44267578125, |
| "epoch": 0.5173807599029911, |
| "grad_norm": 52.85020446777344, |
| "learning_rate": 5.581848602325041e-07, |
| "logits/chosen": -1.0075269843195531, |
| "logits/rejected": -1.0690200247116604, |
| "logps/chosen": -536.4, |
| "logps/rejected": -866.6, |
| "loss": 0.3122, |
| "mean_token_accuracy": 0.7294569611549377, |
| "num_tokens": 34132231.0, |
| "rewards/accuracies": 0.878125, |
| "rewards/chosen": -4.7246493101119995, |
| "rewards/margins": 3.508440887928009, |
| "rewards/rejected": -8.233090209960938, |
| "setc/cal_net_lr": 1.9476418857796457e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07001486346125603, |
| "setc/logratio_margin_delta": 22.394107818603516, |
| "setc/logratio_margin_vanilla": 328.45, |
| "setc/logratio_margin_weighted": 350.8441078186035, |
| "step": 960 |
| }, |
| { |
| "entropy": 0.4556640625, |
| "epoch": 0.5227701428186473, |
| "grad_norm": 22.979949951171875, |
| "learning_rate": 5.488330544872797e-07, |
| "logits/chosen": -1.0871756321135606, |
| "logits/rejected": -1.1191254896199978, |
| "logps/chosen": -508.9, |
| "logps/rejected": -850.4, |
| "loss": 0.3046, |
| "mean_token_accuracy": 0.7397548407316208, |
| "num_tokens": 34502355.0, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -4.326072335243225, |
| "rewards/margins": 3.6845016717910766, |
| "rewards/rejected": -8.010574007034302, |
| "setc/cal_net_lr": 1.4584340605943599e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07133265994489194, |
| "setc/logratio_margin_delta": 22.850178527832032, |
| "setc/logratio_margin_vanilla": 345.6, |
| "setc/logratio_margin_weighted": 368.450178527832, |
| "step": 970 |
| }, |
| { |
| "entropy": 0.43857421875, |
| "epoch": 0.5281595257343035, |
| "grad_norm": 25.516042709350586, |
| "learning_rate": 5.394639677812387e-07, |
| "logits/chosen": -1.088331639351806, |
| "logits/rejected": -1.134811225998461, |
| "logps/chosen": -455.4, |
| "logps/rejected": -754.1, |
| "loss": 0.3234, |
| "mean_token_accuracy": 0.7478211104869843, |
| "num_tokens": 34852059.0, |
| "rewards/accuracies": 0.871875, |
| "rewards/chosen": -3.8503175020217895, |
| "rewards/margins": 3.19860480427742, |
| "rewards/rejected": -7.048922348022461, |
| "setc/cal_net_lr": 1.0390978458315019e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07125458978116513, |
| "setc/logratio_margin_delta": 19.160488891601563, |
| "setc/logratio_margin_vanilla": 300.7, |
| "setc/logratio_margin_weighted": 319.86048889160156, |
| "step": 980 |
| }, |
| { |
| "entropy": 0.41787109375, |
| "epoch": 0.5335489086499596, |
| "grad_norm": 22.045780181884766, |
| "learning_rate": 5.300809156313389e-07, |
| "logits/chosen": -1.1293688528513106, |
| "logits/rejected": -1.1482995959404332, |
| "logps/chosen": -498.4, |
| "logps/rejected": -795.4, |
| "loss": 0.3843, |
| "mean_token_accuracy": 0.7528087586164475, |
| "num_tokens": 35212012.0, |
| "rewards/accuracies": 0.86875, |
| "rewards/chosen": -4.3080038785934445, |
| "rewards/margins": 3.206386703252792, |
| "rewards/rejected": -7.514390516281128, |
| "setc/cal_net_lr": 6.902368416441938e-06, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07167959362268447, |
| "setc/logratio_margin_delta": 22.038671493530273, |
| "setc/logratio_margin_vanilla": 298.6, |
| "setc/logratio_margin_weighted": 320.6386714935303, |
| "step": 990 |
| }, |
| { |
| "entropy": 0.4220703125, |
| "epoch": 0.5389382915656158, |
| "grad_norm": 29.74481773376465, |
| "learning_rate": 5.206872184966065e-07, |
| "logits/chosen": -1.2005332112522926, |
| "logits/rejected": -1.223600991098159, |
| "logps/chosen": -475.5, |
| "logps/rejected": -771.0, |
| "loss": 0.3032, |
| "mean_token_accuracy": 0.7501377999782562, |
| "num_tokens": 35559559.0, |
| "rewards/accuracies": 0.884375, |
| "rewards/chosen": -4.051128447055817, |
| "rewards/margins": 3.163189709186554, |
| "rewards/rejected": -7.214318156242371, |
| "setc/cal_net_lr": 4.123532048892359e-06, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07157905176281928, |
| "setc/logratio_margin_delta": 21.718975830078126, |
| "setc/logratio_margin_vanilla": 294.6, |
| "setc/logratio_margin_weighted": 316.3189758300781, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.5389382915656158, |
| "eval_entropy": 0.42791748046875, |
| "eval_logits/chosen": -1.205733974360692, |
| "eval_logits/rejected": -1.2256254141801155, |
| "eval_logps/chosen": -520.25, |
| "eval_logps/rejected": -881.125, |
| "eval_loss": 0.26266777515411377, |
| "eval_mean_token_accuracy": 0.7448298633098602, |
| "eval_num_tokens": 35559559.0, |
| "eval_rewards/accuracies": 0.912109375, |
| "eval_rewards/chosen": -4.464685194194317, |
| "eval_rewards/margins": 3.8784405440092087, |
| "eval_rewards/rejected": -8.343125775456429, |
| "eval_runtime": 14.2575, |
| "eval_samples_per_second": 35.069, |
| "eval_setc/cal_net_lr": 2.9204217399144684e-06, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.07189792324788868, |
| "eval_setc/logratio_margin_delta": 25.44654369354248, |
| "eval_setc/logratio_margin_vanilla": 364.4375, |
| "eval_setc/logratio_margin_weighted": 389.8840436935425, |
| "eval_steps_per_second": 2.244, |
| "step": 1000 |
| }, |
| { |
| "entropy": 0.4033203125, |
| "epoch": 0.5443276744812718, |
| "grad_norm": 84.13672637939453, |
| "learning_rate": 5.11286200603097e-07, |
| "logits/chosen": -1.1744355391727033, |
| "logits/rejected": -1.1982522802884472, |
| "logps/chosen": -589.9, |
| "logps/rejected": -966.7, |
| "loss": 0.3591, |
| "mean_token_accuracy": 0.7260809451341629, |
| "num_tokens": 35914588.0, |
| "rewards/accuracies": 0.86875, |
| "rewards/chosen": -5.367753648757935, |
| "rewards/margins": 4.012299507856369, |
| "rewards/rejected": -9.380053186416626, |
| "setc/cal_net_lr": 2.058469263134096e-06, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07252811081707478, |
| "setc/logratio_margin_delta": 27.179954528808594, |
| "setc/logratio_margin_vanilla": 374.05, |
| "setc/logratio_margin_weighted": 401.2299545288086, |
| "step": 1010 |
| }, |
| { |
| "entropy": 0.41494140625, |
| "epoch": 0.549717057396928, |
| "grad_norm": 45.3025016784668, |
| "learning_rate": 5.018811887675243e-07, |
| "logits/chosen": -1.2095668156933808, |
| "logits/rejected": -1.2754331190012376, |
| "logps/chosen": -638.7, |
| "logps/rejected": -1027.8, |
| "loss": 0.2909, |
| "mean_token_accuracy": 0.7233078300952911, |
| "num_tokens": 36272220.0, |
| "rewards/accuracies": 0.9, |
| "rewards/chosen": -5.741470265388489, |
| "rewards/margins": 4.172668009996414, |
| "rewards/rejected": -9.914138317108154, |
| "setc/cal_net_lr": 7.101525479954687e-07, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07240922413766385, |
| "setc/logratio_margin_delta": 29.066813659667968, |
| "setc/logratio_margin_vanilla": 388.2, |
| "setc/logratio_margin_weighted": 417.26681365966795, |
| "step": 1020 |
| }, |
| { |
| "entropy": 0.41572265625, |
| "epoch": 0.5551064403125842, |
| "grad_norm": 54.05473327636719, |
| "learning_rate": 4.924755112199719e-07, |
| "logits/chosen": -1.1848740348447429, |
| "logits/rejected": -1.2239745932740136, |
| "logps/chosen": -530.6, |
| "logps/rejected": -873.4, |
| "loss": 0.3204, |
| "mean_token_accuracy": 0.737355038523674, |
| "num_tokens": 36608348.0, |
| "rewards/accuracies": 0.846875, |
| "rewards/chosen": -4.719290387630463, |
| "rewards/margins": 3.6306647956371307, |
| "rewards/rejected": -8.349955177307129, |
| "setc/cal_net_lr": 8.052269501096502e-08, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07150659598410129, |
| "setc/logratio_margin_delta": 23.716495513916016, |
| "setc/logratio_margin_vanilla": 339.35, |
| "setc/logratio_margin_weighted": 363.06649551391604, |
| "step": 1030 |
| }, |
| { |
| "entropy": 0.40263671875, |
| "epoch": 0.5604958232282403, |
| "grad_norm": 51.89723587036133, |
| "learning_rate": 4.830724964261044e-07, |
| "logits/chosen": -1.2390042148153568, |
| "logits/rejected": -1.2885369769102084, |
| "logps/chosen": -552.1, |
| "logps/rejected": -873.2, |
| "loss": 0.3291, |
| "mean_token_accuracy": 0.7273021250963211, |
| "num_tokens": 36931299.0, |
| "rewards/accuracies": 0.8875, |
| "rewards/chosen": -4.9631112813949585, |
| "rewards/margins": 3.396042114496231, |
| "rewards/rejected": -8.359153366088867, |
| "setc/cal_net_lr": 1.704860048107271e-07, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0706510066986084, |
| "setc/logratio_margin_delta": 24.204216766357423, |
| "setc/logratio_margin_vanilla": 315.4, |
| "setc/logratio_margin_weighted": 339.6042167663574, |
| "step": 1040 |
| }, |
| { |
| "entropy": 0.42197265625, |
| "epoch": 0.5658852061438965, |
| "grad_norm": 108.92530822753906, |
| "learning_rate": 4.736754719092948e-07, |
| "logits/chosen": -1.2478903284456264, |
| "logits/rejected": -1.3215152839167994, |
| "logps/chosen": -660.8, |
| "logps/rejected": -1058.0, |
| "loss": 0.317, |
| "mean_token_accuracy": 0.7129073649644851, |
| "num_tokens": 37295844.0, |
| "rewards/accuracies": 0.8875, |
| "rewards/chosen": -5.961016845703125, |
| "rewards/margins": 4.262525498867035, |
| "rewards/rejected": -10.223542261123658, |
| "setc/cal_net_lr": 9.799129825749437e-07, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0731208048760891, |
| "setc/logratio_margin_delta": 28.65255813598633, |
| "setc/logratio_margin_vanilla": 397.6, |
| "setc/logratio_margin_weighted": 426.2525581359863, |
| "step": 1050 |
| }, |
| { |
| "entropy": 0.40234375, |
| "epoch": 0.5712745890595526, |
| "grad_norm": 23.12958335876465, |
| "learning_rate": 4.6428776307308795e-07, |
| "logits/chosen": -1.2329498912657004, |
| "logits/rejected": -1.247452417260519, |
| "logps/chosen": -669.1, |
| "logps/rejected": -1018.6, |
| "loss": 0.3105, |
| "mean_token_accuracy": 0.7226340204477311, |
| "num_tokens": 37665756.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -6.078487741947174, |
| "rewards/margins": 3.7634204626083374, |
| "rewards/rejected": -9.841908311843872, |
| "setc/cal_net_lr": 2.5076385244310457e-06, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07050310261547565, |
| "setc/logratio_margin_delta": 26.592060089111328, |
| "setc/logratio_margin_vanilla": 349.75, |
| "setc/logratio_margin_weighted": 376.3420600891113, |
| "step": 1060 |
| }, |
| { |
| "entropy": 0.42890625, |
| "epoch": 0.5766639719752088, |
| "grad_norm": 54.92173385620117, |
| "learning_rate": 4.5491269202441044e-07, |
| "logits/chosen": -1.2268895919032963, |
| "logits/rejected": -1.2605704569389382, |
| "logps/chosen": -601.6, |
| "logps/rejected": -936.8, |
| "loss": 0.2924, |
| "mean_token_accuracy": 0.7248776346445084, |
| "num_tokens": 38031467.0, |
| "rewards/accuracies": 0.871875, |
| "rewards/chosen": -5.383153641223908, |
| "rewards/margins": 3.53509316444397, |
| "rewards/rejected": -8.918246793746949, |
| "setc/cal_net_lr": 4.751463594525389e-06, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07096486799418926, |
| "setc/logratio_margin_delta": 21.009319305419922, |
| "setc/logratio_margin_vanilla": 332.5, |
| "setc/logratio_margin_weighted": 353.5093193054199, |
| "step": 1070 |
| }, |
| { |
| "entropy": 0.43212890625, |
| "epoch": 0.582053354890865, |
| "grad_norm": 29.253252029418945, |
| "learning_rate": 4.455535763979489e-07, |
| "logits/chosen": -1.2967908691930823, |
| "logits/rejected": -1.334540072490827, |
| "logps/chosen": -673.7, |
| "logps/rejected": -1018.8, |
| "loss": 0.276, |
| "mean_token_accuracy": 0.7103154689073563, |
| "num_tokens": 38406113.0, |
| "rewards/accuracies": 0.8875, |
| "rewards/chosen": -6.113679957389832, |
| "rewards/margins": 3.689132344722748, |
| "rewards/rejected": -9.802812147140504, |
| "setc/cal_net_lr": 7.708158390355337e-06, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07218135371804238, |
| "setc/logratio_margin_delta": 23.763233947753907, |
| "setc/logratio_margin_vanilla": 345.15, |
| "setc/logratio_margin_weighted": 368.9132339477539, |
| "step": 1080 |
| }, |
| { |
| "entropy": 0.43740234375, |
| "epoch": 0.5874427378065211, |
| "grad_norm": 63.62852096557617, |
| "learning_rate": 4.3621372818211233e-07, |
| "logits/chosen": -1.2966434675325627, |
| "logits/rejected": -1.3388852385699441, |
| "logps/chosen": -717.7, |
| "logps/rejected": -1140.4, |
| "loss": 0.2884, |
| "mean_token_accuracy": 0.7034007340669632, |
| "num_tokens": 38739807.0, |
| "rewards/accuracies": 0.9, |
| "rewards/chosen": -6.62959771156311, |
| "rewards/margins": 4.472747385501862, |
| "rewards/rejected": -11.1023451089859, |
| "setc/cal_net_lr": 1.1373466991805524e-05, |
| "setc/credit_mean": 0.9999999970197677, |
| "setc/credit_std": 0.07208193615078926, |
| "setc/logratio_margin_delta": 29.124752044677734, |
| "setc/logratio_margin_vanilla": 418.15, |
| "setc/logratio_margin_weighted": 447.2747520446777, |
| "step": 1090 |
| }, |
| { |
| "entropy": 0.43876953125, |
| "epoch": 0.5928321207221773, |
| "grad_norm": 58.927433013916016, |
| "learning_rate": 4.2689645254698956e-07, |
| "logits/chosen": -1.3082769209270118, |
| "logits/rejected": -1.357060948196913, |
| "logps/chosen": -709.7, |
| "logps/rejected": -1137.4, |
| "loss": 0.2492, |
| "mean_token_accuracy": 0.713100990653038, |
| "num_tokens": 39121036.0, |
| "rewards/accuracies": 0.8875, |
| "rewards/chosen": -6.4656531572341915, |
| "rewards/margins": 4.571207290887832, |
| "rewards/rejected": -11.03686032295227, |
| "setc/cal_net_lr": 1.5742113487196524e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07217907235026359, |
| "setc/logratio_margin_delta": 28.670735931396486, |
| "setc/logratio_margin_vanilla": 428.45, |
| "setc/logratio_margin_weighted": 457.1207359313965, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.5928321207221773, |
| "eval_entropy": 0.44512939453125, |
| "eval_logits/chosen": -1.3603613258581002, |
| "eval_logits/rejected": -1.3904849397392145, |
| "eval_logps/chosen": -694.0625, |
| "eval_logps/rejected": -1149.25, |
| "eval_loss": 0.2531175911426544, |
| "eval_mean_token_accuracy": 0.7049690876156092, |
| "eval_num_tokens": 39121036.0, |
| "eval_rewards/accuracies": 0.904296875, |
| "eval_rewards/chosen": -6.33006377518177, |
| "eval_rewards/margins": 4.873635433614254, |
| "eval_rewards/rejected": -11.20369903743267, |
| "eval_runtime": 14.1864, |
| "eval_samples_per_second": 35.245, |
| "eval_setc/cal_net_lr": 1.8286307534312433e-05, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.07362057128921151, |
| "eval_setc/logratio_margin_delta": 30.792871475219727, |
| "eval_setc/logratio_margin_vanilla": 457.6875, |
| "eval_setc/logratio_margin_weighted": 488.4803714752197, |
| "eval_steps_per_second": 2.256, |
| "step": 1100 |
| }, |
| { |
| "entropy": 0.43310546875, |
| "epoch": 0.5982215036378334, |
| "grad_norm": 40.20852279663086, |
| "learning_rate": 4.176050466747224e-07, |
| "logits/chosen": -1.3270166713790517, |
| "logits/rejected": -1.4020509422736323, |
| "logps/chosen": -716.6, |
| "logps/rejected": -1198.2, |
| "loss": 0.2997, |
| "mean_token_accuracy": 0.6954291999340058, |
| "num_tokens": 39473664.0, |
| "rewards/accuracies": 0.878125, |
| "rewards/chosen": -6.649608445167542, |
| "rewards/margins": 5.128565120697021, |
| "rewards/rejected": -11.778173661231994, |
| "setc/cal_net_lr": 2.0807809567527706e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07330171279609203, |
| "setc/logratio_margin_delta": 34.506527709960935, |
| "setc/logratio_margin_vanilla": 478.35, |
| "setc/logratio_margin_weighted": 512.8565277099609, |
| "step": 1110 |
| }, |
| { |
| "entropy": 0.4423828125, |
| "epoch": 0.6036108865534896, |
| "grad_norm": 70.10661315917969, |
| "learning_rate": 4.0834279859270284e-07, |
| "logits/chosen": -1.3479278371743117, |
| "logits/rejected": -1.402101682907808, |
| "logps/chosen": -684.2, |
| "logps/rejected": -1106.6, |
| "loss": 0.2735, |
| "mean_token_accuracy": 0.698912826180458, |
| "num_tokens": 39821548.0, |
| "rewards/accuracies": 0.8875, |
| "rewards/chosen": -6.291271615028381, |
| "rewards/margins": 4.475275766849518, |
| "rewards/rejected": -10.766547250747681, |
| "setc/cal_net_lr": 2.6563263577983417e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07238798253238202, |
| "setc/logratio_margin_delta": 27.777584838867188, |
| "setc/logratio_margin_vanilla": 419.75, |
| "setc/logratio_margin_weighted": 447.5275848388672, |
| "step": 1120 |
| }, |
| { |
| "entropy": 0.43671875, |
| "epoch": 0.6090002694691458, |
| "grad_norm": 20.157817840576172, |
| "learning_rate": 3.9911298601001273e-07, |
| "logits/chosen": -1.3612183495667927, |
| "logits/rejected": -1.3977843345025138, |
| "logps/chosen": -601.0, |
| "logps/rejected": -1085.4, |
| "loss": 0.2369, |
| "mean_token_accuracy": 0.7253735572099685, |
| "num_tokens": 40192876.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -5.32529776096344, |
| "rewards/margins": 5.10829610824585, |
| "rewards/rejected": -10.433593916893006, |
| "setc/cal_net_lr": 3.3000191013673035e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07107761949300766, |
| "setc/logratio_margin_delta": 29.07962646484375, |
| "setc/logratio_margin_vanilla": 481.75, |
| "setc/logratio_margin_weighted": 510.8296264648437, |
| "step": 1130 |
| }, |
| { |
| "entropy": 0.45537109375, |
| "epoch": 0.6143896523848019, |
| "grad_norm": 47.181636810302734, |
| "learning_rate": 3.8991887515751285e-07, |
| "logits/chosen": -1.3811772662010782, |
| "logits/rejected": -1.4073035922265509, |
| "logps/chosen": -634.4, |
| "logps/rejected": -992.8, |
| "loss": 0.3292, |
| "mean_token_accuracy": 0.7175350487232208, |
| "num_tokens": 40559383.0, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -5.5887420058250425, |
| "rewards/margins": 3.8901084780693056, |
| "rewards/rejected": -9.478850364685059, |
| "setc/cal_net_lr": 4.010932644449762e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07180496528744698, |
| "setc/logratio_margin_delta": 26.260860443115234, |
| "setc/logratio_margin_vanilla": 362.75, |
| "setc/logratio_margin_weighted": 389.01086044311523, |
| "step": 1140 |
| }, |
| { |
| "entropy": 0.43125, |
| "epoch": 0.6197790353004581, |
| "grad_norm": 69.74315643310547, |
| "learning_rate": 3.807637196319943e-07, |
| "logits/chosen": -1.4106397734688008, |
| "logits/rejected": -1.4677872876176674, |
| "logps/chosen": -626.8, |
| "logps/rejected": -1127.6, |
| "loss": 0.3109, |
| "mean_token_accuracy": 0.7143973648548126, |
| "num_tokens": 40918488.0, |
| "rewards/accuracies": 0.871875, |
| "rewards/chosen": -5.679463398456574, |
| "rewards/margins": 5.340268242359161, |
| "rewards/rejected": -11.019731426239014, |
| "setc/cal_net_lr": 4.7880436851978056e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07189065329730511, |
| "setc/logratio_margin_delta": 37.0768325805664, |
| "setc/logratio_margin_vanilla": 496.95, |
| "setc/logratio_margin_weighted": 534.0268325805664, |
| "step": 1150 |
| }, |
| { |
| "entropy": 0.45703125, |
| "epoch": 0.6251684182161142, |
| "grad_norm": 98.07344055175781, |
| "learning_rate": 3.716507592448015e-07, |
| "logits/chosen": -1.388660761223864, |
| "logits/rejected": -1.3975069124599455, |
| "logps/chosen": -674.2, |
| "logps/rejected": -1123.8, |
| "loss": 0.2989, |
| "mean_token_accuracy": 0.711270448565483, |
| "num_tokens": 41284352.0, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -6.101946842670441, |
| "rewards/margins": 4.818000304698944, |
| "rewards/rejected": -10.919947171211243, |
| "setc/cal_net_lr": 5.630233635884748e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07173714898526669, |
| "setc/logratio_margin_delta": 30.45003662109375, |
| "setc/logratio_margin_vanilla": 451.35, |
| "setc/logratio_margin_weighted": 481.80003662109374, |
| "step": 1160 |
| }, |
| { |
| "entropy": 0.43251953125, |
| "epoch": 0.6305578011317704, |
| "grad_norm": 35.384769439697266, |
| "learning_rate": 3.625832188753326e-07, |
| "logits/chosen": -1.3047580708198727, |
| "logits/rejected": -1.369543789182836, |
| "logps/chosen": -584.7, |
| "logps/rejected": -974.6, |
| "loss": 0.3559, |
| "mean_token_accuracy": 0.7312755823135376, |
| "num_tokens": 41646831.0, |
| "rewards/accuracies": 0.88125, |
| "rewards/chosen": -5.220397734642029, |
| "rewards/margins": 4.157508033514023, |
| "rewards/rejected": -9.377905797958373, |
| "setc/cal_net_lr": 6.536290233020582e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07165979258716107, |
| "setc/logratio_margin_delta": 27.70081787109375, |
| "setc/logratio_margin_vanilla": 388.05, |
| "setc/logratio_margin_weighted": 415.7508178710938, |
| "step": 1170 |
| }, |
| { |
| "entropy": 0.45009765625, |
| "epoch": 0.6359471840474266, |
| "grad_norm": 32.074398040771484, |
| "learning_rate": 3.5356430732982537e-07, |
| "logits/chosen": -1.3558946550666882, |
| "logits/rejected": -1.367336323455139, |
| "logps/chosen": -480.1, |
| "logps/rejected": -838.2, |
| "loss": 0.3099, |
| "mean_token_accuracy": 0.7344884812831879, |
| "num_tokens": 41977848.0, |
| "rewards/accuracies": 0.86875, |
| "rewards/chosen": -4.130843257904052, |
| "rewards/margins": 3.8041255950927733, |
| "rewards/rejected": -7.934968829154968, |
| "setc/cal_net_lr": 7.504909282306044e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07002530321478843, |
| "setc/logratio_margin_delta": 23.66255874633789, |
| "setc/logratio_margin_vanilla": 356.75, |
| "setc/logratio_margin_weighted": 380.4125587463379, |
| "step": 1180 |
| }, |
| { |
| "entropy": 0.4537109375, |
| "epoch": 0.6413365669630827, |
| "grad_norm": 93.54570007324219, |
| "learning_rate": 3.445972162058286e-07, |
| "logits/chosen": -1.348678425603877, |
| "logits/rejected": -1.3903070843099647, |
| "logps/chosen": -526.7, |
| "logps/rejected": -908.2, |
| "loss": 0.2711, |
| "mean_token_accuracy": 0.7370507389307022, |
| "num_tokens": 42336278.0, |
| "rewards/accuracies": 0.9, |
| "rewards/chosen": -4.534214103221894, |
| "rewards/margins": 4.061459875106811, |
| "rewards/rejected": -8.595674014091491, |
| "setc/cal_net_lr": 8.534696535913519e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0704065527766943, |
| "setc/logratio_margin_delta": 27.095995330810545, |
| "setc/logratio_margin_vanilla": 379.05, |
| "setc/logratio_margin_weighted": 406.14599533081054, |
| "step": 1190 |
| }, |
| { |
| "entropy": 0.46689453125, |
| "epoch": 0.6467259498787389, |
| "grad_norm": 44.626190185546875, |
| "learning_rate": 3.356851187627665e-07, |
| "logits/chosen": -1.3881511589012898, |
| "logits/rejected": -1.4340061933201436, |
| "logps/chosen": -612.0, |
| "logps/rejected": -1049.2, |
| "loss": 0.2864, |
| "mean_token_accuracy": 0.7160806059837341, |
| "num_tokens": 42714490.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -5.426479089260101, |
| "rewards/margins": 4.657109332084656, |
| "rewards/rejected": -10.083588361740112, |
| "setc/cal_net_lr": 9.624169699392697e-05, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0706709910184145, |
| "setc/logratio_margin_delta": 29.010940551757812, |
| "setc/logratio_margin_vanilla": 436.7, |
| "setc/logratio_margin_weighted": 465.7109405517578, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.6467259498787389, |
| "eval_entropy": 0.48931884765625, |
| "eval_logits/chosen": -1.3905256226255422, |
| "eval_logits/rejected": -1.424202928967488, |
| "eval_logps/chosen": -595.625, |
| "eval_logps/rejected": -1020.375, |
| "eval_loss": 0.26192784309387207, |
| "eval_mean_token_accuracy": 0.7144393119961023, |
| "eval_num_tokens": 42714490.0, |
| "eval_rewards/accuracies": 0.90625, |
| "eval_rewards/chosen": -5.241253539919853, |
| "eval_rewards/margins": 4.524848110973835, |
| "eval_rewards/rejected": -9.766101628541946, |
| "eval_runtime": 14.2072, |
| "eval_samples_per_second": 35.193, |
| "eval_setc/cal_net_lr": 0.0001021713083937391, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.07142715808004141, |
| "eval_setc/logratio_margin_delta": 26.581721305847168, |
| "eval_setc/logratio_margin_vanilla": 426.4375, |
| "eval_setc/logratio_margin_weighted": 453.01922130584717, |
| "eval_steps_per_second": 2.252, |
| "step": 1200 |
| }, |
| { |
| "entropy": 0.48623046875, |
| "epoch": 0.652115332794395, |
| "grad_norm": 33.88088607788086, |
| "learning_rate": 3.2683116879898995e-07, |
| "logits/chosen": -1.3838917142029392, |
| "logits/rejected": -1.4172299467669451, |
| "logps/chosen": -591.7, |
| "logps/rejected": -991.4, |
| "loss": 0.2971, |
| "mean_token_accuracy": 0.7138142108917236, |
| "num_tokens": 43061800.0, |
| "rewards/accuracies": 0.8875, |
| "rewards/chosen": -5.238682866096497, |
| "rewards/margins": 4.275832235813141, |
| "rewards/rejected": -9.514515256881714, |
| "setc/cal_net_lr": 0.00010771760565312037, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07107691392302513, |
| "setc/logratio_margin_delta": 23.933246612548828, |
| "setc/logratio_margin_vanilla": 403.65, |
| "setc/logratio_margin_weighted": 427.5832466125488, |
| "step": 1210 |
| }, |
| { |
| "entropy": 0.47412109375, |
| "epoch": 0.6575047157100512, |
| "grad_norm": 24.915206909179688, |
| "learning_rate": 3.180384995357155e-07, |
| "logits/chosen": -1.3262967787135937, |
| "logits/rejected": -1.349477324233645, |
| "logps/chosen": -513.3, |
| "logps/rejected": -880.6, |
| "loss": 0.282, |
| "mean_token_accuracy": 0.7303446799516677, |
| "num_tokens": 43422934.0, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -4.445479357242585, |
| "rewards/margins": 3.892670524120331, |
| "rewards/rejected": -8.338150024414062, |
| "setc/cal_net_lr": 0.00011975817270565024, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06996985897421837, |
| "setc/logratio_margin_delta": 25.342060852050782, |
| "setc/logratio_margin_vanilla": 363.925, |
| "setc/logratio_margin_weighted": 389.2670608520508, |
| "step": 1220 |
| }, |
| { |
| "entropy": 0.47568359375, |
| "epoch": 0.6628940986257074, |
| "grad_norm": 41.575965881347656, |
| "learning_rate": 3.093102225082468e-07, |
| "logits/chosen": -1.4003033108084804, |
| "logits/rejected": -1.4641346348518416, |
| "logps/chosen": -629.7, |
| "logps/rejected": -1016.2, |
| "loss": 0.3094, |
| "mean_token_accuracy": 0.7089363127946854, |
| "num_tokens": 43787087.0, |
| "rewards/accuracies": 0.896875, |
| "rewards/chosen": -5.634258687496185, |
| "rewards/margins": 4.108717429637909, |
| "rewards/rejected": -9.742976069450378, |
| "setc/cal_net_lr": 0.00013234606674091868, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06970880702137947, |
| "setc/logratio_margin_delta": 24.44675521850586, |
| "setc/logratio_margin_vanilla": 386.425, |
| "setc/logratio_margin_weighted": 410.8717552185059, |
| "step": 1230 |
| }, |
| { |
| "entropy": 0.4681640625, |
| "epoch": 0.6682834815413635, |
| "grad_norm": 42.247772216796875, |
| "learning_rate": 3.006494264648687e-07, |
| "logits/chosen": -1.5038021364170624, |
| "logits/rejected": -1.5682670959321656, |
| "logps/chosen": -746.6, |
| "logps/rejected": -1242.8, |
| "loss": 0.3417, |
| "mean_token_accuracy": 0.6826083391904831, |
| "num_tokens": 44151449.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -6.940376424789429, |
| "rewards/margins": 5.315678989887237, |
| "rewards/rejected": -12.256055474281311, |
| "setc/cal_net_lr": 0.00014546316851594206, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07185462042689324, |
| "setc/logratio_margin_delta": 34.71790542602539, |
| "setc/logratio_margin_vanilla": 496.85, |
| "setc/logratio_margin_weighted": 531.5679054260254, |
| "step": 1240 |
| }, |
| { |
| "entropy": 0.45966796875, |
| "epoch": 0.6736728644570197, |
| "grad_norm": 16.57280731201172, |
| "learning_rate": 2.9205917627380717e-07, |
| "logits/chosen": -1.450344017268741, |
| "logits/rejected": -1.4847681129953636, |
| "logps/chosen": -670.9, |
| "logps/rejected": -1120.8, |
| "loss": 0.2943, |
| "mean_token_accuracy": 0.7004325985908508, |
| "num_tokens": 44493045.0, |
| "rewards/accuracies": 0.8625, |
| "rewards/chosen": -6.097113466262817, |
| "rewards/margins": 4.771983242034912, |
| "rewards/rejected": -10.869096612930297, |
| "setc/cal_net_lr": 0.00015909059703651744, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07053840048611164, |
| "setc/logratio_margin_delta": 30.54833221435547, |
| "setc/logratio_margin_vanilla": 446.65, |
| "setc/logratio_margin_weighted": 477.19833221435545, |
| "step": 1250 |
| }, |
| { |
| "entropy": 0.4638671875, |
| "epoch": 0.6790622473726758, |
| "grad_norm": 22.801801681518555, |
| "learning_rate": 2.8354251183863833e-07, |
| "logits/chosen": -1.4518458887210037, |
| "logits/rejected": -1.505511142314074, |
| "logps/chosen": -650.8, |
| "logps/rejected": -1089.8, |
| "loss": 0.2263, |
| "mean_token_accuracy": 0.709148183465004, |
| "num_tokens": 44862628.0, |
| "rewards/accuracies": 0.896875, |
| "rewards/chosen": -5.8279621481895445, |
| "rewards/margins": 4.705725991725922, |
| "rewards/rejected": -10.533688116073609, |
| "setc/cal_net_lr": 0.00017320873673486848, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06929908581078052, |
| "setc/logratio_margin_delta": 31.472618103027344, |
| "setc/logratio_margin_vanilla": 439.1, |
| "setc/logratio_margin_weighted": 470.57261810302737, |
| "step": 1260 |
| }, |
| { |
| "entropy": 0.45927734375, |
| "epoch": 0.684451630288332, |
| "grad_norm": 81.56884002685547, |
| "learning_rate": 2.7510244702253166e-07, |
| "logits/chosen": -1.5361301765199689, |
| "logits/rejected": -1.6149365987857098, |
| "logps/chosen": -617.4, |
| "logps/rejected": -1038.0, |
| "loss": 0.3026, |
| "mean_token_accuracy": 0.7010406374931335, |
| "num_tokens": 45189795.0, |
| "rewards/accuracies": 0.8875, |
| "rewards/chosen": -5.554684376716613, |
| "rewards/margins": 4.473910105228424, |
| "rewards/rejected": -10.028594422340394, |
| "setc/cal_net_lr": 0.0001877972657046496, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0700082466006279, |
| "setc/logratio_margin_delta": 30.791018676757812, |
| "setc/logratio_margin_vanilla": 416.6, |
| "setc/logratio_margin_weighted": 447.3910186767578, |
| "step": 1270 |
| }, |
| { |
| "entropy": 0.43818359375, |
| "epoch": 0.6898410132039882, |
| "grad_norm": 42.3488883972168, |
| "learning_rate": 2.667419685817105e-07, |
| "logits/chosen": -1.4125931971850059, |
| "logits/rejected": -1.4648399384142876, |
| "logps/chosen": -500.7, |
| "logps/rejected": -882.1, |
| "loss": 0.3085, |
| "mean_token_accuracy": 0.7451159060001373, |
| "num_tokens": 45554415.0, |
| "rewards/accuracies": 0.871875, |
| "rewards/chosen": -4.311119174957275, |
| "rewards/margins": 4.081181025505066, |
| "rewards/rejected": -8.392300033569336, |
| "setc/cal_net_lr": 0.00020283518495266647, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06918111927807331, |
| "setc/logratio_margin_delta": 28.018105697631835, |
| "setc/logratio_margin_vanilla": 380.1, |
| "setc/logratio_margin_weighted": 408.11810569763185, |
| "step": 1280 |
| }, |
| { |
| "entropy": 0.4404296875, |
| "epoch": 0.6952303961196443, |
| "grad_norm": 52.7696647644043, |
| "learning_rate": 2.5846403510850225e-07, |
| "logits/chosen": -1.5529430524670726, |
| "logits/rejected": -1.5987173557025316, |
| "logps/chosen": -522.1, |
| "logps/rejected": -968.8, |
| "loss": 0.2509, |
| "mean_token_accuracy": 0.7253134727478028, |
| "num_tokens": 45901757.0, |
| "rewards/accuracies": 0.896875, |
| "rewards/chosen": -4.538894402980804, |
| "rewards/margins": 4.75179933309555, |
| "rewards/rejected": -9.290693855285644, |
| "setc/cal_net_lr": 0.00021830084862520705, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0703603059053421, |
| "setc/logratio_margin_delta": 34.279936981201175, |
| "setc/logratio_margin_vanilla": 440.9, |
| "setc/logratio_margin_weighted": 475.1799369812012, |
| "step": 1290 |
| }, |
| { |
| "entropy": 0.44638671875, |
| "epoch": 0.7006197790353005, |
| "grad_norm": 31.051944732666016, |
| "learning_rate": 2.5027157598435777e-07, |
| "logits/chosen": -1.612937918631245, |
| "logits/rejected": -1.7009304808047285, |
| "logps/chosen": -665.4, |
| "logps/rejected": -1133.0, |
| "loss": 0.358, |
| "mean_token_accuracy": 0.7091849476099015, |
| "num_tokens": 46262002.0, |
| "rewards/accuracies": 0.8625, |
| "rewards/chosen": -5.984306502342224, |
| "rewards/margins": 5.062739598751068, |
| "rewards/rejected": -11.047045946121216, |
| "setc/cal_net_lr": 0.00023417199516547726, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07216440998017788, |
| "setc/logratio_margin_delta": 34.07397308349609, |
| "setc/logratio_margin_vanilla": 472.2, |
| "setc/logratio_margin_weighted": 506.2739730834961, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.7006197790353005, |
| "eval_entropy": 0.45013427734375, |
| "eval_logits/chosen": -1.5569858902716183, |
| "eval_logits/rejected": -1.6061105394898563, |
| "eval_logps/chosen": -589.3125, |
| "eval_logps/rejected": -1073.625, |
| "eval_loss": 0.2594917416572571, |
| "eval_mean_token_accuracy": 0.721536174416542, |
| "eval_num_tokens": 46262002.0, |
| "eval_rewards/accuracies": 0.90234375, |
| "eval_rewards/chosen": -5.197996735572815, |
| "eval_rewards/margins": 5.20976897329092, |
| "eval_rewards/rejected": -10.407765805721283, |
| "eval_runtime": 14.2113, |
| "eval_samples_per_second": 35.183, |
| "eval_setc/cal_net_lr": 0.00024264359402975666, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.0713688328396529, |
| "eval_setc/logratio_margin_delta": 34.81822967529297, |
| "eval_setc/logratio_margin_vanilla": 487.0, |
| "eval_setc/logratio_margin_weighted": 521.818229675293, |
| "eval_steps_per_second": 2.252, |
| "step": 1300 |
| }, |
| { |
| "entropy": 0.41630859375, |
| "epoch": 0.7060091619509566, |
| "grad_norm": 77.33255004882812, |
| "learning_rate": 2.4216749034320574e-07, |
| "logits/chosen": -1.5246676903481826, |
| "logits/rejected": -1.6122385093574638, |
| "logps/chosen": -567.6, |
| "logps/rejected": -984.0, |
| "loss": 0.3325, |
| "mean_token_accuracy": 0.728985533118248, |
| "num_tokens": 46607558.0, |
| "rewards/accuracies": 0.878125, |
| "rewards/chosen": -5.136111176013946, |
| "rewards/margins": 4.455406987667084, |
| "rewards/rejected": -9.591518354415893, |
| "setc/cal_net_lr": 0.0002504257793572883, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07067357562482357, |
| "setc/logratio_margin_delta": 31.340717315673828, |
| "setc/logratio_margin_vanilla": 414.2, |
| "setc/logratio_margin_weighted": 445.5407173156738, |
| "step": 1310 |
| }, |
| { |
| "entropy": 0.440625, |
| "epoch": 0.7113985448666128, |
| "grad_norm": 23.72609519958496, |
| "learning_rate": 2.341546460455126e-07, |
| "logits/chosen": -1.540273190267993, |
| "logits/rejected": -1.6056650116281355, |
| "logps/chosen": -530.0, |
| "logps/rejected": -924.0, |
| "loss": 0.2819, |
| "mean_token_accuracy": 0.7307059586048126, |
| "num_tokens": 46949311.0, |
| "rewards/accuracies": 0.9, |
| "rewards/chosen": -4.646138024330139, |
| "rewards/margins": 4.262345945835113, |
| "rewards/rejected": -8.908483982086182, |
| "setc/cal_net_lr": 0.0002670388052088759, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06920704822987318, |
| "setc/logratio_margin_delta": 31.384617614746094, |
| "setc/logratio_margin_vanilla": 394.85, |
| "setc/logratio_margin_weighted": 426.23461761474607, |
| "step": 1320 |
| }, |
| { |
| "entropy": 0.47109375, |
| "epoch": 0.716787927782269, |
| "grad_norm": 20.47443389892578, |
| "learning_rate": 2.2623587866340888e-07, |
| "logits/chosen": -1.5787250099693941, |
| "logits/rejected": -1.6378218184439, |
| "logps/chosen": -555.9, |
| "logps/rejected": -959.8, |
| "loss": 0.3328, |
| "mean_token_accuracy": 0.7292061507701874, |
| "num_tokens": 47301716.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -4.8078773021698, |
| "rewards/margins": 4.328281128406525, |
| "rewards/rejected": -9.13615837097168, |
| "setc/cal_net_lr": 0.00028398715962951504, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06858081389218569, |
| "setc/logratio_margin_delta": 31.928114318847655, |
| "setc/logratio_margin_vanilla": 400.9, |
| "setc/logratio_margin_weighted": 432.82811431884767, |
| "step": 1330 |
| }, |
| { |
| "entropy": 0.44931640625, |
| "epoch": 0.7221773106979251, |
| "grad_norm": 30.538745880126953, |
| "learning_rate": 2.1841399047724047e-07, |
| "logits/chosen": -1.52773450533801, |
| "logits/rejected": -1.58617710115334, |
| "logps/chosen": -509.1, |
| "logps/rejected": -886.6, |
| "loss": 0.2378, |
| "mean_token_accuracy": 0.7392058879137039, |
| "num_tokens": 47673875.0, |
| "rewards/accuracies": 0.9125, |
| "rewards/chosen": -4.407375812530518, |
| "rewards/margins": 4.048559367656708, |
| "rewards/rejected": -8.455935263633728, |
| "setc/cal_net_lr": 0.0003012464468504569, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06842790953814984, |
| "setc/logratio_margin_delta": 29.705943298339843, |
| "setc/logratio_margin_vanilla": 375.15, |
| "setc/logratio_margin_weighted": 404.85594329833987, |
| "step": 1340 |
| }, |
| { |
| "entropy": 0.442578125, |
| "epoch": 0.7275666936135813, |
| "grad_norm": 72.51182556152344, |
| "learning_rate": 2.1069174948390435e-07, |
| "logits/chosen": -1.584456418416361, |
| "logits/rejected": -1.6523068810983297, |
| "logps/chosen": -554.8, |
| "logps/rejected": -955.6, |
| "loss": 0.3051, |
| "mean_token_accuracy": 0.7066467195749283, |
| "num_tokens": 47987918.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -4.997482490539551, |
| "rewards/margins": 4.298661887645721, |
| "rewards/rejected": -9.296144366264343, |
| "setc/cal_net_lr": 0.00031879182354063983, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07268548682332039, |
| "setc/logratio_margin_delta": 33.21618576049805, |
| "setc/logratio_margin_vanilla": 396.65, |
| "setc/logratio_margin_weighted": 429.86618576049807, |
| "step": 1350 |
| }, |
| { |
| "entropy": 0.46162109375, |
| "epoch": 0.7329560765292374, |
| "grad_norm": 53.63888931274414, |
| "learning_rate": 2.0307188841731193e-07, |
| "logits/chosen": -1.6495496355737422, |
| "logits/rejected": -1.6986198259558798, |
| "logps/chosen": -670.8, |
| "logps/rejected": -1126.8, |
| "loss": 0.3292, |
| "mean_token_accuracy": 0.7082957863807678, |
| "num_tokens": 48363901.0, |
| "rewards/accuracies": 0.884375, |
| "rewards/chosen": -6.070537447929382, |
| "rewards/margins": 4.876463973522187, |
| "rewards/rejected": -10.947001552581787, |
| "setc/cal_net_lr": 0.0003365980345666318, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0717622920870781, |
| "setc/logratio_margin_delta": 33.046409606933594, |
| "setc/logratio_margin_vanilla": 454.6, |
| "setc/logratio_margin_weighted": 487.6464096069336, |
| "step": 1360 |
| }, |
| { |
| "entropy": 0.45615234375, |
| "epoch": 0.7383454594448936, |
| "grad_norm": 68.25013732910156, |
| "learning_rate": 1.9555710378133584e-07, |
| "logits/chosen": -1.5475741416931696, |
| "logits/rejected": -1.5981262042934623, |
| "logps/chosen": -553.0, |
| "logps/rejected": -915.8, |
| "loss": 0.3524, |
| "mean_token_accuracy": 0.719152620434761, |
| "num_tokens": 48721520.0, |
| "rewards/accuracies": 0.89375, |
| "rewards/chosen": -4.9144437432289125, |
| "rewards/margins": 3.8710513949394225, |
| "rewards/rejected": -8.785495042800903, |
| "setc/cal_net_lr": 0.00035463944934532626, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07162475697696209, |
| "setc/logratio_margin_delta": 26.60514907836914, |
| "setc/logratio_margin_vanilla": 360.5, |
| "setc/logratio_margin_weighted": 387.10514907836915, |
| "step": 1370 |
| }, |
| { |
| "entropy": 0.47626953125, |
| "epoch": 0.7437348423605498, |
| "grad_norm": 27.951370239257812, |
| "learning_rate": 1.88150054895574e-07, |
| "logits/chosen": -1.5752492477710134, |
| "logits/rejected": -1.6267678817590063, |
| "logps/chosen": -530.9, |
| "logps/rejected": -860.2, |
| "loss": 0.331, |
| "mean_token_accuracy": 0.7230644017457962, |
| "num_tokens": 49073907.0, |
| "rewards/accuracies": 0.878125, |
| "rewards/chosen": -4.60977201461792, |
| "rewards/margins": 3.515028989315033, |
| "rewards/rejected": -8.12480103969574, |
| "setc/cal_net_lr": 0.00037289009873706997, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0722591146826744, |
| "setc/logratio_margin_delta": 24.0529052734375, |
| "setc/logratio_margin_vanilla": 327.45, |
| "setc/logratio_margin_weighted": 351.5029052734375, |
| "step": 1380 |
| }, |
| { |
| "entropy": 0.4615234375, |
| "epoch": 0.7491242252762059, |
| "grad_norm": 55.398651123046875, |
| "learning_rate": 1.808533629542751e-07, |
| "logits/chosen": -1.519858605305458, |
| "logits/rejected": -1.6093533244431188, |
| "logps/chosen": -509.7, |
| "logps/rejected": -855.6, |
| "loss": 0.3028, |
| "mean_token_accuracy": 0.7340264290571212, |
| "num_tokens": 49418680.0, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -4.401390933990479, |
| "rewards/margins": 3.69234881401062, |
| "rewards/rejected": -8.093739652633667, |
| "setc/cal_net_lr": 0.0003913237124261124, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.071388328820467, |
| "setc/logratio_margin_delta": 25.53488998413086, |
| "setc/logratio_margin_vanilla": 343.7, |
| "setc/logratio_margin_weighted": 369.23488998413086, |
| "step": 1390 |
| }, |
| { |
| "entropy": 0.47451171875, |
| "epoch": 0.7545136081918621, |
| "grad_norm": 20.171005249023438, |
| "learning_rate": 1.736696100987543e-07, |
| "logits/chosen": -1.6546586863128225, |
| "logits/rejected": -1.7063404244684879, |
| "logps/chosen": -534.9, |
| "logps/rejected": -856.2, |
| "loss": 0.3403, |
| "mean_token_accuracy": 0.7271544426679611, |
| "num_tokens": 49772876.0, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -4.6236349701881405, |
| "rewards/margins": 3.4730340123176573, |
| "rewards/rejected": -8.096669101715088, |
| "setc/cal_net_lr": 0.0004099137567345749, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07029190417379141, |
| "setc/logratio_margin_delta": 24.003412628173827, |
| "setc/logratio_margin_vanilla": 323.3, |
| "setc/logratio_margin_weighted": 347.3034126281738, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.7545136081918621, |
| "eval_entropy": 0.4881591796875, |
| "eval_logits/chosen": -1.5865094371675443, |
| "eval_logits/rejected": -1.6286456849256612, |
| "eval_logps/chosen": -503.875, |
| "eval_logps/rejected": -873.75, |
| "eval_loss": 0.25063228607177734, |
| "eval_mean_token_accuracy": 0.7336005866527557, |
| "eval_num_tokens": 49772876.0, |
| "eval_rewards/accuracies": 0.9140625, |
| "eval_rewards/chosen": -4.2661414965987206, |
| "eval_rewards/margins": 3.9734192714095116, |
| "eval_rewards/rejected": -8.239560678601265, |
| "eval_runtime": 14.2136, |
| "eval_samples_per_second": 35.178, |
| "eval_setc/cal_net_lr": 0.00041972234003674093, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.06988485460169613, |
| "eval_setc/logratio_margin_delta": 25.711304664611816, |
| "eval_setc/logratio_margin_vanilla": 373.5625, |
| "eval_setc/logratio_margin_weighted": 399.2738046646118, |
| "eval_steps_per_second": 2.251, |
| "step": 1400 |
| }, |
| { |
| "entropy": 0.46767578125, |
| "epoch": 0.7599029911075182, |
| "grad_norm": 59.77590560913086, |
| "learning_rate": 1.6660133850362878e-07, |
| "logits/chosen": -1.5703001768437603, |
| "logits/rejected": -1.6146571741836044, |
| "logps/chosen": -567.2, |
| "logps/rejected": -932.8, |
| "loss": 0.3168, |
| "mean_token_accuracy": 0.7329632073640824, |
| "num_tokens": 50163681.0, |
| "rewards/accuracies": 0.884375, |
| "rewards/chosen": -4.9076042652130125, |
| "rewards/margins": 3.9311932802200316, |
| "rewards/rejected": -8.838797640800475, |
| "setc/cal_net_lr": 0.00042863347281550787, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07002903185784817, |
| "setc/logratio_margin_delta": 26.31934051513672, |
| "setc/logratio_margin_vanilla": 366.8, |
| "setc/logratio_margin_weighted": 393.11934051513674, |
| "step": 1410 |
| }, |
| { |
| "entropy": 0.46171875, |
| "epoch": 0.7652923740231743, |
| "grad_norm": 56.6906852722168, |
| "learning_rate": 1.5965104947719816e-07, |
| "logits/chosen": -1.5351415663599084, |
| "logits/rejected": -1.622101327043853, |
| "logps/chosen": -546.1, |
| "logps/rejected": -945.4, |
| "loss": 0.2704, |
| "mean_token_accuracy": 0.7306698024272918, |
| "num_tokens": 50536109.0, |
| "rewards/accuracies": 0.896875, |
| "rewards/chosen": -4.773424637317658, |
| "rewards/margins": 4.259200811386108, |
| "rewards/rejected": -9.032625222206116, |
| "setc/cal_net_lr": 0.0004474559151700609, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07033420763909817, |
| "setc/logratio_margin_delta": 29.820078659057618, |
| "setc/logratio_margin_vanilla": 396.1, |
| "setc/logratio_margin_weighted": 425.9200786590576, |
| "step": 1420 |
| }, |
| { |
| "entropy": 0.4712890625, |
| "epoch": 0.7706817569388305, |
| "grad_norm": 27.157922744750977, |
| "learning_rate": 1.5282120257628493e-07, |
| "logits/chosen": -1.5769850546803867, |
| "logits/rejected": -1.675950265945526, |
| "logps/chosen": -572.0, |
| "logps/rejected": -961.6, |
| "loss": 0.3383, |
| "mean_token_accuracy": 0.7177787780761719, |
| "num_tokens": 50898774.0, |
| "rewards/accuracies": 0.85, |
| "rewards/chosen": -5.021525096893311, |
| "rewards/margins": 4.185178649425507, |
| "rewards/rejected": -9.206703734397887, |
| "setc/cal_net_lr": 0.00046635399043332197, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.07177232243120671, |
| "setc/logratio_margin_delta": 28.917872619628906, |
| "setc/logratio_margin_vanilla": 389.6, |
| "setc/logratio_margin_weighted": 418.5178726196289, |
| "step": 1430 |
| }, |
| { |
| "entropy": 0.46376953125, |
| "epoch": 0.7760711398544866, |
| "grad_norm": 40.695457458496094, |
| "learning_rate": 1.46114214735851e-07, |
| "logits/chosen": -1.6385950412691188, |
| "logits/rejected": -1.686338215651635, |
| "logps/chosen": -551.6, |
| "logps/rejected": -926.2, |
| "loss": 0.3597, |
| "mean_token_accuracy": 0.7252315312623978, |
| "num_tokens": 51264449.0, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -4.829545390605927, |
| "rewards/margins": 3.9724243760108946, |
| "rewards/rejected": -8.801969718933105, |
| "setc/cal_net_lr": 0.00048530049637299856, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06923934705555439, |
| "setc/logratio_margin_delta": 24.39244613647461, |
| "setc/logratio_margin_vanilla": 372.85, |
| "setc/logratio_margin_weighted": 397.2424461364746, |
| "step": 1440 |
| }, |
| { |
| "entropy": 0.4931640625, |
| "epoch": 0.7814605227701428, |
| "grad_norm": 35.001766204833984, |
| "learning_rate": 1.3953245941369606e-07, |
| "logits/chosen": -1.5720843194136846, |
| "logits/rejected": -1.6047685664358426, |
| "logps/chosen": -596.7, |
| "logps/rejected": -962.6, |
| "loss": 0.4049, |
| "mean_token_accuracy": 0.7161958605051041, |
| "num_tokens": 51627142.0, |
| "rewards/accuracies": 0.86875, |
| "rewards/chosen": -5.245314371585846, |
| "rewards/margins": 3.897673261165619, |
| "rewards/rejected": -9.142987942695617, |
| "setc/cal_net_lr": 0.0005042681610448041, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06911584474146366, |
| "setc/logratio_margin_delta": 24.167342376708984, |
| "setc/logratio_margin_vanilla": 365.6, |
| "setc/logratio_margin_weighted": 389.767342376709, |
| "step": 1450 |
| }, |
| { |
| "entropy": 0.46611328125, |
| "epoch": 0.7868499056857989, |
| "grad_norm": 25.934104919433594, |
| "learning_rate": 1.3307826575054303e-07, |
| "logits/chosen": -1.5709140280737057, |
| "logits/rejected": -1.6279985828979888, |
| "logps/chosen": -520.7, |
| "logps/rejected": -859.0, |
| "loss": 0.2908, |
| "mean_token_accuracy": 0.7397780060768128, |
| "num_tokens": 51996636.0, |
| "rewards/accuracies": 0.8875, |
| "rewards/chosen": -4.433149778842926, |
| "rewards/margins": 3.6351953089237212, |
| "rewards/rejected": -8.068345022201537, |
| "setc/cal_net_lr": 0.0005232296820481889, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06630576048046351, |
| "setc/logratio_margin_delta": 23.869540405273437, |
| "setc/logratio_margin_vanilla": 339.65, |
| "setc/logratio_margin_weighted": 363.51954040527346, |
| "step": 1460 |
| }, |
| { |
| "entropy": 0.4646484375, |
| "epoch": 0.7922392886014551, |
| "grad_norm": 109.72233581542969, |
| "learning_rate": 1.267539177458053e-07, |
| "logits/chosen": -1.5302456922524228, |
| "logits/rejected": -1.5671663859152356, |
| "logps/chosen": -500.8, |
| "logps/rejected": -826.2, |
| "loss": 0.3222, |
| "mean_token_accuracy": 0.7304188579320907, |
| "num_tokens": 52363307.0, |
| "rewards/accuracies": 0.871875, |
| "rewards/chosen": -4.318540203571319, |
| "rewards/margins": 3.475985234975815, |
| "rewards/rejected": -7.794525456428528, |
| "setc/cal_net_lr": 0.0005421577658259117, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06553097013384104, |
| "setc/logratio_margin_delta": 22.448529052734376, |
| "setc/logratio_margin_vanilla": 325.15, |
| "setc/logratio_margin_weighted": 347.59852905273436, |
| "step": 1470 |
| }, |
| { |
| "entropy": 0.46298828125, |
| "epoch": 0.7976286715171113, |
| "grad_norm": 28.780488967895508, |
| "learning_rate": 1.2056165344932827e-07, |
| "logits/chosen": -1.4931537332170401, |
| "logits/rejected": -1.569671151563729, |
| "logps/chosen": -528.1, |
| "logps/rejected": -940.6, |
| "loss": 0.2665, |
| "mean_token_accuracy": 0.7405376106500625, |
| "num_tokens": 52725552.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -4.563779902458191, |
| "rewards/margins": 4.378486478328705, |
| "rewards/rejected": -8.942266464233398, |
| "setc/cal_net_lr": 0.0005610251669508799, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06495412047952413, |
| "setc/logratio_margin_delta": 26.19866180419922, |
| "setc/logratio_margin_vanilla": 411.65, |
| "setc/logratio_margin_weighted": 437.8486618041992, |
| "step": 1480 |
| }, |
| { |
| "entropy": 0.44052734375, |
| "epoch": 0.8030180544327674, |
| "grad_norm": 62.47037124633789, |
| "learning_rate": 1.1450366416939295e-07, |
| "logits/chosen": -1.611906114241489, |
| "logits/rejected": -1.6829238200447194, |
| "logps/chosen": -531.7, |
| "logps/rejected": -886.6, |
| "loss": 0.286, |
| "mean_token_accuracy": 0.7277172565460205, |
| "num_tokens": 53058665.0, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -4.697575926780701, |
| "rewards/margins": 3.8217047095298766, |
| "rewards/rejected": -8.519280743598937, |
| "setc/cal_net_lr": 0.000579804727343712, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06567572541534901, |
| "setc/logratio_margin_delta": 26.470491790771483, |
| "setc/logratio_margin_vanilla": 355.7, |
| "setc/logratio_margin_weighted": 382.1704917907715, |
| "step": 1490 |
| }, |
| { |
| "entropy": 0.46953125, |
| "epoch": 0.8084074373484236, |
| "grad_norm": 42.8690185546875, |
| "learning_rate": 1.0858209369725851e-07, |
| "logits/chosen": -1.5465363961992153, |
| "logits/rejected": -1.6092895249530645, |
| "logps/chosen": -617.6, |
| "logps/rejected": -1037.6, |
| "loss": 0.2758, |
| "mean_token_accuracy": 0.7125173568725586, |
| "num_tokens": 53427519.0, |
| "rewards/accuracies": 0.88125, |
| "rewards/chosen": -5.522391545772552, |
| "rewards/margins": 4.461647641658783, |
| "rewards/rejected": -9.984039378166198, |
| "setc/cal_net_lr": 0.0005984694153645696, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06612777952104806, |
| "setc/logratio_margin_delta": 29.16478729248047, |
| "setc/logratio_margin_vanilla": 417.0, |
| "setc/logratio_margin_weighted": 446.16478729248047, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.8084074373484236, |
| "eval_entropy": 0.4835205078125, |
| "eval_logits/chosen": -1.574804083909467, |
| "eval_logits/rejected": -1.6192383259931051, |
| "eval_logps/chosen": -594.5625, |
| "eval_logps/rejected": -1023.125, |
| "eval_loss": 0.24590468406677246, |
| "eval_mean_token_accuracy": 0.7153119649738073, |
| "eval_num_tokens": 53427519.0, |
| "eval_rewards/accuracies": 0.908203125, |
| "eval_rewards/chosen": -5.237039923667908, |
| "eval_rewards/margins": 4.583509214222431, |
| "eval_rewards/rejected": -9.820549130439758, |
| "eval_runtime": 14.2433, |
| "eval_samples_per_second": 35.104, |
| "eval_setc/cal_net_lr": 0.0006082198069690511, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.06576518306974322, |
| "eval_setc/logratio_margin_delta": 28.384427070617676, |
| "eval_setc/logratio_margin_vanilla": 431.625, |
| "eval_setc/logratio_margin_weighted": 460.0094270706177, |
| "eval_steps_per_second": 2.247, |
| "step": 1500 |
| }, |
| { |
| "entropy": 0.4947265625, |
| "epoch": 0.8137968202640797, |
| "grad_norm": 25.9521427154541, |
| "learning_rate": 1.0279903754852164e-07, |
| "logits/chosen": -1.5614659093580356, |
| "logits/rejected": -1.6606548524269127, |
| "logps/chosen": -632.0, |
| "logps/rejected": -1072.0, |
| "loss": 0.2877, |
| "mean_token_accuracy": 0.7104504853487015, |
| "num_tokens": 53792050.0, |
| "rewards/accuracies": 0.9125, |
| "rewards/chosen": -5.629306566715241, |
| "rewards/margins": 4.753253519535065, |
| "rewards/rejected": -10.382560396194458, |
| "setc/cal_net_lr": 0.0006169923647229892, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0657088104635477, |
| "setc/logratio_margin_delta": 28.475372314453125, |
| "setc/logratio_margin_vanilla": 446.85, |
| "setc/logratio_margin_weighted": 475.32537231445315, |
| "step": 1510 |
| }, |
| { |
| "entropy": 0.484375, |
| "epoch": 0.8191862031797359, |
| "grad_norm": 33.998130798339844, |
| "learning_rate": 9.715654222155811e-08, |
| "logits/chosen": -1.5534200931238813, |
| "logits/rejected": -1.5930939373874997, |
| "logps/chosen": -587.5, |
| "logps/rejected": -1001.8, |
| "loss": 0.3289, |
| "mean_token_accuracy": 0.72098990380764, |
| "num_tokens": 54136636.0, |
| "rewards/accuracies": 0.878125, |
| "rewards/chosen": -5.229832494258881, |
| "rewards/margins": 4.401166546344757, |
| "rewards/rejected": -9.63099913597107, |
| "setc/cal_net_lr": 0.0006353469131497097, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06840050462633371, |
| "setc/logratio_margin_delta": 29.116664123535156, |
| "setc/logratio_margin_vanilla": 411.0, |
| "setc/logratio_margin_weighted": 440.11666412353514, |
| "step": 1520 |
| }, |
| { |
| "entropy": 0.4716796875, |
| "epoch": 0.824575586095392, |
| "grad_norm": 50.3288688659668, |
| "learning_rate": 9.16566044733122e-08, |
| "logits/chosen": -1.5786561767257699, |
| "logits/rejected": -1.642224351067974, |
| "logps/chosen": -626.7, |
| "logps/rejected": -1025.2, |
| "loss": 0.2522, |
| "mean_token_accuracy": 0.7102588206529618, |
| "num_tokens": 54516451.0, |
| "rewards/accuracies": 0.896875, |
| "rewards/chosen": -5.588171422481537, |
| "rewards/margins": 4.30382000207901, |
| "rewards/rejected": -9.891991662979127, |
| "setc/cal_net_lr": 0.0006535066407748258, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0688111089169979, |
| "setc/logratio_margin_delta": 28.632022094726562, |
| "setc/logratio_margin_vanilla": 401.75, |
| "setc/logratio_margin_weighted": 430.38202209472655, |
| "step": 1530 |
| }, |
| { |
| "entropy": 0.4650390625, |
| "epoch": 0.8299649690110482, |
| "grad_norm": 61.28715515136719, |
| "learning_rate": 8.630117061268733e-08, |
| "logits/chosen": -1.6015775772223662, |
| "logits/rejected": -1.6506740047739197, |
| "logps/chosen": -673.1, |
| "logps/rejected": -1067.2, |
| "loss": 0.3355, |
| "mean_token_accuracy": 0.7012999951839447, |
| "num_tokens": 54867398.0, |
| "rewards/accuracies": 0.8625, |
| "rewards/chosen": -6.10805311203003, |
| "rewards/margins": 4.206076884269715, |
| "rewards/rejected": -10.314130067825317, |
| "setc/cal_net_lr": 0.0006714454081570259, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06854588873684406, |
| "setc/logratio_margin_delta": 26.507701110839843, |
| "setc/logratio_margin_vanilla": 394.1, |
| "setc/logratio_margin_weighted": 420.60770111083986, |
| "step": 1540 |
| }, |
| { |
| "entropy": 0.46708984375, |
| "epoch": 0.8353543519267044, |
| "grad_norm": 66.6949234008789, |
| "learning_rate": 8.109213581178897e-08, |
| "logits/chosen": -1.519720828447748, |
| "logits/rejected": -1.5808756968990825, |
| "logps/chosen": -617.7, |
| "logps/rejected": -967.2, |
| "loss": 0.3575, |
| "mean_token_accuracy": 0.7194855481386184, |
| "num_tokens": 55239887.0, |
| "rewards/accuracies": 0.9, |
| "rewards/chosen": -5.524090123176575, |
| "rewards/margins": 3.7780436635017396, |
| "rewards/rejected": -9.302133798599243, |
| "setc/cal_net_lr": 0.0006891373939091811, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0681806318461895, |
| "setc/logratio_margin_delta": 26.4543701171875, |
| "setc/logratio_margin_vanilla": 351.35, |
| "setc/logratio_margin_weighted": 377.8043701171875, |
| "step": 1550 |
| }, |
| { |
| "entropy": 0.4662109375, |
| "epoch": 0.8407437348423605, |
| "grad_norm": 27.215456008911133, |
| "learning_rate": 7.603134343526502e-08, |
| "logits/chosen": -1.5236826772256906, |
| "logits/rejected": -1.570188853218213, |
| "logps/chosen": -587.4, |
| "logps/rejected": -975.4, |
| "loss": 0.233, |
| "mean_token_accuracy": 0.7267722725868225, |
| "num_tokens": 55609384.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -5.191406297683716, |
| "rewards/margins": 4.120009970664978, |
| "rewards/rejected": -9.311416292190552, |
| "setc/cal_net_lr": 0.0007065571318661164, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06620657239109277, |
| "setc/logratio_margin_delta": 27.050999450683594, |
| "setc/logratio_margin_vanilla": 384.95, |
| "setc/logratio_margin_weighted": 412.0009994506836, |
| "step": 1560 |
| }, |
| { |
| "entropy": 0.46328125, |
| "epoch": 0.8461331177580167, |
| "grad_norm": 47.70095443725586, |
| "learning_rate": 7.112058438797858e-08, |
| "logits/chosen": -1.5296358009335131, |
| "logits/rejected": -1.5978488752687707, |
| "logps/chosen": -614.6, |
| "logps/rejected": -1019.2, |
| "loss": 0.2631, |
| "mean_token_accuracy": 0.7203928977251053, |
| "num_tokens": 55970954.0, |
| "rewards/accuracies": 0.903125, |
| "rewards/chosen": -5.481227231025696, |
| "rewards/margins": 4.321574485301971, |
| "rewards/rejected": -9.80280179977417, |
| "setc/cal_net_lr": 0.0007236795477410702, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06795285735279322, |
| "setc/logratio_margin_delta": 27.957456970214842, |
| "setc/logratio_margin_vanilla": 404.2, |
| "setc/logratio_margin_weighted": 432.15745697021487, |
| "step": 1570 |
| }, |
| { |
| "entropy": 0.46845703125, |
| "epoch": 0.8515225006736729, |
| "grad_norm": 36.03289794921875, |
| "learning_rate": 6.636159648124557e-08, |
| "logits/chosen": -1.5813003009716322, |
| "logits/rejected": -1.6186931994037377, |
| "logps/chosen": -607.3, |
| "logps/rejected": -967.4, |
| "loss": 0.3036, |
| "mean_token_accuracy": 0.7082692325115204, |
| "num_tokens": 56337079.0, |
| "rewards/accuracies": 0.85, |
| "rewards/chosen": -5.434537315368653, |
| "rewards/margins": 3.795243227481842, |
| "rewards/rejected": -9.229780340194703, |
| "setc/cal_net_lr": 0.0007404799952180797, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06792235635221004, |
| "setc/logratio_margin_delta": 25.124325561523438, |
| "setc/logratio_margin_vanilla": 354.4, |
| "setc/logratio_margin_weighted": 379.5243255615234, |
| "step": 1580 |
| }, |
| { |
| "entropy": 0.4748046875, |
| "epoch": 0.856911883589329, |
| "grad_norm": 24.535364151000977, |
| "learning_rate": 6.175606381786069e-08, |
| "logits/chosen": -1.5430637306828987, |
| "logits/rejected": -1.6079818626695932, |
| "logps/chosen": -650.9, |
| "logps/rejected": -1084.8, |
| "loss": 0.2913, |
| "mean_token_accuracy": 0.7224538773298264, |
| "num_tokens": 56731554.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -5.72764368057251, |
| "rewards/margins": 4.639845299720764, |
| "rewards/rejected": -10.367488980293274, |
| "setc/cal_net_lr": 0.0007569342914283324, |
| "setc/credit_mean": 0.9999999970197677, |
| "setc/credit_std": 0.06582386922091246, |
| "setc/logratio_margin_delta": 29.334539794921874, |
| "setc/logratio_margin_vanilla": 434.65, |
| "setc/logratio_margin_weighted": 463.9845397949219, |
| "step": 1590 |
| }, |
| { |
| "entropy": 0.48642578125, |
| "epoch": 0.8623012665049852, |
| "grad_norm": 79.2051773071289, |
| "learning_rate": 5.7305616196130334e-08, |
| "logits/chosen": -1.6237664047229114, |
| "logits/rejected": -1.6430690136847275, |
| "logps/chosen": -653.3, |
| "logps/rejected": -1039.8, |
| "loss": 0.2893, |
| "mean_token_accuracy": 0.7086077839136123, |
| "num_tokens": 57110285.0, |
| "rewards/accuracies": 0.85625, |
| "rewards/chosen": -5.711839783191681, |
| "rewards/margins": 4.166626226902008, |
| "rewards/rejected": -9.87846598625183, |
| "setc/cal_net_lr": 0.0007730187517594288, |
| "setc/credit_mean": 0.9999999970197677, |
| "setc/credit_std": 0.06541631631553173, |
| "setc/logratio_margin_delta": 25.612635803222656, |
| "setc/logratio_margin_vanilla": 391.05, |
| "setc/logratio_margin_weighted": 416.66263580322266, |
| "step": 1600 |
| }, |
| { |
| "epoch": 0.8623012665049852, |
| "eval_entropy": 0.4923095703125, |
| "eval_logits/chosen": -1.603833879515406, |
| "eval_logits/rejected": -1.6448864097538396, |
| "eval_logps/chosen": -600.5, |
| "eval_logps/rejected": -1020.375, |
| "eval_loss": 0.24327774345874786, |
| "eval_mean_token_accuracy": 0.7129223179072142, |
| "eval_num_tokens": 57110285.0, |
| "eval_rewards/accuracies": 0.916015625, |
| "eval_rewards/chosen": -5.2806950733065605, |
| "eval_rewards/margins": 4.483514308929443, |
| "eval_rewards/rejected": -9.764209300279617, |
| "eval_runtime": 14.2982, |
| "eval_samples_per_second": 34.969, |
| "eval_setc/cal_net_lr": 0.0007813240527369953, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.06434770347550511, |
| "eval_setc/logratio_margin_delta": 26.474138259887695, |
| "eval_setc/logratio_margin_vanilla": 423.375, |
| "eval_setc/logratio_margin_weighted": 449.8491382598877, |
| "eval_steps_per_second": 2.238, |
| "step": 1600 |
| }, |
| { |
| "entropy": 0.48271484375, |
| "epoch": 0.8676906494206413, |
| "grad_norm": 70.79142761230469, |
| "learning_rate": 5.3011828533122024e-08, |
| "logits/chosen": -1.6149875326285763, |
| "logits/rejected": -1.6959871557083672, |
| "logps/chosen": -639.3, |
| "logps/rejected": -1038.2, |
| "loss": 0.2599, |
| "mean_token_accuracy": 0.713002547621727, |
| "num_tokens": 57472462.0, |
| "rewards/accuracies": 0.878125, |
| "rewards/chosen": -5.632159161567688, |
| "rewards/margins": 4.25579309463501, |
| "rewards/rejected": -9.887952256202698, |
| "setc/cal_net_lr": 0.0007887102239474405, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06394433472305536, |
| "setc/logratio_margin_delta": 26.4293212890625, |
| "setc/logratio_margin_vanilla": 399.15, |
| "setc/logratio_margin_weighted": 425.5793212890625, |
| "step": 1610 |
| }, |
| { |
| "entropy": 0.475, |
| "epoch": 0.8730800323362975, |
| "grad_norm": 39.525123596191406, |
| "learning_rate": 4.887622030733507e-08, |
| "logits/chosen": -1.5679981820658682, |
| "logits/rejected": -1.6499297438183103, |
| "logps/chosen": -620.7, |
| "logps/rejected": -1020.0, |
| "loss": 0.2992, |
| "mean_token_accuracy": 0.7138593286275864, |
| "num_tokens": 57826454.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -5.5272397756576535, |
| "rewards/margins": 4.316784369945526, |
| "rewards/rejected": -9.84402413368225, |
| "setc/cal_net_lr": 0.0008039861214027002, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.060222255811095235, |
| "setc/logratio_margin_delta": 27.97843780517578, |
| "setc/logratio_margin_vanilla": 403.7, |
| "setc/logratio_margin_weighted": 431.6784378051758, |
| "step": 1620 |
| }, |
| { |
| "entropy": 0.4849609375, |
| "epoch": 0.8784694152519537, |
| "grad_norm": 50.614253997802734, |
| "learning_rate": 4.4900255020990764e-08, |
| "logits/chosen": -1.554491747697552, |
| "logits/rejected": -1.6372942528114076, |
| "logps/chosen": -619.1, |
| "logps/rejected": -1022.0, |
| "loss": 0.3179, |
| "mean_token_accuracy": 0.7143799781799316, |
| "num_tokens": 58225092.0, |
| "rewards/accuracies": 0.871875, |
| "rewards/chosen": -5.4253825664520265, |
| "rewards/margins": 4.251791071891785, |
| "rewards/rejected": -9.677173686027526, |
| "setc/cal_net_lr": 0.0008188244557213483, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.05570842530578375, |
| "setc/logratio_margin_delta": 23.57911911010742, |
| "setc/logratio_margin_vanilla": 401.6, |
| "setc/logratio_margin_weighted": 425.1791191101074, |
| "step": 1630 |
| }, |
| { |
| "entropy": 0.47099609375, |
| "epoch": 0.8838587981676098, |
| "grad_norm": 33.20830535888672, |
| "learning_rate": 4.108533968212963e-08, |
| "logits/chosen": -1.5388461767468533, |
| "logits/rejected": -1.583654628206283, |
| "logps/chosen": -595.7, |
| "logps/rejected": -1035.6, |
| "loss": 0.2626, |
| "mean_token_accuracy": 0.713904058933258, |
| "num_tokens": 58584790.0, |
| "rewards/accuracies": 0.896875, |
| "rewards/chosen": -5.293009197711944, |
| "rewards/margins": 4.6426942348480225, |
| "rewards/rejected": -9.93570351600647, |
| "setc/cal_net_lr": 0.000833203868335841, |
| "setc/credit_mean": 0.9999999970197677, |
| "setc/credit_std": 0.05715073775500059, |
| "setc/logratio_margin_delta": 29.46942901611328, |
| "setc/logratio_margin_vanilla": 434.8, |
| "setc/logratio_margin_weighted": 464.2694290161133, |
| "step": 1640 |
| }, |
| { |
| "entropy": 0.4619140625, |
| "epoch": 0.889248181083266, |
| "grad_norm": 49.0516242980957, |
| "learning_rate": 3.743282430670286e-08, |
| "logits/chosen": -1.5955134523755936, |
| "logits/rejected": -1.631971502049342, |
| "logps/chosen": -559.5, |
| "logps/rejected": -978.0, |
| "loss": 0.278, |
| "mean_token_accuracy": 0.7260301500558853, |
| "num_tokens": 58947907.0, |
| "rewards/accuracies": 0.88125, |
| "rewards/chosen": -4.951177608966828, |
| "rewards/margins": 4.426893401145935, |
| "rewards/rejected": -9.378070974349976, |
| "setc/cal_net_lr": 0.0008471036612588582, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.057837378792464735, |
| "setc/logratio_margin_delta": 26.0393424987793, |
| "setc/logratio_margin_vanilla": 416.65, |
| "setc/logratio_margin_weighted": 442.6893424987793, |
| "step": 1650 |
| }, |
| { |
| "entropy": 0.476953125, |
| "epoch": 0.8946375639989221, |
| "grad_norm": 61.91923141479492, |
| "learning_rate": 3.3944001440829986e-08, |
| "logits/chosen": -1.5750558117502387, |
| "logits/rejected": -1.657317549975772, |
| "logps/chosen": -623.0, |
| "logps/rejected": -1002.8, |
| "loss": 0.3331, |
| "mean_token_accuracy": 0.7019858151674271, |
| "num_tokens": 59303281.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -5.602843141555786, |
| "rewards/margins": 4.078434026241302, |
| "rewards/rejected": -9.681277179718018, |
| "setc/cal_net_lr": 0.0008605038268763623, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.059012738056480885, |
| "setc/logratio_margin_delta": 26.393409729003906, |
| "setc/logratio_margin_vanilla": 381.45, |
| "setc/logratio_margin_weighted": 407.8434097290039, |
| "step": 1660 |
| }, |
| { |
| "entropy": 0.4888671875, |
| "epoch": 0.9000269469145783, |
| "grad_norm": 40.752716064453125, |
| "learning_rate": 3.0620105703395075e-08, |
| "logits/chosen": -1.6270705285389657, |
| "logits/rejected": -1.6553936166815, |
| "logps/chosen": -588.7, |
| "logps/rejected": -1051.8, |
| "loss": 0.2932, |
| "mean_token_accuracy": 0.7158574312925339, |
| "num_tokens": 59687228.0, |
| "rewards/accuracies": 0.88125, |
| "rewards/chosen": -5.1385871052742, |
| "rewards/margins": 4.8742870688438416, |
| "rewards/rejected": -10.012874150276184, |
| "setc/cal_net_lr": 0.0008733850767469216, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.05472625363618135, |
| "setc/logratio_margin_delta": 24.0787109375, |
| "setc/logratio_margin_vanilla": 463.35, |
| "setc/logratio_margin_weighted": 487.4287109375, |
| "step": 1670 |
| }, |
| { |
| "entropy": 0.47392578125, |
| "epoch": 0.9054163298302345, |
| "grad_norm": 57.972511291503906, |
| "learning_rate": 2.7462313349142063e-08, |
| "logits/chosen": -1.5235790048606868, |
| "logits/rejected": -1.5691862969507655, |
| "logps/chosen": -614.3, |
| "logps/rejected": -1054.2, |
| "loss": 0.2667, |
| "mean_token_accuracy": 0.7157090991735459, |
| "num_tokens": 60046942.0, |
| "rewards/accuracies": 0.878125, |
| "rewards/chosen": -5.51959844827652, |
| "rewards/margins": 4.6187913656234745, |
| "rewards/rejected": -10.138389611244202, |
| "setc/cal_net_lr": 0.0008857288693658431, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.05935143679380417, |
| "setc/logratio_margin_delta": 24.97913818359375, |
| "setc/logratio_margin_vanilla": 436.9, |
| "setc/logratio_margin_weighted": 461.87913818359374, |
| "step": 1680 |
| }, |
| { |
| "entropy": 0.478125, |
| "epoch": 0.9108057127458906, |
| "grad_norm": 56.78584289550781, |
| "learning_rate": 2.4471741852423233e-08, |
| "logits/chosen": -1.6328370435487536, |
| "logits/rejected": -1.6685099427783272, |
| "logps/chosen": -598.0, |
| "logps/rejected": -951.4, |
| "loss": 0.3008, |
| "mean_token_accuracy": 0.7114303112030029, |
| "num_tokens": 60420768.0, |
| "rewards/accuracies": 0.88125, |
| "rewards/chosen": -5.270635271072388, |
| "rewards/margins": 3.7526545763015746, |
| "rewards/rejected": -9.023289966583253, |
| "setc/cal_net_lr": 0.0008975174368541483, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06038776412606239, |
| "setc/logratio_margin_delta": 21.315467834472656, |
| "setc/logratio_margin_vanilla": 353.95, |
| "setc/logratio_margin_weighted": 375.26546783447264, |
| "step": 1690 |
| }, |
| { |
| "entropy": 0.4927734375, |
| "epoch": 0.9161950956615468, |
| "grad_norm": 57.52949142456055, |
| "learning_rate": 2.1649449511749796e-08, |
| "logits/chosen": -1.5933153416721983, |
| "logits/rejected": -1.6209732453760946, |
| "logps/chosen": -600.1, |
| "logps/rejected": -1020.8, |
| "loss": 0.2691, |
| "mean_token_accuracy": 0.7082908779382706, |
| "num_tokens": 60787745.0, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -5.318318796157837, |
| "rewards/margins": 4.3899754106998445, |
| "rewards/rejected": -9.708294129371643, |
| "setc/cal_net_lr": 0.0009087338105339832, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.06079798210412264, |
| "setc/logratio_margin_delta": 25.297548675537108, |
| "setc/logratio_margin_vanilla": 413.7, |
| "setc/logratio_margin_weighted": 438.99754867553713, |
| "step": 1700 |
| }, |
| { |
| "epoch": 0.9161950956615468, |
| "eval_entropy": 0.49981689453125, |
| "eval_logits/chosen": -1.64129217363471, |
| "eval_logits/rejected": -1.6824805209804925, |
| "eval_logps/chosen": -589.3125, |
| "eval_logps/rejected": -1005.5, |
| "eval_loss": 0.24332645535469055, |
| "eval_mean_token_accuracy": 0.7128801997750998, |
| "eval_num_tokens": 60787745.0, |
| "eval_rewards/accuracies": 0.916015625, |
| "eval_rewards/chosen": -5.122073702514172, |
| "eval_rewards/margins": 4.437938891351223, |
| "eval_rewards/rejected": -9.560012698173523, |
| "eval_runtime": 14.1962, |
| "eval_samples_per_second": 35.221, |
| "eval_setc/cal_net_lr": 0.0009144126721245054, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.05909399001393467, |
| "eval_setc/logratio_margin_delta": 25.14619731903076, |
| "eval_setc/logratio_margin_vanilla": 420.25, |
| "eval_setc/logratio_margin_weighted": 445.39619731903076, |
| "eval_steps_per_second": 2.254, |
| "step": 1700 |
| }, |
| { |
| "entropy": 0.47744140625, |
| "epoch": 0.9215844785772029, |
| "grad_norm": 32.23965072631836, |
| "learning_rate": 1.8996435075282425e-08, |
| "logits/chosen": -1.589121973571888, |
| "logits/rejected": -1.6433100488728811, |
| "logps/chosen": -585.4, |
| "logps/rejected": -1005.6, |
| "loss": 0.2847, |
| "mean_token_accuracy": 0.7224965184926987, |
| "num_tokens": 61140539.0, |
| "rewards/accuracies": 0.896875, |
| "rewards/chosen": -5.167207801342011, |
| "rewards/margins": 4.406006598472596, |
| "rewards/rejected": -9.573214316368103, |
| "setc/cal_net_lr": 0.0009193618453536389, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.05848235320299864, |
| "setc/logratio_margin_delta": 24.300669860839843, |
| "setc/logratio_margin_vanilla": 416.3, |
| "setc/logratio_margin_weighted": 440.60066986083984, |
| "step": 1710 |
| }, |
| { |
| "entropy": 0.49404296875, |
| "epoch": 0.9269738614928591, |
| "grad_norm": 104.77435302734375, |
| "learning_rate": 1.6513637387396138e-08, |
| "logits/chosen": -1.5879688472104307, |
| "logits/rejected": -1.6660038253212073, |
| "logps/chosen": -603.2, |
| "logps/rejected": -1010.6, |
| "loss": 0.3055, |
| "mean_token_accuracy": 0.7246152043342591, |
| "num_tokens": 61515724.0, |
| "rewards/accuracies": 0.88125, |
| "rewards/chosen": -5.219302845001221, |
| "rewards/margins": 4.304101645946503, |
| "rewards/rejected": -9.5234046459198, |
| "setc/cal_net_lr": 0.0009293862431270326, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.060384664312005044, |
| "setc/logratio_margin_delta": 24.260169982910156, |
| "setc/logratio_margin_vanilla": 406.15, |
| "setc/logratio_margin_weighted": 430.41016998291013, |
| "step": 1720 |
| }, |
| { |
| "entropy": 0.47578125, |
| "epoch": 0.9323632444085153, |
| "grad_norm": 62.20964813232422, |
| "learning_rate": 1.4201935056443681e-08, |
| "logits/chosen": -1.6094559658891932, |
| "logits/rejected": -1.6524878440333168, |
| "logps/chosen": -615.5, |
| "logps/rejected": -1023.2, |
| "loss": 0.2376, |
| "mean_token_accuracy": 0.714499345421791, |
| "num_tokens": 61880478.0, |
| "rewards/accuracies": 0.9125, |
| "rewards/chosen": -5.445954358577728, |
| "rewards/margins": 4.328825449943542, |
| "rewards/rejected": -9.774779748916625, |
| "setc/cal_net_lr": 0.000938792574554196, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.05732750911265612, |
| "setc/logratio_margin_delta": 26.48255157470703, |
| "setc/logratio_margin_vanilla": 406.4, |
| "setc/logratio_margin_weighted": 432.882551574707, |
| "step": 1730 |
| }, |
| { |
| "entropy": 0.47080078125, |
| "epoch": 0.9377526273241714, |
| "grad_norm": 26.240835189819336, |
| "learning_rate": 1.2062146143834939e-08, |
| "logits/chosen": -1.6424857519172662, |
| "logits/rejected": -1.6840320911940327, |
| "logps/chosen": -589.7, |
| "logps/rejected": -973.6, |
| "loss": 0.3045, |
| "mean_token_accuracy": 0.710444837808609, |
| "num_tokens": 62237198.0, |
| "rewards/accuracies": 0.865625, |
| "rewards/chosen": -5.231074094772339, |
| "rewards/margins": 4.04384742975235, |
| "rewards/rejected": -9.274921464920045, |
| "setc/cal_net_lr": 0.0009475672999910659, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.05614407397806644, |
| "setc/logratio_margin_delta": 23.08474655151367, |
| "setc/logratio_margin_vanilla": 381.3, |
| "setc/logratio_margin_weighted": 404.3847465515137, |
| "step": 1740 |
| }, |
| { |
| "entropy": 0.47841796875, |
| "epoch": 0.9431420102398276, |
| "grad_norm": 25.031240463256836, |
| "learning_rate": 1.009502787454264e-08, |
| "logits/chosen": -1.628094637586519, |
| "logits/rejected": -1.6946888854450202, |
| "logps/chosen": -561.6, |
| "logps/rejected": -962.4, |
| "loss": 0.2709, |
| "mean_token_accuracy": 0.7199163377285004, |
| "num_tokens": 62613336.0, |
| "rewards/accuracies": 0.871875, |
| "rewards/chosen": -4.926135575771331, |
| "rewards/margins": 4.213748967647552, |
| "rewards/rejected": -9.13988437652588, |
| "setc/cal_net_lr": 0.0009556977889387003, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.05476319156587124, |
| "setc/logratio_margin_delta": 26.124908447265625, |
| "setc/logratio_margin_vanilla": 395.25, |
| "setc/logratio_margin_weighted": 421.3749084472656, |
| "step": 1750 |
| }, |
| { |
| "entropy": 0.47578125, |
| "epoch": 0.9485313931554837, |
| "grad_norm": 33.9154052734375, |
| "learning_rate": 8.301276369136912e-09, |
| "logits/chosen": -1.5798610346038533, |
| "logits/rejected": -1.6392516028325452, |
| "logps/chosen": -602.1, |
| "logps/rejected": -1041.4, |
| "loss": 0.2397, |
| "mean_token_accuracy": 0.7180195420980453, |
| "num_tokens": 63005622.0, |
| "rewards/accuracies": 0.9, |
| "rewards/chosen": -5.272761929035187, |
| "rewards/margins": 4.6579270362854, |
| "rewards/rejected": -9.930689001083374, |
| "setc/cal_net_lr": 0.0009631723382238432, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.056856663711369035, |
| "setc/logratio_margin_delta": 27.44272232055664, |
| "setc/logratio_margin_vanilla": 438.35, |
| "setc/logratio_margin_weighted": 465.79272232055666, |
| "step": 1760 |
| }, |
| { |
| "entropy": 0.48935546875, |
| "epoch": 0.9539207760711399, |
| "grad_norm": 71.26800537109375, |
| "learning_rate": 6.68152639744346e-09, |
| "logits/chosen": -1.650358383302045, |
| "logits/rejected": -1.7143002367810154, |
| "logps/chosen": -613.1, |
| "logps/rejected": -987.6, |
| "loss": 0.2931, |
| "mean_token_accuracy": 0.7065669685602188, |
| "num_tokens": 63373491.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -5.3662453413009645, |
| "rewards/margins": 3.9946064949035645, |
| "rewards/rejected": -9.360851907730103, |
| "setc/cal_net_lr": 0.0009699801888446867, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.05508008189499378, |
| "setc/logratio_margin_delta": 24.06066436767578, |
| "setc/logratio_margin_vanilla": 375.4, |
| "setc/logratio_margin_weighted": 399.4606643676758, |
| "step": 1770 |
| }, |
| { |
| "entropy": 0.46416015625, |
| "epoch": 0.9593101589867961, |
| "grad_norm": 25.516454696655273, |
| "learning_rate": 5.236351153911855e-09, |
| "logits/chosen": -1.7053886135754528, |
| "logits/rejected": -1.753958930302091, |
| "logps/chosen": -552.3, |
| "logps/rejected": -933.8, |
| "loss": 0.2468, |
| "mean_token_accuracy": 0.7095615237951278, |
| "num_tokens": 63708180.0, |
| "rewards/accuracies": 0.884375, |
| "rewards/chosen": -4.928412961959839, |
| "rewards/margins": 4.00942040681839, |
| "rewards/rejected": -8.937833166122436, |
| "setc/cal_net_lr": 0.0009761115414575713, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.05466606095433235, |
| "setc/logratio_margin_delta": 26.192051696777344, |
| "setc/logratio_margin_vanilla": 374.75, |
| "setc/logratio_margin_weighted": 400.94205169677736, |
| "step": 1780 |
| }, |
| { |
| "entropy": 0.46875, |
| "epoch": 0.9646995419024522, |
| "grad_norm": 46.8420295715332, |
| "learning_rate": 3.966262054774794e-09, |
| "logits/chosen": -1.686366542637256, |
| "logits/rejected": -1.7361481268819614, |
| "logps/chosen": -585.3, |
| "logps/rejected": -962.2, |
| "loss": 0.2608, |
| "mean_token_accuracy": 0.7172308325767517, |
| "num_tokens": 64063107.0, |
| "rewards/accuracies": 0.8875, |
| "rewards/chosen": -5.106739115715027, |
| "rewards/margins": 4.049427050352096, |
| "rewards/rejected": -9.156166195869446, |
| "setc/cal_net_lr": 0.0009815575704823403, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.055073692835867404, |
| "setc/logratio_margin_delta": 27.29270706176758, |
| "setc/logratio_margin_vanilla": 377.65, |
| "setc/logratio_margin_weighted": 404.9427070617676, |
| "step": 1790 |
| }, |
| { |
| "entropy": 0.47333984375, |
| "epoch": 0.9700889248181084, |
| "grad_norm": 41.61103820800781, |
| "learning_rate": 2.8717085570689194e-09, |
| "logits/chosen": -1.6285714864267278, |
| "logits/rejected": -1.7007568628321508, |
| "logps/chosen": -543.7, |
| "logps/rejected": -972.0, |
| "loss": 0.2488, |
| "mean_token_accuracy": 0.7220604807138443, |
| "num_tokens": 64415287.0, |
| "rewards/accuracies": 0.915625, |
| "rewards/chosen": -4.766039276123047, |
| "rewards/margins": 4.599205017089844, |
| "rewards/rejected": -9.365244293212891, |
| "setc/cal_net_lr": 0.0009863104368060393, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.0533838925883174, |
| "setc/logratio_margin_delta": 31.27052230834961, |
| "setc/logratio_margin_vanilla": 428.65, |
| "setc/logratio_margin_weighted": 459.92052230834963, |
| "step": 1800 |
| }, |
| { |
| "epoch": 0.9700889248181084, |
| "eval_entropy": 0.496337890625, |
| "eval_logits/chosen": -1.6497335267505044, |
| "eval_logits/rejected": -1.691259456206689, |
| "eval_logps/chosen": -595.0, |
| "eval_logps/rejected": -1017.0, |
| "eval_loss": 0.242402583360672, |
| "eval_mean_token_accuracy": 0.7130081094801426, |
| "eval_num_tokens": 64415287.0, |
| "eval_rewards/accuracies": 0.916015625, |
| "eval_rewards/chosen": -5.198269315063953, |
| "eval_rewards/margins": 4.524538166821003, |
| "eval_rewards/rejected": -9.722807422280312, |
| "eval_runtime": 14.196, |
| "eval_samples_per_second": 35.221, |
| "eval_setc/cal_net_lr": 0.0009885550958571609, |
| "eval_setc/credit_mean": 1.0, |
| "eval_setc/credit_std": 0.05235194368287921, |
| "eval_setc/logratio_margin_delta": 28.659388542175293, |
| "eval_setc/logratio_margin_vanilla": 425.4375, |
| "eval_setc/logratio_margin_weighted": 454.0968885421753, |
| "eval_steps_per_second": 2.254, |
| "step": 1800 |
| }, |
| { |
| "entropy": 0.49453125, |
| "epoch": 0.9754783077337645, |
| "grad_norm": 58.74081039428711, |
| "learning_rate": 1.953077999581498e-09, |
| "logits/chosen": -1.5973200334126898, |
| "logits/rejected": -1.6452927645084596, |
| "logps/chosen": -579.2, |
| "logps/rejected": -1085.2, |
| "loss": 0.2545, |
| "mean_token_accuracy": 0.7099454998970032, |
| "num_tokens": 64753784.0, |
| "rewards/accuracies": 0.915625, |
| "rewards/chosen": -5.223498034477234, |
| "rewards/margins": 5.342882573604584, |
| "rewards/rejected": -10.566380524635315, |
| "setc/cal_net_lr": 0.0009903632990666783, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.05157312601804733, |
| "setc/logratio_margin_delta": 32.93826675415039, |
| "setc/logratio_margin_vanilla": 501.35, |
| "setc/logratio_margin_weighted": 534.2882667541504, |
| "step": 1810 |
| }, |
| { |
| "entropy": 0.4802734375, |
| "epoch": 0.9808676906494206, |
| "grad_norm": 35.12337112426758, |
| "learning_rate": 1.210695465779732e-09, |
| "logits/chosen": -1.6434376954136094, |
| "logits/rejected": -1.6868111194134685, |
| "logps/chosen": -599.4, |
| "logps/rejected": -951.4, |
| "loss": 0.31, |
| "mean_token_accuracy": 0.7225580751895905, |
| "num_tokens": 65114010.0, |
| "rewards/accuracies": 0.88125, |
| "rewards/chosen": -5.201381134986877, |
| "rewards/margins": 3.772454595565796, |
| "rewards/rejected": -8.973835754394532, |
| "setc/cal_net_lr": 0.0009937103235008124, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.04924789797514677, |
| "setc/logratio_margin_delta": 25.745468139648438, |
| "setc/logratio_margin_vanilla": 351.5, |
| "setc/logratio_margin_weighted": 377.24546813964844, |
| "step": 1820 |
| }, |
| { |
| "entropy": 0.48134765625, |
| "epoch": 0.9862570735650767, |
| "grad_norm": 41.82625198364258, |
| "learning_rate": 6.44823668770833e-10, |
| "logits/chosen": -1.6893693610067781, |
| "logits/rejected": -1.7599484026922272, |
| "logps/chosen": -573.5, |
| "logps/rejected": -927.0, |
| "loss": 0.2913, |
| "mean_token_accuracy": 0.7189024925231934, |
| "num_tokens": 65453240.0, |
| "rewards/accuracies": 0.871875, |
| "rewards/chosen": -4.990872526168824, |
| "rewards/margins": 3.8168386936187746, |
| "rewards/rejected": -8.807711195945739, |
| "setc/cal_net_lr": 0.000996346692340764, |
| "setc/credit_mean": 0.9999999970197677, |
| "setc/credit_std": 0.04604421444237232, |
| "setc/logratio_margin_delta": 27.133870697021486, |
| "setc/logratio_margin_vanilla": 354.55, |
| "setc/logratio_margin_weighted": 381.68387069702146, |
| "step": 1830 |
| }, |
| { |
| "entropy": 0.4810546875, |
| "epoch": 0.9916464564807329, |
| "grad_norm": 64.11349487304688, |
| "learning_rate": 2.556628583335052e-10, |
| "logits/chosen": -1.6629026317813094, |
| "logits/rejected": -1.702124887555161, |
| "logps/chosen": -631.7, |
| "logps/rejected": -1014.0, |
| "loss": 0.2856, |
| "mean_token_accuracy": 0.7035313218832016, |
| "num_tokens": 65813540.0, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -5.657264351844788, |
| "rewards/margins": 4.0819092988967896, |
| "rewards/rejected": -9.739173555374146, |
| "setc/cal_net_lr": 0.0009982686107494103, |
| "setc/credit_mean": 1.0, |
| "setc/credit_std": 0.04834742061793804, |
| "setc/logratio_margin_delta": 25.690938568115236, |
| "setc/logratio_margin_vanilla": 382.5, |
| "setc/logratio_margin_weighted": 408.19093856811526, |
| "step": 1840 |
| }, |
| { |
| "entropy": 0.474609375, |
| "epoch": 0.9970358393963891, |
| "grad_norm": 32.58039855957031, |
| "learning_rate": 4.3350750053627874e-11, |
| "logits/chosen": -1.6136593090507105, |
| "logits/rejected": -1.6928662572741515, |
| "logps/chosen": -604.8, |
| "logps/rejected": -1012.2, |
| "loss": 0.294, |
| "mean_token_accuracy": 0.712560287117958, |
| "num_tokens": 66170915.0, |
| "rewards/accuracies": 0.846875, |
| "rewards/chosen": -5.385611450672149, |
| "rewards/margins": 4.348533475399018, |
| "rewards/rejected": -9.734145045280457, |
| "setc/cal_net_lr": 0.0009994733122825335, |
| "setc/credit_mean": 0.9999999970197677, |
| "setc/credit_std": 0.04455969799309969, |
| "setc/logratio_margin_delta": 27.10336227416992, |
| "setc/logratio_margin_vanilla": 407.75, |
| "setc/logratio_margin_weighted": 434.8533622741699, |
| "step": 1850 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 1856, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 4.3990617254086574e+18, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
