{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.9852216748768474, "eval_steps": 500, "global_step": 202, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.04926108374384237, "grad_norm": 108.0, "learning_rate": 2.52589e-06, "logits/chosen": 0.06021759659051895, "logits/rejected": -0.009350213222205639, "logps/chosen": -714.9625244140625, "logps/rejected": -992.0533447265625, "loss": 0.6838, "rewards/accuracies": 0.366666704416275, "rewards/chosen": 0.011166826821863651, "rewards/margins": 0.022149886935949326, "rewards/rejected": -0.010983060114085674, "step": 5 }, { "epoch": 0.09852216748768473, "grad_norm": 73.5, "learning_rate": 5.6832524999999995e-06, "logits/chosen": 0.045873817056417465, "logits/rejected": 0.003844240214675665, "logps/chosen": -671.487060546875, "logps/rejected": -1002.9957885742188, "loss": 0.5469, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.0998205691576004, "rewards/margins": 0.3382015824317932, "rewards/rejected": -0.2383810579776764, "step": 10 }, { "epoch": 0.1477832512315271, "grad_norm": 30.625, "learning_rate": 8.840615e-06, "logits/chosen": -0.023920465260744095, "logits/rejected": 0.042549438774585724, "logps/chosen": -677.3431396484375, "logps/rejected": -989.5335693359375, "loss": 0.2497, "rewards/accuracies": 0.98333340883255, "rewards/chosen": 0.3677126467227936, "rewards/margins": 1.4746158123016357, "rewards/rejected": -1.106903076171875, "step": 15 }, { "epoch": 0.19704433497536947, "grad_norm": 3.71875, "learning_rate": 1.19979775e-05, "logits/chosen": 0.008176160976290703, "logits/rejected": 0.018394723534584045, "logps/chosen": -703.6475830078125, "logps/rejected": -1041.759765625, "loss": 0.0449, "rewards/accuracies": 1.0, "rewards/chosen": 0.787887454032898, "rewards/margins": 4.096963405609131, "rewards/rejected": -3.3090755939483643, "step": 20 }, { "epoch": 0.24630541871921183, "grad_norm": 12.5, "learning_rate": 1.5155340000000002e-05, "logits/chosen": -0.024350730702280998, "logits/rejected": 0.03716661036014557, "logps/chosen": -717.7275390625, "logps/rejected": -1072.546630859375, "loss": 0.0161, "rewards/accuracies": 1.0, "rewards/chosen": 1.2740657329559326, "rewards/margins": 7.921713829040527, "rewards/rejected": -6.647648811340332, "step": 25 }, { "epoch": 0.2955665024630542, "grad_norm": 0.2109375, "learning_rate": 1.8312702500000002e-05, "logits/chosen": 0.02022477611899376, "logits/rejected": -0.005769997835159302, "logps/chosen": -672.1201171875, "logps/rejected": -1118.390380859375, "loss": 0.0009, "rewards/accuracies": 1.0, "rewards/chosen": 1.8031457662582397, "rewards/margins": 12.140481948852539, "rewards/rejected": -10.337335586547852, "step": 30 }, { "epoch": 0.3448275862068966, "grad_norm": 8.6875, "learning_rate": 2.1470065e-05, "logits/chosen": 0.0037163645029067993, "logits/rejected": -0.027010012418031693, "logps/chosen": -665.9528198242188, "logps/rejected": -1054.0537109375, "loss": 0.0097, "rewards/accuracies": 1.0, "rewards/chosen": 1.6622908115386963, "rewards/margins": 13.775215148925781, "rewards/rejected": -12.112924575805664, "step": 35 }, { "epoch": 0.39408866995073893, "grad_norm": 4.53125, "learning_rate": 2.2092428004483083e-05, "logits/chosen": 0.0688682422041893, "logits/rejected": 0.05512089654803276, "logps/chosen": -715.6795654296875, "logps/rejected": -1175.619140625, "loss": 0.0022, "rewards/accuracies": 1.0, "rewards/chosen": 2.4531707763671875, "rewards/margins": 15.8966703414917, "rewards/rejected": -13.443501472473145, "step": 40 }, { "epoch": 0.4433497536945813, "grad_norm": 0.0022735595703125, "learning_rate": 2.205545499586072e-05, "logits/chosen": -0.025104057043790817, "logits/rejected": -0.059289705008268356, "logps/chosen": -683.1431884765625, "logps/rejected": -1164.0970458984375, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": 2.8538169860839844, "rewards/margins": 16.61087989807129, "rewards/rejected": -13.757061958312988, "step": 45 }, { "epoch": 0.49261083743842365, "grad_norm": 0.5078125, "learning_rate": 2.1990176011484834e-05, "logits/chosen": 0.06939034163951874, "logits/rejected": -0.03862878680229187, "logps/chosen": -663.315185546875, "logps/rejected": -1101.8394775390625, "loss": 0.0012, "rewards/accuracies": 1.0, "rewards/chosen": 2.704948663711548, "rewards/margins": 16.27048110961914, "rewards/rejected": -13.565534591674805, "step": 50 }, { "epoch": 0.541871921182266, "grad_norm": 1.09375, "learning_rate": 2.1896815242896443e-05, "logits/chosen": -0.07729315012693405, "logits/rejected": -0.07702396810054779, "logps/chosen": -613.7337036132812, "logps/rejected": -1092.618408203125, "loss": 0.0014, "rewards/accuracies": 1.0, "rewards/chosen": 2.867260694503784, "rewards/margins": 16.498395919799805, "rewards/rejected": -13.631135940551758, "step": 55 }, { "epoch": 0.5911330049261084, "grad_norm": 0.002410888671875, "learning_rate": 2.1775693324596527e-05, "logits/chosen": -0.03670992702245712, "logits/rejected": -0.035895608365535736, "logps/chosen": -666.19970703125, "logps/rejected": -1143.6500244140625, "loss": 0.0005, "rewards/accuracies": 1.0, "rewards/chosen": 3.211841583251953, "rewards/margins": 17.307846069335938, "rewards/rejected": -14.0960054397583, "step": 60 }, { "epoch": 0.6403940886699507, "grad_norm": 0.001922607421875, "learning_rate": 2.1627226232871688e-05, "logits/chosen": -0.031517110764980316, "logits/rejected": -0.16813872754573822, "logps/chosen": -728.2982177734375, "logps/rejected": -1131.3756103515625, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 3.4621803760528564, "rewards/margins": 17.45178985595703, "rewards/rejected": -13.98961067199707, "step": 65 }, { "epoch": 0.6896551724137931, "grad_norm": 0.11474609375, "learning_rate": 2.1451923857181784e-05, "logits/chosen": -0.003007475985214114, "logits/rejected": -0.1008603423833847, "logps/chosen": -675.1759033203125, "logps/rejected": -1094.3150634765625, "loss": 0.0007, "rewards/accuracies": 1.0, "rewards/chosen": 3.4204094409942627, "rewards/margins": 16.498920440673828, "rewards/rejected": -13.078511238098145, "step": 70 }, { "epoch": 0.7389162561576355, "grad_norm": 0.302734375, "learning_rate": 2.12503882490159e-05, "logits/chosen": -0.08852293342351913, "logits/rejected": -0.10156140476465225, "logps/chosen": -670.6781005859375, "logps/rejected": -1148.835693359375, "loss": 0.0003, "rewards/accuracies": 1.0, "rewards/chosen": 3.3611931800842285, "rewards/margins": 16.560178756713867, "rewards/rejected": -13.198987007141113, "step": 75 }, { "epoch": 0.7881773399014779, "grad_norm": 0.29296875, "learning_rate": 2.1023311554230692e-05, "logits/chosen": 0.0469479113817215, "logits/rejected": -0.06342597305774689, "logps/chosen": -684.2789916992188, "logps/rejected": -1146.1676025390625, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": 3.4045166969299316, "rewards/margins": 16.89340591430664, "rewards/rejected": -13.488889694213867, "step": 80 }, { "epoch": 0.8374384236453202, "grad_norm": 0.1240234375, "learning_rate": 2.0771473635972312e-05, "logits/chosen": -0.07357874512672424, "logits/rejected": -0.06026272848248482, "logps/chosen": -642.2686767578125, "logps/rejected": -1190.205810546875, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 3.354203701019287, "rewards/margins": 17.001527786254883, "rewards/rejected": -13.647323608398438, "step": 85 }, { "epoch": 0.8866995073891626, "grad_norm": 0.060302734375, "learning_rate": 2.0495739396345457e-05, "logits/chosen": -0.08571887761354446, "logits/rejected": -0.1507987231016159, "logps/chosen": -629.473388671875, "logps/rejected": -1103.9293212890625, "loss": 0.0003, "rewards/accuracies": 1.0, "rewards/chosen": 3.417731761932373, "rewards/margins": 17.251663208007812, "rewards/rejected": -13.833930969238281, "step": 90 }, { "epoch": 0.9359605911330049, "grad_norm": 0.0128173828125, "learning_rate": 2.01970558060281e-05, "logits/chosen": -0.027310481294989586, "logits/rejected": -0.04605848342180252, "logps/chosen": -725.526611328125, "logps/rejected": -1180.865478515625, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 3.788097858428955, "rewards/margins": 17.4461727142334, "rewards/rejected": -13.658073425292969, "step": 95 }, { "epoch": 0.9852216748768473, "grad_norm": 0.004241943359375, "learning_rate": 1.9876448652033145e-05, "logits/chosen": -0.12386312335729599, "logits/rejected": -0.10513818264007568, "logps/chosen": -615.34033203125, "logps/rejected": -1112.7601318359375, "loss": 0.001, "rewards/accuracies": 1.0, "rewards/chosen": 3.5329208374023438, "rewards/margins": 16.703248977661133, "rewards/rejected": -13.170324325561523, "step": 100 }, { "epoch": 0.9950738916256158, "eval_logits/chosen": -0.11170963197946548, "eval_logits/rejected": -0.11342550069093704, "eval_logps/chosen": -621.7440795898438, "eval_logps/rejected": -1078.06591796875, "eval_loss": 0.00033413898199796677, "eval_rewards/accuracies": 1.0, "eval_rewards/chosen": 3.6759870052337646, "eval_rewards/margins": 16.46457862854004, "eval_rewards/rejected": -12.788591384887695, "eval_runtime": 16.5627, "eval_samples_per_second": 12.075, "eval_steps_per_second": 12.075, "step": 101 }, { "epoch": 1.0295566502463054, "grad_norm": 0.006683349609375, "learning_rate": 1.9535019014786414e-05, "logits/chosen": -0.06293856352567673, "logits/rejected": -0.09947305917739868, "logps/chosen": -672.5732421875, "logps/rejected": -1174.2159423828125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 3.6842219829559326, "rewards/margins": 17.364566802978516, "rewards/rejected": -13.680344581604004, "step": 105 }, { "epoch": 1.0788177339901477, "grad_norm": 0.0015106201171875, "learning_rate": 1.9173939486619933e-05, "logits/chosen": -0.05223611742258072, "logits/rejected": -0.10244528949260712, "logps/chosen": -647.3909912109375, "logps/rejected": -1189.0113525390625, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 4.147972583770752, "rewards/margins": 17.80228614807129, "rewards/rejected": -13.654312133789062, "step": 110 }, { "epoch": 1.1280788177339902, "grad_norm": 0.01220703125, "learning_rate": 1.8794450144667584e-05, "logits/chosen": -0.03947216272354126, "logits/rejected": -0.12559063732624054, "logps/chosen": -772.1909790039062, "logps/rejected": -1098.6795654296875, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 4.7596025466918945, "rewards/margins": 18.306901931762695, "rewards/rejected": -13.547297477722168, "step": 115 }, { "epoch": 1.1773399014778325, "grad_norm": 0.000743865966796875, "learning_rate": 1.839785429199364e-05, "logits/chosen": -0.0065459804609417915, "logits/rejected": -0.1465989649295807, "logps/chosen": -711.794189453125, "logps/rejected": -1193.7640380859375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 4.160403251647949, "rewards/margins": 18.479259490966797, "rewards/rejected": -14.318857192993164, "step": 120 }, { "epoch": 1.2266009852216748, "grad_norm": 0.003936767578125, "learning_rate": 1.7985513981580707e-05, "logits/chosen": -0.0810680016875267, "logits/rejected": -0.14764997363090515, "logps/chosen": -656.9730224609375, "logps/rejected": -1095.385986328125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 3.6672158241271973, "rewards/margins": 17.432126998901367, "rewards/rejected": -13.764910697937012, "step": 125 }, { "epoch": 1.2758620689655173, "grad_norm": 0.0162353515625, "learning_rate": 1.7558845338549242e-05, "logits/chosen": -0.07523246854543686, "logits/rejected": -0.25009721517562866, "logps/chosen": -668.7523193359375, "logps/rejected": -1086.5867919921875, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 4.281540393829346, "rewards/margins": 17.89385986328125, "rewards/rejected": -13.61231803894043, "step": 130 }, { "epoch": 1.3251231527093597, "grad_norm": 0.002838134765625, "learning_rate": 1.711931369667393e-05, "logits/chosen": -0.1100362166762352, "logits/rejected": -0.200296089053154, "logps/chosen": -666.0404052734375, "logps/rejected": -1082.836669921875, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 3.762917995452881, "rewards/margins": 17.2962703704834, "rewards/rejected": -13.533352851867676, "step": 135 }, { "epoch": 1.374384236453202, "grad_norm": 0.0068359375, "learning_rate": 1.666842856589978e-05, "logits/chosen": -0.11170603334903717, "logits/rejected": -0.11424056440591812, "logps/chosen": -634.7129516601562, "logps/rejected": -1162.6055908203125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 3.8669981956481934, "rewards/margins": 17.46556854248047, "rewards/rejected": -13.59857177734375, "step": 140 }, { "epoch": 1.4236453201970443, "grad_norm": 0.0010833740234375, "learning_rate": 1.6207738448141366e-05, "logits/chosen": -0.07618793845176697, "logits/rejected": -0.1659926474094391, "logps/chosen": -629.3489379882812, "logps/rejected": -1141.2314453125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 3.594289779663086, "rewards/margins": 17.63915252685547, "rewards/rejected": -14.044862747192383, "step": 145 }, { "epoch": 1.4729064039408866, "grad_norm": 0.0028839111328125, "learning_rate": 1.573882551916961e-05, "logits/chosen": -0.1288968324661255, "logits/rejected": -0.11787639558315277, "logps/chosen": -682.9420166015625, "logps/rejected": -1165.747802734375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 3.8218026161193848, "rewards/margins": 17.532339096069336, "rewards/rejected": -13.710535049438477, "step": 150 }, { "epoch": 1.522167487684729, "grad_norm": 0.0712890625, "learning_rate": 1.5263300194850375e-05, "logits/chosen": -0.14020918309688568, "logits/rejected": -0.17282378673553467, "logps/chosen": -562.5584716796875, "logps/rejected": -1065.2425537109375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 3.819398880004883, "rewards/margins": 16.870691299438477, "rewards/rejected": -13.051290512084961, "step": 155 }, { "epoch": 1.5714285714285714, "grad_norm": 0.01177978515625, "learning_rate": 1.478279560039646e-05, "logits/chosen": 0.0017956334631890059, "logits/rejected": -0.04993182793259621, "logps/chosen": -631.7723388671875, "logps/rejected": -1118.0230712890625, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 3.4135658740997314, "rewards/margins": 16.631460189819336, "rewards/rejected": -13.217893600463867, "step": 160 }, { "epoch": 1.6206896551724137, "grad_norm": 0.0032806396484375, "learning_rate": 1.429896196162745e-05, "logits/chosen": -0.10030888020992279, "logits/rejected": -0.0873989462852478, "logps/chosen": -593.56884765625, "logps/rejected": -1196.1925048828125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 4.140743255615234, "rewards/margins": 18.093753814697266, "rewards/rejected": -13.953012466430664, "step": 165 }, { "epoch": 1.6699507389162562, "grad_norm": 0.004180908203125, "learning_rate": 1.3813460937500001e-05, "logits/chosen": -0.022314613685011864, "logits/rejected": -0.1113683357834816, "logps/chosen": -704.1790771484375, "logps/rejected": -1212.56103515625, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 3.9814553260803223, "rewards/margins": 18.219303131103516, "rewards/rejected": -14.237849235534668, "step": 170 }, { "epoch": 1.7192118226600985, "grad_norm": 0.0185546875, "learning_rate": 1.332795991337255e-05, "logits/chosen": -0.0795547217130661, "logits/rejected": -0.18906092643737793, "logps/chosen": -714.6734619140625, "logps/rejected": -1112.668212890625, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 4.523798942565918, "rewards/margins": 18.23130989074707, "rewards/rejected": -13.70750904083252, "step": 175 }, { "epoch": 1.7684729064039408, "grad_norm": 0.000331878662109375, "learning_rate": 1.2844126274603544e-05, "logits/chosen": -0.04283156991004944, "logits/rejected": -0.11096321046352386, "logps/chosen": -729.1672973632812, "logps/rejected": -1158.585205078125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 4.16518497467041, "rewards/margins": 17.872875213623047, "rewards/rejected": -13.707687377929688, "step": 180 }, { "epoch": 1.8177339901477834, "grad_norm": 0.005584716796875, "learning_rate": 1.2363621680149627e-05, "logits/chosen": 0.017648298293352127, "logits/rejected": -0.08744337409734726, "logps/chosen": -655.9172973632812, "logps/rejected": -1120.0281982421875, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 3.544255495071411, "rewards/margins": 17.05453109741211, "rewards/rejected": -13.510274887084961, "step": 185 }, { "epoch": 1.8669950738916257, "grad_norm": 0.00147247314453125, "learning_rate": 1.1888096355830394e-05, "logits/chosen": -0.009853709489107132, "logits/rejected": -0.07630355656147003, "logps/chosen": -641.5875244140625, "logps/rejected": -1153.7431640625, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 3.8983428478240967, "rewards/margins": 17.25222396850586, "rewards/rejected": -13.353883743286133, "step": 190 }, { "epoch": 1.916256157635468, "grad_norm": 0.040771484375, "learning_rate": 1.1419183426858638e-05, "logits/chosen": -0.11551934480667114, "logits/rejected": -0.1467764675617218, "logps/chosen": -606.7501220703125, "logps/rejected": -1105.7303466796875, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 4.016844749450684, "rewards/margins": 17.583324432373047, "rewards/rejected": -13.56648063659668, "step": 195 }, { "epoch": 1.9655172413793105, "grad_norm": 0.029541015625, "learning_rate": 1.0958493309100225e-05, "logits/chosen": -0.050420112907886505, "logits/rejected": -0.09584550559520721, "logps/chosen": -662.5157470703125, "logps/rejected": -1182.98486328125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 4.08192253112793, "rewards/margins": 18.255043029785156, "rewards/rejected": -14.173120498657227, "step": 200 }, { "epoch": 1.9852216748768474, "eval_logits/chosen": -0.11707988381385803, "eval_logits/rejected": -0.12201124429702759, "eval_logps/chosen": -621.3673706054688, "eval_logps/rejected": -1078.1207275390625, "eval_loss": 0.00028668390586972237, "eval_rewards/accuracies": 1.0, "eval_rewards/chosen": 3.7136542797088623, "eval_rewards/margins": 16.507728576660156, "eval_rewards/rejected": -12.794075012207031, "eval_runtime": 17.6249, "eval_samples_per_second": 11.348, "eval_steps_per_second": 11.348, "step": 202 } ], "logging_steps": 5, "max_steps": 303, "num_input_tokens_seen": 0, "num_train_epochs": 3, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 12, "trial_name": null, "trial_params": null }