| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 9.401709401709402, |
| "eval_steps": 500, |
| "global_step": 22000, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.21367521367521367, |
| "grad_norm": 4.928079605102539, |
| "learning_rate": 5e-05, |
| "logits/chosen": -17.893346786499023, |
| "logits/rejected": -17.935224533081055, |
| "logps/chosen": -247.3836669921875, |
| "logps/rejected": -228.11244201660156, |
| "loss": 0.7455, |
| "rewards/accuracies": 0.5327500104904175, |
| "rewards/chosen": 0.5997859835624695, |
| "rewards/margins": 0.08256139606237411, |
| "rewards/rejected": 0.5172246098518372, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.21367521367521367, |
| "eval_logits/chosen": -17.495702743530273, |
| "eval_logits/rejected": -17.58708381652832, |
| "eval_logps/chosen": -242.62672424316406, |
| "eval_logps/rejected": -221.27163696289062, |
| "eval_loss": 0.7328621745109558, |
| "eval_rewards/accuracies": 0.5503731369972229, |
| "eval_rewards/chosen": 1.2266465425491333, |
| "eval_rewards/margins": 0.1979004591703415, |
| "eval_rewards/rejected": 1.0287461280822754, |
| "eval_runtime": 228.1137, |
| "eval_samples_per_second": 11.718, |
| "eval_steps_per_second": 1.469, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.42735042735042733, |
| "grad_norm": 4.208562850952148, |
| "learning_rate": 4.890829694323144e-05, |
| "logits/chosen": -16.971363067626953, |
| "logits/rejected": -17.00401496887207, |
| "logps/chosen": -247.00656127929688, |
| "logps/rejected": -228.8548126220703, |
| "loss": 0.7843, |
| "rewards/accuracies": 0.5517500042915344, |
| "rewards/chosen": 0.8434626460075378, |
| "rewards/margins": 0.19367894530296326, |
| "rewards/rejected": 0.649783730506897, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.42735042735042733, |
| "eval_logits/chosen": -17.079561233520508, |
| "eval_logits/rejected": -17.085018157958984, |
| "eval_logps/chosen": -242.22113037109375, |
| "eval_logps/rejected": -221.7228546142578, |
| "eval_loss": 0.7642466425895691, |
| "eval_rewards/accuracies": 0.5776119232177734, |
| "eval_rewards/chosen": 1.2672061920166016, |
| "eval_rewards/margins": 0.28358009457588196, |
| "eval_rewards/rejected": 0.9836260676383972, |
| "eval_runtime": 228.0897, |
| "eval_samples_per_second": 11.719, |
| "eval_steps_per_second": 1.469, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.6410256410256411, |
| "grad_norm": 3.5691583156585693, |
| "learning_rate": 4.7816593886462886e-05, |
| "logits/chosen": -16.51204490661621, |
| "logits/rejected": -16.47532844543457, |
| "logps/chosen": -240.64077758789062, |
| "logps/rejected": -222.9105987548828, |
| "loss": 0.7908, |
| "rewards/accuracies": 0.5792499780654907, |
| "rewards/chosen": 1.2004680633544922, |
| "rewards/margins": 0.30569130182266235, |
| "rewards/rejected": 0.8947767615318298, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.6410256410256411, |
| "eval_logits/chosen": -16.36088752746582, |
| "eval_logits/rejected": -16.358932495117188, |
| "eval_logps/chosen": -239.6025848388672, |
| "eval_logps/rejected": -219.82733154296875, |
| "eval_loss": 0.7727888226509094, |
| "eval_rewards/accuracies": 0.5697761178016663, |
| "eval_rewards/chosen": 1.5290582180023193, |
| "eval_rewards/margins": 0.35588183999061584, |
| "eval_rewards/rejected": 1.1731764078140259, |
| "eval_runtime": 228.179, |
| "eval_samples_per_second": 11.714, |
| "eval_steps_per_second": 1.468, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.8547008547008547, |
| "grad_norm": 6.145715713500977, |
| "learning_rate": 4.672489082969432e-05, |
| "logits/chosen": -16.44040870666504, |
| "logits/rejected": -16.468050003051758, |
| "logps/chosen": -245.83175659179688, |
| "logps/rejected": -227.9639892578125, |
| "loss": 0.7897, |
| "rewards/accuracies": 0.5802500247955322, |
| "rewards/chosen": 1.1456317901611328, |
| "rewards/margins": 0.3306076228618622, |
| "rewards/rejected": 0.8150242567062378, |
| "step": 2000 |
| }, |
| { |
| "epoch": 0.8547008547008547, |
| "eval_logits/chosen": -16.609996795654297, |
| "eval_logits/rejected": -16.722354888916016, |
| "eval_logps/chosen": -241.63665771484375, |
| "eval_logps/rejected": -222.01730346679688, |
| "eval_loss": 0.7797393202781677, |
| "eval_rewards/accuracies": 0.5716418027877808, |
| "eval_rewards/chosen": 1.3256531953811646, |
| "eval_rewards/margins": 0.37147435545921326, |
| "eval_rewards/rejected": 0.9541788101196289, |
| "eval_runtime": 227.8733, |
| "eval_samples_per_second": 11.73, |
| "eval_steps_per_second": 1.47, |
| "step": 2000 |
| }, |
| { |
| "epoch": 1.0683760683760684, |
| "grad_norm": 8.053547859191895, |
| "learning_rate": 4.563318777292577e-05, |
| "logits/chosen": -16.771224975585938, |
| "logits/rejected": -16.817113876342773, |
| "logps/chosen": -241.07923889160156, |
| "logps/rejected": -230.9042510986328, |
| "loss": 0.6165, |
| "rewards/accuracies": 0.6802499890327454, |
| "rewards/chosen": 1.3186030387878418, |
| "rewards/margins": 0.9475009441375732, |
| "rewards/rejected": 0.37110206484794617, |
| "step": 2500 |
| }, |
| { |
| "epoch": 1.0683760683760684, |
| "eval_logits/chosen": -18.916522979736328, |
| "eval_logits/rejected": -19.013525009155273, |
| "eval_logps/chosen": -245.43980407714844, |
| "eval_logps/rejected": -226.0865936279297, |
| "eval_loss": 0.8502845168113708, |
| "eval_rewards/accuracies": 0.5727611780166626, |
| "eval_rewards/chosen": 0.9453384876251221, |
| "eval_rewards/margins": 0.39808395504951477, |
| "eval_rewards/rejected": 0.5472545027732849, |
| "eval_runtime": 227.8708, |
| "eval_samples_per_second": 11.73, |
| "eval_steps_per_second": 1.47, |
| "step": 2500 |
| }, |
| { |
| "epoch": 1.282051282051282, |
| "grad_norm": 5.05163049697876, |
| "learning_rate": 4.4541484716157205e-05, |
| "logits/chosen": -18.562543869018555, |
| "logits/rejected": -18.527542114257812, |
| "logps/chosen": -240.0729217529297, |
| "logps/rejected": -241.17103576660156, |
| "loss": 0.2892, |
| "rewards/accuracies": 0.8794999718666077, |
| "rewards/chosen": 1.5824379920959473, |
| "rewards/margins": 2.353099822998047, |
| "rewards/rejected": -0.7706621289253235, |
| "step": 3000 |
| }, |
| { |
| "epoch": 1.282051282051282, |
| "eval_logits/chosen": -18.576160430908203, |
| "eval_logits/rejected": -18.738643646240234, |
| "eval_logps/chosen": -244.65501403808594, |
| "eval_logps/rejected": -225.6937713623047, |
| "eval_loss": 0.8693752884864807, |
| "eval_rewards/accuracies": 0.579104483127594, |
| "eval_rewards/chosen": 1.0238155126571655, |
| "eval_rewards/margins": 0.4372811019420624, |
| "eval_rewards/rejected": 0.5865345597267151, |
| "eval_runtime": 227.9156, |
| "eval_samples_per_second": 11.728, |
| "eval_steps_per_second": 1.47, |
| "step": 3000 |
| }, |
| { |
| "epoch": 1.4957264957264957, |
| "grad_norm": 7.019180774688721, |
| "learning_rate": 4.344978165938865e-05, |
| "logits/chosen": -18.66375160217285, |
| "logits/rejected": -18.701576232910156, |
| "logps/chosen": -238.48146057128906, |
| "logps/rejected": -242.01947021484375, |
| "loss": 0.2854, |
| "rewards/accuracies": 0.8837500214576721, |
| "rewards/chosen": 1.5945905447006226, |
| "rewards/margins": 2.4082257747650146, |
| "rewards/rejected": -0.8136354088783264, |
| "step": 3500 |
| }, |
| { |
| "epoch": 1.4957264957264957, |
| "eval_logits/chosen": -18.779308319091797, |
| "eval_logits/rejected": -18.989845275878906, |
| "eval_logps/chosen": -243.39840698242188, |
| "eval_logps/rejected": -224.86245727539062, |
| "eval_loss": 0.8864023685455322, |
| "eval_rewards/accuracies": 0.5776119232177734, |
| "eval_rewards/chosen": 1.1494779586791992, |
| "eval_rewards/margins": 0.4798123836517334, |
| "eval_rewards/rejected": 0.6696655750274658, |
| "eval_runtime": 227.9853, |
| "eval_samples_per_second": 11.724, |
| "eval_steps_per_second": 1.469, |
| "step": 3500 |
| }, |
| { |
| "epoch": 1.7094017094017095, |
| "grad_norm": 7.860182285308838, |
| "learning_rate": 4.235807860262009e-05, |
| "logits/chosen": -18.62232208251953, |
| "logits/rejected": -18.68227767944336, |
| "logps/chosen": -239.76577758789062, |
| "logps/rejected": -242.22262573242188, |
| "loss": 0.3174, |
| "rewards/accuracies": 0.859499990940094, |
| "rewards/chosen": 1.3996559381484985, |
| "rewards/margins": 2.317948818206787, |
| "rewards/rejected": -0.9182929396629333, |
| "step": 4000 |
| }, |
| { |
| "epoch": 1.7094017094017095, |
| "eval_logits/chosen": -19.075952529907227, |
| "eval_logits/rejected": -19.315759658813477, |
| "eval_logps/chosen": -247.94288635253906, |
| "eval_logps/rejected": -229.47616577148438, |
| "eval_loss": 0.8939631581306458, |
| "eval_rewards/accuracies": 0.5742537379264832, |
| "eval_rewards/chosen": 0.6950293779373169, |
| "eval_rewards/margins": 0.4867364168167114, |
| "eval_rewards/rejected": 0.20829293131828308, |
| "eval_runtime": 228.1073, |
| "eval_samples_per_second": 11.718, |
| "eval_steps_per_second": 1.469, |
| "step": 4000 |
| }, |
| { |
| "epoch": 1.9230769230769231, |
| "grad_norm": 7.142192840576172, |
| "learning_rate": 4.126637554585153e-05, |
| "logits/chosen": -18.834980010986328, |
| "logits/rejected": -18.85399627685547, |
| "logps/chosen": -237.92694091796875, |
| "logps/rejected": -241.12193298339844, |
| "loss": 0.3407, |
| "rewards/accuracies": 0.8569999933242798, |
| "rewards/chosen": 1.4859353303909302, |
| "rewards/margins": 2.1926088333129883, |
| "rewards/rejected": -0.7066735625267029, |
| "step": 4500 |
| }, |
| { |
| "epoch": 1.9230769230769231, |
| "eval_logits/chosen": -18.660301208496094, |
| "eval_logits/rejected": -18.76552391052246, |
| "eval_logps/chosen": -245.375, |
| "eval_logps/rejected": -226.55128479003906, |
| "eval_loss": 0.8752386569976807, |
| "eval_rewards/accuracies": 0.579104483127594, |
| "eval_rewards/chosen": 0.95181804895401, |
| "eval_rewards/margins": 0.45103806257247925, |
| "eval_rewards/rejected": 0.5007798671722412, |
| "eval_runtime": 227.9353, |
| "eval_samples_per_second": 11.727, |
| "eval_steps_per_second": 1.47, |
| "step": 4500 |
| }, |
| { |
| "epoch": 2.1367521367521367, |
| "grad_norm": 3.366326093673706, |
| "learning_rate": 4.017467248908297e-05, |
| "logits/chosen": -19.808855056762695, |
| "logits/rejected": -19.856393814086914, |
| "logps/chosen": -241.35366821289062, |
| "logps/rejected": -259.6560363769531, |
| "loss": 0.1671, |
| "rewards/accuracies": 0.9362499713897705, |
| "rewards/chosen": 1.4077472686767578, |
| "rewards/margins": 3.8491322994232178, |
| "rewards/rejected": -2.441384792327881, |
| "step": 5000 |
| }, |
| { |
| "epoch": 2.1367521367521367, |
| "eval_logits/chosen": -22.046789169311523, |
| "eval_logits/rejected": -22.61486053466797, |
| "eval_logps/chosen": -268.6415710449219, |
| "eval_logps/rejected": -251.24929809570312, |
| "eval_loss": 1.0633221864700317, |
| "eval_rewards/accuracies": 0.5742537379264832, |
| "eval_rewards/chosen": -1.3748414516448975, |
| "eval_rewards/margins": 0.5941786170005798, |
| "eval_rewards/rejected": -1.9690202474594116, |
| "eval_runtime": 228.035, |
| "eval_samples_per_second": 11.722, |
| "eval_steps_per_second": 1.469, |
| "step": 5000 |
| }, |
| { |
| "epoch": 2.3504273504273505, |
| "grad_norm": 0.4629862606525421, |
| "learning_rate": 3.9082969432314415e-05, |
| "logits/chosen": -21.48691177368164, |
| "logits/rejected": -21.680583953857422, |
| "logps/chosen": -248.6252899169922, |
| "logps/rejected": -278.22113037109375, |
| "loss": 0.0894, |
| "rewards/accuracies": 0.9677500128746033, |
| "rewards/chosen": 0.5657418966293335, |
| "rewards/margins": 4.973217964172363, |
| "rewards/rejected": -4.407476425170898, |
| "step": 5500 |
| }, |
| { |
| "epoch": 2.3504273504273505, |
| "eval_logits/chosen": -21.514081954956055, |
| "eval_logits/rejected": -21.708589553833008, |
| "eval_logps/chosen": -266.9635314941406, |
| "eval_logps/rejected": -249.8363800048828, |
| "eval_loss": 1.1009100675582886, |
| "eval_rewards/accuracies": 0.562686562538147, |
| "eval_rewards/chosen": -1.2070350646972656, |
| "eval_rewards/margins": 0.6206951141357422, |
| "eval_rewards/rejected": -1.8277301788330078, |
| "eval_runtime": 228.1018, |
| "eval_samples_per_second": 11.718, |
| "eval_steps_per_second": 1.469, |
| "step": 5500 |
| }, |
| { |
| "epoch": 2.564102564102564, |
| "grad_norm": 8.165853500366211, |
| "learning_rate": 3.799126637554585e-05, |
| "logits/chosen": -21.36423110961914, |
| "logits/rejected": -21.592361450195312, |
| "logps/chosen": -247.65838623046875, |
| "logps/rejected": -276.07623291015625, |
| "loss": 0.0929, |
| "rewards/accuracies": 0.9667500257492065, |
| "rewards/chosen": 0.566991925239563, |
| "rewards/margins": 4.974726676940918, |
| "rewards/rejected": -4.4077348709106445, |
| "step": 6000 |
| }, |
| { |
| "epoch": 2.564102564102564, |
| "eval_logits/chosen": -21.843263626098633, |
| "eval_logits/rejected": -22.647571563720703, |
| "eval_logps/chosen": -271.59844970703125, |
| "eval_logps/rejected": -254.54527282714844, |
| "eval_loss": 1.1119718551635742, |
| "eval_rewards/accuracies": 0.5697761178016663, |
| "eval_rewards/chosen": -1.6705284118652344, |
| "eval_rewards/margins": 0.6280881762504578, |
| "eval_rewards/rejected": -2.298616647720337, |
| "eval_runtime": 228.1658, |
| "eval_samples_per_second": 11.715, |
| "eval_steps_per_second": 1.468, |
| "step": 6000 |
| }, |
| { |
| "epoch": 2.7777777777777777, |
| "grad_norm": 0.7172908782958984, |
| "learning_rate": 3.68995633187773e-05, |
| "logits/chosen": -21.93791389465332, |
| "logits/rejected": -22.361909866333008, |
| "logps/chosen": -255.6808319091797, |
| "logps/rejected": -286.0832214355469, |
| "loss": 0.0918, |
| "rewards/accuracies": 0.965749979019165, |
| "rewards/chosen": -0.06728626787662506, |
| "rewards/margins": 5.06732177734375, |
| "rewards/rejected": -5.134607315063477, |
| "step": 6500 |
| }, |
| { |
| "epoch": 2.7777777777777777, |
| "eval_logits/chosen": -21.824033737182617, |
| "eval_logits/rejected": -22.275728225708008, |
| "eval_logps/chosen": -270.8564147949219, |
| "eval_logps/rejected": -254.7839813232422, |
| "eval_loss": 1.133668303489685, |
| "eval_rewards/accuracies": 0.5802238583564758, |
| "eval_rewards/chosen": -1.596319317817688, |
| "eval_rewards/margins": 0.7261707186698914, |
| "eval_rewards/rejected": -2.3224899768829346, |
| "eval_runtime": 227.9485, |
| "eval_samples_per_second": 11.726, |
| "eval_steps_per_second": 1.47, |
| "step": 6500 |
| }, |
| { |
| "epoch": 2.9914529914529915, |
| "grad_norm": 2.079650640487671, |
| "learning_rate": 3.5807860262008734e-05, |
| "logits/chosen": -22.237197875976562, |
| "logits/rejected": -22.402421951293945, |
| "logps/chosen": -257.87725830078125, |
| "logps/rejected": -288.0488586425781, |
| "loss": 0.1018, |
| "rewards/accuracies": 0.9635000228881836, |
| "rewards/chosen": -0.3456314504146576, |
| "rewards/margins": 5.053824424743652, |
| "rewards/rejected": -5.399456024169922, |
| "step": 7000 |
| }, |
| { |
| "epoch": 2.9914529914529915, |
| "eval_logits/chosen": -22.45467758178711, |
| "eval_logits/rejected": -22.827308654785156, |
| "eval_logps/chosen": -275.2033996582031, |
| "eval_logps/rejected": -259.20159912109375, |
| "eval_loss": 1.1700248718261719, |
| "eval_rewards/accuracies": 0.5701492428779602, |
| "eval_rewards/chosen": -2.031022787094116, |
| "eval_rewards/margins": 0.7332298755645752, |
| "eval_rewards/rejected": -2.7642529010772705, |
| "eval_runtime": 227.8703, |
| "eval_samples_per_second": 11.73, |
| "eval_steps_per_second": 1.47, |
| "step": 7000 |
| }, |
| { |
| "epoch": 3.2051282051282053, |
| "grad_norm": 3.4078361988067627, |
| "learning_rate": 3.4716157205240176e-05, |
| "logits/chosen": -23.701507568359375, |
| "logits/rejected": -23.8421688079834, |
| "logps/chosen": -263.1128234863281, |
| "logps/rejected": -315.31719970703125, |
| "loss": 0.0299, |
| "rewards/accuracies": 0.9897500276565552, |
| "rewards/chosen": -0.8062784075737, |
| "rewards/margins": 7.403973579406738, |
| "rewards/rejected": -8.210251808166504, |
| "step": 7500 |
| }, |
| { |
| "epoch": 3.2051282051282053, |
| "eval_logits/chosen": -24.26601791381836, |
| "eval_logits/rejected": -24.537738800048828, |
| "eval_logps/chosen": -295.60528564453125, |
| "eval_logps/rejected": -280.7348327636719, |
| "eval_loss": 1.3489611148834229, |
| "eval_rewards/accuracies": 0.5686567425727844, |
| "eval_rewards/chosen": -4.071210861206055, |
| "eval_rewards/margins": 0.846366286277771, |
| "eval_rewards/rejected": -4.917576789855957, |
| "eval_runtime": 228.1695, |
| "eval_samples_per_second": 11.715, |
| "eval_steps_per_second": 1.468, |
| "step": 7500 |
| }, |
| { |
| "epoch": 3.4188034188034186, |
| "grad_norm": 0.8472763895988464, |
| "learning_rate": 3.362445414847162e-05, |
| "logits/chosen": -25.03811264038086, |
| "logits/rejected": -25.47620964050293, |
| "logps/chosen": -272.96221923828125, |
| "logps/rejected": -330.9865417480469, |
| "loss": 0.0335, |
| "rewards/accuracies": 0.9869999885559082, |
| "rewards/chosen": -1.7266676425933838, |
| "rewards/margins": 7.801348686218262, |
| "rewards/rejected": -9.528016090393066, |
| "step": 8000 |
| }, |
| { |
| "epoch": 3.4188034188034186, |
| "eval_logits/chosen": -25.045902252197266, |
| "eval_logits/rejected": -26.036436080932617, |
| "eval_logps/chosen": -298.59942626953125, |
| "eval_logps/rejected": -283.6900329589844, |
| "eval_loss": 1.3672035932540894, |
| "eval_rewards/accuracies": 0.5697761178016663, |
| "eval_rewards/chosen": -4.370626449584961, |
| "eval_rewards/margins": 0.8424644470214844, |
| "eval_rewards/rejected": -5.2130913734436035, |
| "eval_runtime": 228.1698, |
| "eval_samples_per_second": 11.715, |
| "eval_steps_per_second": 1.468, |
| "step": 8000 |
| }, |
| { |
| "epoch": 3.6324786324786325, |
| "grad_norm": 0.4369616210460663, |
| "learning_rate": 3.2532751091703054e-05, |
| "logits/chosen": -25.275936126708984, |
| "logits/rejected": -25.85784339904785, |
| "logps/chosen": -272.4268493652344, |
| "logps/rejected": -328.8055725097656, |
| "loss": 0.0342, |
| "rewards/accuracies": 0.9862499833106995, |
| "rewards/chosen": -1.7520734071731567, |
| "rewards/margins": 7.620083332061768, |
| "rewards/rejected": -9.372157096862793, |
| "step": 8500 |
| }, |
| { |
| "epoch": 3.6324786324786325, |
| "eval_logits/chosen": -25.376535415649414, |
| "eval_logits/rejected": -25.840946197509766, |
| "eval_logps/chosen": -302.3465576171875, |
| "eval_logps/rejected": -286.4819641113281, |
| "eval_loss": 1.4073855876922607, |
| "eval_rewards/accuracies": 0.5574626922607422, |
| "eval_rewards/chosen": -4.745336055755615, |
| "eval_rewards/margins": 0.7469544410705566, |
| "eval_rewards/rejected": -5.492290019989014, |
| "eval_runtime": 228.1692, |
| "eval_samples_per_second": 11.715, |
| "eval_steps_per_second": 1.468, |
| "step": 8500 |
| }, |
| { |
| "epoch": 3.8461538461538463, |
| "grad_norm": 0.20020511746406555, |
| "learning_rate": 3.14410480349345e-05, |
| "logits/chosen": -25.79690933227539, |
| "logits/rejected": -26.186351776123047, |
| "logps/chosen": -275.40826416015625, |
| "logps/rejected": -333.3716125488281, |
| "loss": 0.0406, |
| "rewards/accuracies": 0.9865000247955322, |
| "rewards/chosen": -2.3063974380493164, |
| "rewards/margins": 7.773219108581543, |
| "rewards/rejected": -10.079617500305176, |
| "step": 9000 |
| }, |
| { |
| "epoch": 3.8461538461538463, |
| "eval_logits/chosen": -25.633359909057617, |
| "eval_logits/rejected": -26.33572769165039, |
| "eval_logps/chosen": -305.1189270019531, |
| "eval_logps/rejected": -290.773193359375, |
| "eval_loss": 1.4024347066879272, |
| "eval_rewards/accuracies": 0.5660447478294373, |
| "eval_rewards/chosen": -5.022571086883545, |
| "eval_rewards/margins": 0.8988366723060608, |
| "eval_rewards/rejected": -5.921407699584961, |
| "eval_runtime": 227.3147, |
| "eval_samples_per_second": 11.759, |
| "eval_steps_per_second": 1.474, |
| "step": 9000 |
| }, |
| { |
| "epoch": 4.05982905982906, |
| "grad_norm": 0.4492078125476837, |
| "learning_rate": 3.0349344978165938e-05, |
| "logits/chosen": -25.544862747192383, |
| "logits/rejected": -25.84202003479004, |
| "logps/chosen": -275.60540771484375, |
| "logps/rejected": -333.7979431152344, |
| "loss": 0.0353, |
| "rewards/accuracies": 0.9869999885559082, |
| "rewards/chosen": -2.0801212787628174, |
| "rewards/margins": 7.923834800720215, |
| "rewards/rejected": -10.00395679473877, |
| "step": 9500 |
| }, |
| { |
| "epoch": 4.05982905982906, |
| "eval_logits/chosen": -26.5059871673584, |
| "eval_logits/rejected": -26.621540069580078, |
| "eval_logps/chosen": -315.5564880371094, |
| "eval_logps/rejected": -300.5618591308594, |
| "eval_loss": 1.492741584777832, |
| "eval_rewards/accuracies": 0.5660447478294373, |
| "eval_rewards/chosen": -6.066328525543213, |
| "eval_rewards/margins": 0.833947479724884, |
| "eval_rewards/rejected": -6.900275707244873, |
| "eval_runtime": 227.4988, |
| "eval_samples_per_second": 11.75, |
| "eval_steps_per_second": 1.473, |
| "step": 9500 |
| }, |
| { |
| "epoch": 4.273504273504273, |
| "grad_norm": 0.786451518535614, |
| "learning_rate": 2.9257641921397383e-05, |
| "logits/chosen": -26.779680252075195, |
| "logits/rejected": -26.80259132385254, |
| "logps/chosen": -283.5688171386719, |
| "logps/rejected": -356.3512268066406, |
| "loss": 0.0127, |
| "rewards/accuracies": 0.9944999814033508, |
| "rewards/chosen": -2.842845916748047, |
| "rewards/margins": 9.423320770263672, |
| "rewards/rejected": -12.266166687011719, |
| "step": 10000 |
| }, |
| { |
| "epoch": 4.273504273504273, |
| "eval_logits/chosen": -27.469865798950195, |
| "eval_logits/rejected": -27.798917770385742, |
| "eval_logps/chosen": -325.7637023925781, |
| "eval_logps/rejected": -311.94171142578125, |
| "eval_loss": 1.5876415967941284, |
| "eval_rewards/accuracies": 0.5615671873092651, |
| "eval_rewards/chosen": -7.0870537757873535, |
| "eval_rewards/margins": 0.9512065649032593, |
| "eval_rewards/rejected": -8.038260459899902, |
| "eval_runtime": 227.4171, |
| "eval_samples_per_second": 11.754, |
| "eval_steps_per_second": 1.473, |
| "step": 10000 |
| }, |
| { |
| "epoch": 4.487179487179487, |
| "grad_norm": 0.010021047666668892, |
| "learning_rate": 2.816593886462882e-05, |
| "logits/chosen": -27.027307510375977, |
| "logits/rejected": -27.183887481689453, |
| "logps/chosen": -284.8421325683594, |
| "logps/rejected": -359.85992431640625, |
| "loss": 0.0208, |
| "rewards/accuracies": 0.9887499809265137, |
| "rewards/chosen": -3.0061757564544678, |
| "rewards/margins": 9.546944618225098, |
| "rewards/rejected": -12.553121566772461, |
| "step": 10500 |
| }, |
| { |
| "epoch": 4.487179487179487, |
| "eval_logits/chosen": -27.8445987701416, |
| "eval_logits/rejected": -28.134754180908203, |
| "eval_logps/chosen": -331.4013977050781, |
| "eval_logps/rejected": -317.50494384765625, |
| "eval_loss": 1.6331137418746948, |
| "eval_rewards/accuracies": 0.5701492428779602, |
| "eval_rewards/chosen": -7.650820732116699, |
| "eval_rewards/margins": 0.9437649250030518, |
| "eval_rewards/rejected": -8.594585418701172, |
| "eval_runtime": 227.6795, |
| "eval_samples_per_second": 11.74, |
| "eval_steps_per_second": 1.471, |
| "step": 10500 |
| }, |
| { |
| "epoch": 4.700854700854701, |
| "grad_norm": 0.36542195081710815, |
| "learning_rate": 2.7074235807860267e-05, |
| "logits/chosen": -27.794286727905273, |
| "logits/rejected": -27.915931701660156, |
| "logps/chosen": -291.84320068359375, |
| "logps/rejected": -370.82891845703125, |
| "loss": 0.0187, |
| "rewards/accuracies": 0.9929999709129333, |
| "rewards/chosen": -3.9194016456604004, |
| "rewards/margins": 9.847352027893066, |
| "rewards/rejected": -13.766753196716309, |
| "step": 11000 |
| }, |
| { |
| "epoch": 4.700854700854701, |
| "eval_logits/chosen": -28.695451736450195, |
| "eval_logits/rejected": -29.109943389892578, |
| "eval_logps/chosen": -340.71160888671875, |
| "eval_logps/rejected": -326.3081359863281, |
| "eval_loss": 1.7010728120803833, |
| "eval_rewards/accuracies": 0.5533581972122192, |
| "eval_rewards/chosen": -8.581844329833984, |
| "eval_rewards/margins": 0.8930591344833374, |
| "eval_rewards/rejected": -9.474903106689453, |
| "eval_runtime": 227.9038, |
| "eval_samples_per_second": 11.729, |
| "eval_steps_per_second": 1.47, |
| "step": 11000 |
| }, |
| { |
| "epoch": 4.914529914529915, |
| "grad_norm": 0.3806762099266052, |
| "learning_rate": 2.5982532751091705e-05, |
| "logits/chosen": -28.41119384765625, |
| "logits/rejected": -29.06536865234375, |
| "logps/chosen": -296.35723876953125, |
| "logps/rejected": -373.481201171875, |
| "loss": 0.0225, |
| "rewards/accuracies": 0.9915000200271606, |
| "rewards/chosen": -4.082763671875, |
| "rewards/margins": 9.720556259155273, |
| "rewards/rejected": -13.803319931030273, |
| "step": 11500 |
| }, |
| { |
| "epoch": 4.914529914529915, |
| "eval_logits/chosen": -29.32524871826172, |
| "eval_logits/rejected": -30.48952865600586, |
| "eval_logps/chosen": -336.1420593261719, |
| "eval_logps/rejected": -321.3468933105469, |
| "eval_loss": 1.700086236000061, |
| "eval_rewards/accuracies": 0.5582089424133301, |
| "eval_rewards/chosen": -8.124887466430664, |
| "eval_rewards/margins": 0.8538958430290222, |
| "eval_rewards/rejected": -8.978782653808594, |
| "eval_runtime": 227.8283, |
| "eval_samples_per_second": 11.733, |
| "eval_steps_per_second": 1.47, |
| "step": 11500 |
| }, |
| { |
| "epoch": 5.128205128205128, |
| "grad_norm": 0.2094322144985199, |
| "learning_rate": 2.4890829694323144e-05, |
| "logits/chosen": -28.792156219482422, |
| "logits/rejected": -29.474102020263672, |
| "logps/chosen": -296.71466064453125, |
| "logps/rejected": -377.149658203125, |
| "loss": 0.0158, |
| "rewards/accuracies": 0.9934999942779541, |
| "rewards/chosen": -4.169726848602295, |
| "rewards/margins": 10.347012519836426, |
| "rewards/rejected": -14.516740798950195, |
| "step": 12000 |
| }, |
| { |
| "epoch": 5.128205128205128, |
| "eval_logits/chosen": -30.29293441772461, |
| "eval_logits/rejected": -30.985925674438477, |
| "eval_logps/chosen": -351.3580627441406, |
| "eval_logps/rejected": -336.6136474609375, |
| "eval_loss": 1.815493106842041, |
| "eval_rewards/accuracies": 0.5555970072746277, |
| "eval_rewards/chosen": -9.646490097045898, |
| "eval_rewards/margins": 0.8589639663696289, |
| "eval_rewards/rejected": -10.505454063415527, |
| "eval_runtime": 227.9111, |
| "eval_samples_per_second": 11.728, |
| "eval_steps_per_second": 1.47, |
| "step": 12000 |
| }, |
| { |
| "epoch": 5.3418803418803416, |
| "grad_norm": 0.10701989382505417, |
| "learning_rate": 2.3799126637554586e-05, |
| "logits/chosen": -29.149518966674805, |
| "logits/rejected": -29.762508392333984, |
| "logps/chosen": -296.66693115234375, |
| "logps/rejected": -384.7826843261719, |
| "loss": 0.0129, |
| "rewards/accuracies": 0.9927499890327454, |
| "rewards/chosen": -4.225397109985352, |
| "rewards/margins": 10.851059913635254, |
| "rewards/rejected": -15.076457023620605, |
| "step": 12500 |
| }, |
| { |
| "epoch": 5.3418803418803416, |
| "eval_logits/chosen": -30.771709442138672, |
| "eval_logits/rejected": -31.801382064819336, |
| "eval_logps/chosen": -351.3876953125, |
| "eval_logps/rejected": -337.5948181152344, |
| "eval_loss": 1.822997808456421, |
| "eval_rewards/accuracies": 0.5578358173370361, |
| "eval_rewards/chosen": -9.649447441101074, |
| "eval_rewards/margins": 0.9541246891021729, |
| "eval_rewards/rejected": -10.603572845458984, |
| "eval_runtime": 227.8818, |
| "eval_samples_per_second": 11.73, |
| "eval_steps_per_second": 1.47, |
| "step": 12500 |
| }, |
| { |
| "epoch": 5.555555555555555, |
| "grad_norm": 5.323533535003662, |
| "learning_rate": 2.2707423580786028e-05, |
| "logits/chosen": -29.866849899291992, |
| "logits/rejected": -30.677040100097656, |
| "logps/chosen": -301.1296081542969, |
| "logps/rejected": -388.8807678222656, |
| "loss": 0.0148, |
| "rewards/accuracies": 0.9940000176429749, |
| "rewards/chosen": -4.681130886077881, |
| "rewards/margins": 10.871023178100586, |
| "rewards/rejected": -15.552155494689941, |
| "step": 13000 |
| }, |
| { |
| "epoch": 5.555555555555555, |
| "eval_logits/chosen": -29.594120025634766, |
| "eval_logits/rejected": -30.170185089111328, |
| "eval_logps/chosen": -342.6488952636719, |
| "eval_logps/rejected": -329.54034423828125, |
| "eval_loss": 1.7344962358474731, |
| "eval_rewards/accuracies": 0.5600746273994446, |
| "eval_rewards/chosen": -8.775569915771484, |
| "eval_rewards/margins": 1.0225567817687988, |
| "eval_rewards/rejected": -9.798127174377441, |
| "eval_runtime": 228.0608, |
| "eval_samples_per_second": 11.721, |
| "eval_steps_per_second": 1.469, |
| "step": 13000 |
| }, |
| { |
| "epoch": 5.769230769230769, |
| "grad_norm": 0.2315172553062439, |
| "learning_rate": 2.161572052401747e-05, |
| "logits/chosen": -29.85978889465332, |
| "logits/rejected": -30.509546279907227, |
| "logps/chosen": -301.72674560546875, |
| "logps/rejected": -391.1932373046875, |
| "loss": 0.0189, |
| "rewards/accuracies": 0.9927499890327454, |
| "rewards/chosen": -4.796882152557373, |
| "rewards/margins": 10.797426223754883, |
| "rewards/rejected": -15.594307899475098, |
| "step": 13500 |
| }, |
| { |
| "epoch": 5.769230769230769, |
| "eval_logits/chosen": -30.208812713623047, |
| "eval_logits/rejected": -31.183263778686523, |
| "eval_logps/chosen": -345.8949890136719, |
| "eval_logps/rejected": -331.9636535644531, |
| "eval_loss": 1.7506355047225952, |
| "eval_rewards/accuracies": 0.5593283772468567, |
| "eval_rewards/chosen": -9.100179672241211, |
| "eval_rewards/margins": 0.9402767419815063, |
| "eval_rewards/rejected": -10.040454864501953, |
| "eval_runtime": 227.9921, |
| "eval_samples_per_second": 11.724, |
| "eval_steps_per_second": 1.469, |
| "step": 13500 |
| }, |
| { |
| "epoch": 5.982905982905983, |
| "grad_norm": 4.712192058563232, |
| "learning_rate": 2.052401746724891e-05, |
| "logits/chosen": -29.906803131103516, |
| "logits/rejected": -30.264978408813477, |
| "logps/chosen": -307.0865173339844, |
| "logps/rejected": -395.9546813964844, |
| "loss": 0.0152, |
| "rewards/accuracies": 0.9944999814033508, |
| "rewards/chosen": -5.222396373748779, |
| "rewards/margins": 10.86800479888916, |
| "rewards/rejected": -16.09040069580078, |
| "step": 14000 |
| }, |
| { |
| "epoch": 5.982905982905983, |
| "eval_logits/chosen": -29.900236129760742, |
| "eval_logits/rejected": -30.382579803466797, |
| "eval_logps/chosen": -348.7581787109375, |
| "eval_logps/rejected": -334.68280029296875, |
| "eval_loss": 1.7904853820800781, |
| "eval_rewards/accuracies": 0.5559701323509216, |
| "eval_rewards/chosen": -9.386497497558594, |
| "eval_rewards/margins": 0.9258707165718079, |
| "eval_rewards/rejected": -10.312368392944336, |
| "eval_runtime": 227.9248, |
| "eval_samples_per_second": 11.728, |
| "eval_steps_per_second": 1.47, |
| "step": 14000 |
| }, |
| { |
| "epoch": 6.196581196581197, |
| "grad_norm": 0.004079462960362434, |
| "learning_rate": 1.943231441048035e-05, |
| "logits/chosen": -30.478179931640625, |
| "logits/rejected": -30.970232009887695, |
| "logps/chosen": -313.88165283203125, |
| "logps/rejected": -409.17962646484375, |
| "loss": 0.0126, |
| "rewards/accuracies": 0.9942499995231628, |
| "rewards/chosen": -5.9417924880981445, |
| "rewards/margins": 11.768869400024414, |
| "rewards/rejected": -17.710660934448242, |
| "step": 14500 |
| }, |
| { |
| "epoch": 6.196581196581197, |
| "eval_logits/chosen": -30.50238800048828, |
| "eval_logits/rejected": -31.02549171447754, |
| "eval_logps/chosen": -358.1636047363281, |
| "eval_logps/rejected": -344.337890625, |
| "eval_loss": 1.86916184425354, |
| "eval_rewards/accuracies": 0.5600746273994446, |
| "eval_rewards/chosen": -10.32703971862793, |
| "eval_rewards/margins": 0.9508424401283264, |
| "eval_rewards/rejected": -11.277880668640137, |
| "eval_runtime": 227.418, |
| "eval_samples_per_second": 11.754, |
| "eval_steps_per_second": 1.473, |
| "step": 14500 |
| }, |
| { |
| "epoch": 6.410256410256411, |
| "grad_norm": 0.06197773665189743, |
| "learning_rate": 1.8340611353711792e-05, |
| "logits/chosen": -30.11530876159668, |
| "logits/rejected": -30.605209350585938, |
| "logps/chosen": -308.64007568359375, |
| "logps/rejected": -403.6331787109375, |
| "loss": 0.0124, |
| "rewards/accuracies": 0.9934999942779541, |
| "rewards/chosen": -5.467129707336426, |
| "rewards/margins": 11.565047264099121, |
| "rewards/rejected": -17.032175064086914, |
| "step": 15000 |
| }, |
| { |
| "epoch": 6.410256410256411, |
| "eval_logits/chosen": -30.82184600830078, |
| "eval_logits/rejected": -31.706174850463867, |
| "eval_logps/chosen": -357.69537353515625, |
| "eval_logps/rejected": -344.0749206542969, |
| "eval_loss": 1.8543611764907837, |
| "eval_rewards/accuracies": 0.5597015023231506, |
| "eval_rewards/chosen": -10.280218124389648, |
| "eval_rewards/margins": 0.9713651537895203, |
| "eval_rewards/rejected": -11.251583099365234, |
| "eval_runtime": 227.9507, |
| "eval_samples_per_second": 11.726, |
| "eval_steps_per_second": 1.47, |
| "step": 15000 |
| }, |
| { |
| "epoch": 6.6239316239316235, |
| "grad_norm": 0.03625645488500595, |
| "learning_rate": 1.7248908296943234e-05, |
| "logits/chosen": -31.255956649780273, |
| "logits/rejected": -31.90044403076172, |
| "logps/chosen": -316.18853759765625, |
| "logps/rejected": -415.66796875, |
| "loss": 0.0119, |
| "rewards/accuracies": 0.9942499995231628, |
| "rewards/chosen": -6.230967998504639, |
| "rewards/margins": 11.93471908569336, |
| "rewards/rejected": -18.165685653686523, |
| "step": 15500 |
| }, |
| { |
| "epoch": 6.6239316239316235, |
| "eval_logits/chosen": -31.26849937438965, |
| "eval_logits/rejected": -32.083431243896484, |
| "eval_logps/chosen": -362.8932800292969, |
| "eval_logps/rejected": -351.0294494628906, |
| "eval_loss": 1.8719128370285034, |
| "eval_rewards/accuracies": 0.5701492428779602, |
| "eval_rewards/chosen": -10.800006866455078, |
| "eval_rewards/margins": 1.1470307111740112, |
| "eval_rewards/rejected": -11.947038650512695, |
| "eval_runtime": 227.9667, |
| "eval_samples_per_second": 11.725, |
| "eval_steps_per_second": 1.47, |
| "step": 15500 |
| }, |
| { |
| "epoch": 6.837606837606837, |
| "grad_norm": 0.20229032635688782, |
| "learning_rate": 1.6157205240174673e-05, |
| "logits/chosen": -31.66826629638672, |
| "logits/rejected": -32.30759048461914, |
| "logps/chosen": -319.5751647949219, |
| "logps/rejected": -420.54534912109375, |
| "loss": 0.0109, |
| "rewards/accuracies": 0.9947500228881836, |
| "rewards/chosen": -6.482852935791016, |
| "rewards/margins": 12.024003982543945, |
| "rewards/rejected": -18.50685691833496, |
| "step": 16000 |
| }, |
| { |
| "epoch": 6.837606837606837, |
| "eval_logits/chosen": -31.825002670288086, |
| "eval_logits/rejected": -32.77232360839844, |
| "eval_logps/chosen": -362.9580078125, |
| "eval_logps/rejected": -349.5175476074219, |
| "eval_loss": 1.9149656295776367, |
| "eval_rewards/accuracies": 0.562686562538147, |
| "eval_rewards/chosen": -10.806483268737793, |
| "eval_rewards/margins": 0.9893614649772644, |
| "eval_rewards/rejected": -11.795844078063965, |
| "eval_runtime": 228.1418, |
| "eval_samples_per_second": 11.716, |
| "eval_steps_per_second": 1.468, |
| "step": 16000 |
| }, |
| { |
| "epoch": 7.051282051282051, |
| "grad_norm": 0.00038808645331300795, |
| "learning_rate": 1.5065502183406113e-05, |
| "logits/chosen": -32.047367095947266, |
| "logits/rejected": -32.96037673950195, |
| "logps/chosen": -321.9444274902344, |
| "logps/rejected": -421.9002380371094, |
| "loss": 0.0161, |
| "rewards/accuracies": 0.9925000071525574, |
| "rewards/chosen": -6.6494622230529785, |
| "rewards/margins": 12.112425804138184, |
| "rewards/rejected": -18.76188850402832, |
| "step": 16500 |
| }, |
| { |
| "epoch": 7.051282051282051, |
| "eval_logits/chosen": -33.680885314941406, |
| "eval_logits/rejected": -34.744083404541016, |
| "eval_logps/chosen": -383.9889831542969, |
| "eval_logps/rejected": -370.952392578125, |
| "eval_loss": 2.086275577545166, |
| "eval_rewards/accuracies": 0.5526119470596313, |
| "eval_rewards/chosen": -12.909579277038574, |
| "eval_rewards/margins": 1.0297467708587646, |
| "eval_rewards/rejected": -13.939325332641602, |
| "eval_runtime": 226.7868, |
| "eval_samples_per_second": 11.786, |
| "eval_steps_per_second": 1.477, |
| "step": 16500 |
| }, |
| { |
| "epoch": 7.264957264957265, |
| "grad_norm": 0.022403132170438766, |
| "learning_rate": 1.3973799126637555e-05, |
| "logits/chosen": -33.031978607177734, |
| "logits/rejected": -33.99634552001953, |
| "logps/chosen": -326.0166931152344, |
| "logps/rejected": -433.9259948730469, |
| "loss": 0.0093, |
| "rewards/accuracies": 0.9952499866485596, |
| "rewards/chosen": -7.22238826751709, |
| "rewards/margins": 12.905363082885742, |
| "rewards/rejected": -20.12775230407715, |
| "step": 17000 |
| }, |
| { |
| "epoch": 7.264957264957265, |
| "eval_logits/chosen": -32.838279724121094, |
| "eval_logits/rejected": -33.81065368652344, |
| "eval_logps/chosen": -374.7555847167969, |
| "eval_logps/rejected": -361.67498779296875, |
| "eval_loss": 1.9890711307525635, |
| "eval_rewards/accuracies": 0.5570895671844482, |
| "eval_rewards/chosen": -11.98624038696289, |
| "eval_rewards/margins": 1.0253472328186035, |
| "eval_rewards/rejected": -13.011587142944336, |
| "eval_runtime": 227.8723, |
| "eval_samples_per_second": 11.73, |
| "eval_steps_per_second": 1.47, |
| "step": 17000 |
| }, |
| { |
| "epoch": 7.478632478632479, |
| "grad_norm": 0.01652335375547409, |
| "learning_rate": 1.2882096069868996e-05, |
| "logits/chosen": -32.10330581665039, |
| "logits/rejected": -32.913116455078125, |
| "logps/chosen": -319.0428161621094, |
| "logps/rejected": -421.7063293457031, |
| "loss": 0.0116, |
| "rewards/accuracies": 0.9944999814033508, |
| "rewards/chosen": -6.535008907318115, |
| "rewards/margins": 12.34512996673584, |
| "rewards/rejected": -18.880136489868164, |
| "step": 17500 |
| }, |
| { |
| "epoch": 7.478632478632479, |
| "eval_logits/chosen": -31.835594177246094, |
| "eval_logits/rejected": -32.65978240966797, |
| "eval_logps/chosen": -364.2667541503906, |
| "eval_logps/rejected": -350.83392333984375, |
| "eval_loss": 1.9134238958358765, |
| "eval_rewards/accuracies": 0.5611940026283264, |
| "eval_rewards/chosen": -10.937359809875488, |
| "eval_rewards/margins": 0.9901263117790222, |
| "eval_rewards/rejected": -11.927485466003418, |
| "eval_runtime": 227.8103, |
| "eval_samples_per_second": 11.733, |
| "eval_steps_per_second": 1.471, |
| "step": 17500 |
| }, |
| { |
| "epoch": 7.6923076923076925, |
| "grad_norm": 0.0030172993429005146, |
| "learning_rate": 1.1790393013100438e-05, |
| "logits/chosen": -31.859935760498047, |
| "logits/rejected": -32.7746696472168, |
| "logps/chosen": -316.1585388183594, |
| "logps/rejected": -419.97430419921875, |
| "loss": 0.0105, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": -6.1652021408081055, |
| "rewards/margins": 12.33544921875, |
| "rewards/rejected": -18.500652313232422, |
| "step": 18000 |
| }, |
| { |
| "epoch": 7.6923076923076925, |
| "eval_logits/chosen": -33.39469528198242, |
| "eval_logits/rejected": -34.474544525146484, |
| "eval_logps/chosen": -379.9172668457031, |
| "eval_logps/rejected": -366.830322265625, |
| "eval_loss": 2.0327441692352295, |
| "eval_rewards/accuracies": 0.5638059973716736, |
| "eval_rewards/chosen": -12.502408027648926, |
| "eval_rewards/margins": 1.024713158607483, |
| "eval_rewards/rejected": -13.527120590209961, |
| "eval_runtime": 227.4415, |
| "eval_samples_per_second": 11.752, |
| "eval_steps_per_second": 1.473, |
| "step": 18000 |
| }, |
| { |
| "epoch": 7.905982905982906, |
| "grad_norm": 16.508834838867188, |
| "learning_rate": 1.0698689956331878e-05, |
| "logits/chosen": -32.33524703979492, |
| "logits/rejected": -33.17497253417969, |
| "logps/chosen": -320.648193359375, |
| "logps/rejected": -426.4129333496094, |
| "loss": 0.0141, |
| "rewards/accuracies": 0.9917500019073486, |
| "rewards/chosen": -6.537839412689209, |
| "rewards/margins": 12.470658302307129, |
| "rewards/rejected": -19.008495330810547, |
| "step": 18500 |
| }, |
| { |
| "epoch": 7.905982905982906, |
| "eval_logits/chosen": -32.687374114990234, |
| "eval_logits/rejected": -33.56389236450195, |
| "eval_logps/chosen": -370.7414855957031, |
| "eval_logps/rejected": -357.1487731933594, |
| "eval_loss": 1.9871249198913574, |
| "eval_rewards/accuracies": 0.5548507571220398, |
| "eval_rewards/chosen": -11.584826469421387, |
| "eval_rewards/margins": 0.9741416573524475, |
| "eval_rewards/rejected": -12.558969497680664, |
| "eval_runtime": 227.2911, |
| "eval_samples_per_second": 11.76, |
| "eval_steps_per_second": 1.474, |
| "step": 18500 |
| }, |
| { |
| "epoch": 8.11965811965812, |
| "grad_norm": 0.008148429915308952, |
| "learning_rate": 9.60698689956332e-06, |
| "logits/chosen": -33.27425003051758, |
| "logits/rejected": -34.29746627807617, |
| "logps/chosen": -327.61883544921875, |
| "logps/rejected": -437.17681884765625, |
| "loss": 0.008, |
| "rewards/accuracies": 0.9957500100135803, |
| "rewards/chosen": -7.342708110809326, |
| "rewards/margins": 13.04981803894043, |
| "rewards/rejected": -20.392526626586914, |
| "step": 19000 |
| }, |
| { |
| "epoch": 8.11965811965812, |
| "eval_logits/chosen": -34.412513732910156, |
| "eval_logits/rejected": -35.9024658203125, |
| "eval_logps/chosen": -388.45513916015625, |
| "eval_logps/rejected": -375.42486572265625, |
| "eval_loss": 2.118881940841675, |
| "eval_rewards/accuracies": 0.5604477524757385, |
| "eval_rewards/chosen": -13.356197357177734, |
| "eval_rewards/margins": 1.0303833484649658, |
| "eval_rewards/rejected": -14.386581420898438, |
| "eval_runtime": 227.1891, |
| "eval_samples_per_second": 11.766, |
| "eval_steps_per_second": 1.475, |
| "step": 19000 |
| }, |
| { |
| "epoch": 8.333333333333334, |
| "grad_norm": 0.035114262253046036, |
| "learning_rate": 8.51528384279476e-06, |
| "logits/chosen": -33.662052154541016, |
| "logits/rejected": -34.972896575927734, |
| "logps/chosen": -332.39031982421875, |
| "logps/rejected": -444.69793701171875, |
| "loss": 0.0146, |
| "rewards/accuracies": 0.9940000176429749, |
| "rewards/chosen": -7.840022563934326, |
| "rewards/margins": 13.208553314208984, |
| "rewards/rejected": -21.04857635498047, |
| "step": 19500 |
| }, |
| { |
| "epoch": 8.333333333333334, |
| "eval_logits/chosen": -34.22267532348633, |
| "eval_logits/rejected": -35.91259765625, |
| "eval_logps/chosen": -386.5843200683594, |
| "eval_logps/rejected": -373.9736022949219, |
| "eval_loss": 2.091254711151123, |
| "eval_rewards/accuracies": 0.5593283772468567, |
| "eval_rewards/chosen": -13.169110298156738, |
| "eval_rewards/margins": 1.0723390579223633, |
| "eval_rewards/rejected": -14.241450309753418, |
| "eval_runtime": 227.9067, |
| "eval_samples_per_second": 11.728, |
| "eval_steps_per_second": 1.47, |
| "step": 19500 |
| }, |
| { |
| "epoch": 8.547008547008547, |
| "grad_norm": 0.001891271909698844, |
| "learning_rate": 7.423580786026202e-06, |
| "logits/chosen": -33.975555419921875, |
| "logits/rejected": -35.19069290161133, |
| "logps/chosen": -332.5311279296875, |
| "logps/rejected": -444.55194091796875, |
| "loss": 0.01, |
| "rewards/accuracies": 0.9952499866485596, |
| "rewards/chosen": -7.8769989013671875, |
| "rewards/margins": 13.22789192199707, |
| "rewards/rejected": -21.104890823364258, |
| "step": 20000 |
| }, |
| { |
| "epoch": 8.547008547008547, |
| "eval_logits/chosen": -34.138275146484375, |
| "eval_logits/rejected": -35.49238586425781, |
| "eval_logps/chosen": -389.0328369140625, |
| "eval_logps/rejected": -376.4501953125, |
| "eval_loss": 2.1092472076416016, |
| "eval_rewards/accuracies": 0.5593283772468567, |
| "eval_rewards/chosen": -13.413966178894043, |
| "eval_rewards/margins": 1.0751436948776245, |
| "eval_rewards/rejected": -14.48910903930664, |
| "eval_runtime": 227.7973, |
| "eval_samples_per_second": 11.734, |
| "eval_steps_per_second": 1.471, |
| "step": 20000 |
| }, |
| { |
| "epoch": 8.760683760683762, |
| "grad_norm": 1.0492886304855347, |
| "learning_rate": 6.3318777292576415e-06, |
| "logits/chosen": -33.439266204833984, |
| "logits/rejected": -34.95777893066406, |
| "logps/chosen": -331.6947326660156, |
| "logps/rejected": -441.31298828125, |
| "loss": 0.0126, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -7.511509418487549, |
| "rewards/margins": 13.192078590393066, |
| "rewards/rejected": -20.70358657836914, |
| "step": 20500 |
| }, |
| { |
| "epoch": 8.760683760683762, |
| "eval_logits/chosen": -34.35652542114258, |
| "eval_logits/rejected": -36.15835952758789, |
| "eval_logps/chosen": -387.4725646972656, |
| "eval_logps/rejected": -375.294189453125, |
| "eval_loss": 2.09629487991333, |
| "eval_rewards/accuracies": 0.562313437461853, |
| "eval_rewards/chosen": -13.25793743133545, |
| "eval_rewards/margins": 1.115572452545166, |
| "eval_rewards/rejected": -14.373512268066406, |
| "eval_runtime": 227.8406, |
| "eval_samples_per_second": 11.732, |
| "eval_steps_per_second": 1.47, |
| "step": 20500 |
| }, |
| { |
| "epoch": 8.974358974358974, |
| "grad_norm": 6.9876251220703125, |
| "learning_rate": 5.240174672489083e-06, |
| "logits/chosen": -33.97458267211914, |
| "logits/rejected": -35.557865142822266, |
| "logps/chosen": -330.39239501953125, |
| "logps/rejected": -444.573486328125, |
| "loss": 0.0119, |
| "rewards/accuracies": 0.9944999814033508, |
| "rewards/chosen": -7.664352893829346, |
| "rewards/margins": 13.433502197265625, |
| "rewards/rejected": -21.09785270690918, |
| "step": 21000 |
| }, |
| { |
| "epoch": 8.974358974358974, |
| "eval_logits/chosen": -34.37322235107422, |
| "eval_logits/rejected": -36.26004409790039, |
| "eval_logps/chosen": -386.285888671875, |
| "eval_logps/rejected": -373.96063232421875, |
| "eval_loss": 2.07737135887146, |
| "eval_rewards/accuracies": 0.5649253726005554, |
| "eval_rewards/chosen": -13.139269828796387, |
| "eval_rewards/margins": 1.1008845567703247, |
| "eval_rewards/rejected": -14.240155220031738, |
| "eval_runtime": 227.6828, |
| "eval_samples_per_second": 11.74, |
| "eval_steps_per_second": 1.471, |
| "step": 21000 |
| }, |
| { |
| "epoch": 9.188034188034187, |
| "grad_norm": 0.0024337973445653915, |
| "learning_rate": 4.1484716157205246e-06, |
| "logits/chosen": -33.696048736572266, |
| "logits/rejected": -35.321414947509766, |
| "logps/chosen": -330.987548828125, |
| "logps/rejected": -445.3059387207031, |
| "loss": 0.007, |
| "rewards/accuracies": 0.9957500100135803, |
| "rewards/chosen": -7.504194736480713, |
| "rewards/margins": 13.350516319274902, |
| "rewards/rejected": -20.854713439941406, |
| "step": 21500 |
| }, |
| { |
| "epoch": 9.188034188034187, |
| "eval_logits/chosen": -34.3525505065918, |
| "eval_logits/rejected": -36.3557243347168, |
| "eval_logps/chosen": -384.0542297363281, |
| "eval_logps/rejected": -371.8487243652344, |
| "eval_loss": 2.050776958465576, |
| "eval_rewards/accuracies": 0.5686567425727844, |
| "eval_rewards/chosen": -12.916106224060059, |
| "eval_rewards/margins": 1.1128581762313843, |
| "eval_rewards/rejected": -14.028963088989258, |
| "eval_runtime": 227.9747, |
| "eval_samples_per_second": 11.725, |
| "eval_steps_per_second": 1.469, |
| "step": 21500 |
| }, |
| { |
| "epoch": 9.401709401709402, |
| "grad_norm": 0.03277166187763214, |
| "learning_rate": 3.056768558951965e-06, |
| "logits/chosen": -34.06507873535156, |
| "logits/rejected": -35.69638442993164, |
| "logps/chosen": -332.1161804199219, |
| "logps/rejected": -444.4291076660156, |
| "loss": 0.0106, |
| "rewards/accuracies": 0.9950000047683716, |
| "rewards/chosen": -7.72641658782959, |
| "rewards/margins": 13.240401268005371, |
| "rewards/rejected": -20.966815948486328, |
| "step": 22000 |
| }, |
| { |
| "epoch": 9.401709401709402, |
| "eval_logits/chosen": -34.42190933227539, |
| "eval_logits/rejected": -36.19350051879883, |
| "eval_logps/chosen": -387.06207275390625, |
| "eval_logps/rejected": -374.70416259765625, |
| "eval_loss": 2.0756402015686035, |
| "eval_rewards/accuracies": 0.5649253726005554, |
| "eval_rewards/chosen": -13.216888427734375, |
| "eval_rewards/margins": 1.0976189374923706, |
| "eval_rewards/rejected": -14.314507484436035, |
| "eval_runtime": 227.5352, |
| "eval_samples_per_second": 11.748, |
| "eval_steps_per_second": 1.472, |
| "step": 22000 |
| } |
| ], |
| "logging_steps": 500, |
| "max_steps": 23400, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 10, |
| "save_steps": 1000, |
| "total_flos": 0.0, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
