| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.9927360774818403, | |
| "eval_steps": 500, | |
| "global_step": 618, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 2.0161290322580642e-07, | |
| "logps/chosen": -155.47128295898438, | |
| "logps/rejected": -104.8045883178711, | |
| "loss": 0.7536, | |
| "losses/dpo": 0.7361882925033569, | |
| "losses/sft": 1.3545048236846924, | |
| "losses/total": 0.7361882925033569, | |
| "ref_logps/chosen": -155.36932373046875, | |
| "ref_logps/rejected": -104.93973541259766, | |
| "rewards/accuracies": 0.48450005054473877, | |
| "rewards/chosen": -0.010196244344115257, | |
| "rewards/margins": -0.023710083216428757, | |
| "rewards/rejected": 0.0135138388723135, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 4.0322580645161285e-07, | |
| "logps/chosen": -153.7573699951172, | |
| "logps/rejected": -107.0270004272461, | |
| "loss": 0.7401, | |
| "losses/dpo": 0.7450882792472839, | |
| "losses/sft": 1.3266704082489014, | |
| "losses/total": 0.7450882792472839, | |
| "ref_logps/chosen": -153.8381805419922, | |
| "ref_logps/rejected": -107.06134033203125, | |
| "rewards/accuracies": 0.502500057220459, | |
| "rewards/chosen": 0.00808356050401926, | |
| "rewards/margins": 0.004649868700653315, | |
| "rewards/rejected": 0.003433691570535302, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 4.883093525179856e-07, | |
| "logps/chosen": -151.94009399414062, | |
| "logps/rejected": -105.40670013427734, | |
| "loss": 0.7213, | |
| "losses/dpo": 0.7041549682617188, | |
| "losses/sft": 1.3607875108718872, | |
| "losses/total": 0.7041549682617188, | |
| "ref_logps/chosen": -152.60272216796875, | |
| "ref_logps/rejected": -105.59368896484375, | |
| "rewards/accuracies": 0.5264999866485596, | |
| "rewards/chosen": 0.06626255810260773, | |
| "rewards/margins": 0.04756266996264458, | |
| "rewards/rejected": 0.01869989186525345, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 4.6582733812949637e-07, | |
| "logps/chosen": -149.70230102539062, | |
| "logps/rejected": -106.14666748046875, | |
| "loss": 0.6625, | |
| "losses/dpo": 0.6611780524253845, | |
| "losses/sft": 1.341475009918213, | |
| "losses/total": 0.6611780524253845, | |
| "ref_logps/chosen": -152.02659606933594, | |
| "ref_logps/rejected": -106.69593811035156, | |
| "rewards/accuracies": 0.621999979019165, | |
| "rewards/chosen": 0.2324293702840805, | |
| "rewards/margins": 0.17750366032123566, | |
| "rewards/rejected": 0.05492572858929634, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 4.4334532374100717e-07, | |
| "logps/chosen": -148.40090942382812, | |
| "logps/rejected": -104.86791229248047, | |
| "loss": 0.597, | |
| "losses/dpo": 0.5830589532852173, | |
| "losses/sft": 1.2935805320739746, | |
| "losses/total": 0.5830589532852173, | |
| "ref_logps/chosen": -152.3332977294922, | |
| "ref_logps/rejected": -105.52149200439453, | |
| "rewards/accuracies": 0.7135000824928284, | |
| "rewards/chosen": 0.39323729276657104, | |
| "rewards/margins": 0.32787904143333435, | |
| "rewards/rejected": 0.0653582364320755, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 4.2086330935251797e-07, | |
| "logps/chosen": -147.95791625976562, | |
| "logps/rejected": -106.17552947998047, | |
| "loss": 0.5515, | |
| "losses/dpo": 0.5297109484672546, | |
| "losses/sft": 1.289250135421753, | |
| "losses/total": 0.5297109484672546, | |
| "ref_logps/chosen": -153.35595703125, | |
| "ref_logps/rejected": -106.98403930664062, | |
| "rewards/accuracies": 0.7540000677108765, | |
| "rewards/chosen": 0.5398061871528625, | |
| "rewards/margins": 0.45895519852638245, | |
| "rewards/rejected": 0.0808510109782219, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 3.9838129496402877e-07, | |
| "logps/chosen": -147.28538513183594, | |
| "logps/rejected": -106.73645782470703, | |
| "loss": 0.5171, | |
| "losses/dpo": 0.526803731918335, | |
| "losses/sft": 1.2763028144836426, | |
| "losses/total": 0.526803731918335, | |
| "ref_logps/chosen": -153.7596893310547, | |
| "ref_logps/rejected": -107.54974365234375, | |
| "rewards/accuracies": 0.7815000414848328, | |
| "rewards/chosen": 0.6474303603172302, | |
| "rewards/margins": 0.5661011934280396, | |
| "rewards/rejected": 0.0813291072845459, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 3.7589928057553957e-07, | |
| "logps/chosen": -146.67481994628906, | |
| "logps/rejected": -104.6893539428711, | |
| "loss": 0.4544, | |
| "losses/dpo": 0.4492318332195282, | |
| "losses/sft": 1.284212350845337, | |
| "losses/total": 0.4492318332195282, | |
| "ref_logps/chosen": -154.76519775390625, | |
| "ref_logps/rejected": -105.09065246582031, | |
| "rewards/accuracies": 0.825499951839447, | |
| "rewards/chosen": 0.809037446975708, | |
| "rewards/margins": 0.7689078450202942, | |
| "rewards/rejected": 0.040129706263542175, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 1.09, | |
| "learning_rate": 3.5341726618705037e-07, | |
| "logps/chosen": -144.06991577148438, | |
| "logps/rejected": -105.81298828125, | |
| "loss": 0.4189, | |
| "losses/dpo": 0.42898857593536377, | |
| "losses/sft": 1.2516932487487793, | |
| "losses/total": 0.42898857593536377, | |
| "ref_logps/chosen": -153.25570678710938, | |
| "ref_logps/rejected": -106.06501007080078, | |
| "rewards/accuracies": 0.8469999432563782, | |
| "rewards/chosen": 0.9185801148414612, | |
| "rewards/margins": 0.8933786749839783, | |
| "rewards/rejected": 0.02520136535167694, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 1.21, | |
| "learning_rate": 3.309352517985611e-07, | |
| "logps/chosen": -144.94163513183594, | |
| "logps/rejected": -106.42683410644531, | |
| "loss": 0.3982, | |
| "losses/dpo": 0.41044384241104126, | |
| "losses/sft": 1.2750244140625, | |
| "losses/total": 0.41044384241104126, | |
| "ref_logps/chosen": -154.86021423339844, | |
| "ref_logps/rejected": -106.34297180175781, | |
| "rewards/accuracies": 0.8530000448226929, | |
| "rewards/chosen": 0.9918593764305115, | |
| "rewards/margins": 1.0002468824386597, | |
| "rewards/rejected": -0.008387637324631214, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 1.33, | |
| "learning_rate": 3.0845323741007197e-07, | |
| "logps/chosen": -144.10977172851562, | |
| "logps/rejected": -105.52392578125, | |
| "loss": 0.3652, | |
| "losses/dpo": 0.3518593907356262, | |
| "losses/sft": 1.2418614625930786, | |
| "losses/total": 0.3518593907356262, | |
| "ref_logps/chosen": -154.4581298828125, | |
| "ref_logps/rejected": -104.62450408935547, | |
| "rewards/accuracies": 0.8764999508857727, | |
| "rewards/chosen": 1.0348376035690308, | |
| "rewards/margins": 1.1247813701629639, | |
| "rewards/rejected": -0.08994373679161072, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 1.45, | |
| "learning_rate": 2.859712230215827e-07, | |
| "logps/chosen": -142.1906280517578, | |
| "logps/rejected": -108.99441528320312, | |
| "loss": 0.3624, | |
| "losses/dpo": 0.35955867171287537, | |
| "losses/sft": 1.2526122331619263, | |
| "losses/total": 0.35955867171287537, | |
| "ref_logps/chosen": -152.7955322265625, | |
| "ref_logps/rejected": -107.89449310302734, | |
| "rewards/accuracies": 0.8740000128746033, | |
| "rewards/chosen": 1.0604920387268066, | |
| "rewards/margins": 1.170485496520996, | |
| "rewards/rejected": -0.10999350249767303, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 1.57, | |
| "learning_rate": 2.634892086330935e-07, | |
| "logps/chosen": -141.9879913330078, | |
| "logps/rejected": -107.29058837890625, | |
| "loss": 0.349, | |
| "losses/dpo": 0.3294835686683655, | |
| "losses/sft": 1.2578469514846802, | |
| "losses/total": 0.3294835686683655, | |
| "ref_logps/chosen": -152.93646240234375, | |
| "ref_logps/rejected": -105.73262786865234, | |
| "rewards/accuracies": 0.8819999694824219, | |
| "rewards/chosen": 1.0948452949523926, | |
| "rewards/margins": 1.2506405115127563, | |
| "rewards/rejected": -0.15579518675804138, | |
| "step": 325 | |
| }, | |
| { | |
| "epoch": 1.69, | |
| "learning_rate": 2.410071942446043e-07, | |
| "logps/chosen": -142.55641174316406, | |
| "logps/rejected": -108.43463134765625, | |
| "loss": 0.3091, | |
| "losses/dpo": 0.3135191798210144, | |
| "losses/sft": 1.2662179470062256, | |
| "losses/total": 0.3135191798210144, | |
| "ref_logps/chosen": -154.17462158203125, | |
| "ref_logps/rejected": -105.654052734375, | |
| "rewards/accuracies": 0.8995000720024109, | |
| "rewards/chosen": 1.1618221998214722, | |
| "rewards/margins": 1.4398791790008545, | |
| "rewards/rejected": -0.2780568599700928, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 1.82, | |
| "learning_rate": 2.185251798561151e-07, | |
| "logps/chosen": -141.23895263671875, | |
| "logps/rejected": -108.13822937011719, | |
| "loss": 0.3071, | |
| "losses/dpo": 0.299526184797287, | |
| "losses/sft": 1.2248637676239014, | |
| "losses/total": 0.299526184797287, | |
| "ref_logps/chosen": -153.57838439941406, | |
| "ref_logps/rejected": -105.29540252685547, | |
| "rewards/accuracies": 0.8889999389648438, | |
| "rewards/chosen": 1.2339427471160889, | |
| "rewards/margins": 1.5182260274887085, | |
| "rewards/rejected": -0.28428351879119873, | |
| "step": 375 | |
| }, | |
| { | |
| "epoch": 1.94, | |
| "learning_rate": 1.960431654676259e-07, | |
| "logps/chosen": -143.84800720214844, | |
| "logps/rejected": -109.23783111572266, | |
| "loss": 0.3079, | |
| "losses/dpo": 0.31919562816619873, | |
| "losses/sft": 1.2433806657791138, | |
| "losses/total": 0.31919562816619873, | |
| "ref_logps/chosen": -155.57464599609375, | |
| "ref_logps/rejected": -105.85549926757812, | |
| "rewards/accuracies": 0.8924999237060547, | |
| "rewards/chosen": 1.1726653575897217, | |
| "rewards/margins": 1.510898470878601, | |
| "rewards/rejected": -0.33823302388191223, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 2.06, | |
| "learning_rate": 1.735611510791367e-07, | |
| "logps/chosen": -142.04263305664062, | |
| "logps/rejected": -110.5551528930664, | |
| "loss": 0.2964, | |
| "losses/dpo": 0.29985514283180237, | |
| "losses/sft": 1.256807565689087, | |
| "losses/total": 0.29985514283180237, | |
| "ref_logps/chosen": -153.96051025390625, | |
| "ref_logps/rejected": -106.5252914428711, | |
| "rewards/accuracies": 0.8864999413490295, | |
| "rewards/chosen": 1.1917892694473267, | |
| "rewards/margins": 1.5947766304016113, | |
| "rewards/rejected": -0.4029873013496399, | |
| "step": 425 | |
| }, | |
| { | |
| "epoch": 2.18, | |
| "learning_rate": 1.5107913669064747e-07, | |
| "logps/chosen": -141.11489868164062, | |
| "logps/rejected": -110.66223907470703, | |
| "loss": 0.2921, | |
| "losses/dpo": 0.29900702834129333, | |
| "losses/sft": 1.223684549331665, | |
| "losses/total": 0.29900702834129333, | |
| "ref_logps/chosen": -153.32489013671875, | |
| "ref_logps/rejected": -106.36825561523438, | |
| "rewards/accuracies": 0.9029999375343323, | |
| "rewards/chosen": 1.2209986448287964, | |
| "rewards/margins": 1.6503956317901611, | |
| "rewards/rejected": -0.42939692735671997, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 2.3, | |
| "learning_rate": 1.2859712230215827e-07, | |
| "logps/chosen": -140.06715393066406, | |
| "logps/rejected": -111.3785629272461, | |
| "loss": 0.2771, | |
| "losses/dpo": 0.2982315719127655, | |
| "losses/sft": 1.2262225151062012, | |
| "losses/total": 0.2982315719127655, | |
| "ref_logps/chosen": -152.20010375976562, | |
| "ref_logps/rejected": -106.2849349975586, | |
| "rewards/accuracies": 0.8995000720024109, | |
| "rewards/chosen": 1.2132951021194458, | |
| "rewards/margins": 1.722659707069397, | |
| "rewards/rejected": -0.509364664554596, | |
| "step": 475 | |
| }, | |
| { | |
| "epoch": 2.42, | |
| "learning_rate": 1.0611510791366907e-07, | |
| "logps/chosen": -142.41409301757812, | |
| "logps/rejected": -112.02964782714844, | |
| "loss": 0.2692, | |
| "losses/dpo": 0.26506972312927246, | |
| "losses/sft": 1.2326300144195557, | |
| "losses/total": 0.26506972312927246, | |
| "ref_logps/chosen": -154.93585205078125, | |
| "ref_logps/rejected": -106.74388885498047, | |
| "rewards/accuracies": 0.9050000905990601, | |
| "rewards/chosen": 1.252176284790039, | |
| "rewards/margins": 1.7807520627975464, | |
| "rewards/rejected": -0.5285759568214417, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 2.54, | |
| "learning_rate": 8.363309352517984e-08, | |
| "logps/chosen": -141.35763549804688, | |
| "logps/rejected": -111.21255493164062, | |
| "loss": 0.2639, | |
| "losses/dpo": 0.2448505461215973, | |
| "losses/sft": 1.2256782054901123, | |
| "losses/total": 0.2448505461215973, | |
| "ref_logps/chosen": -153.86410522460938, | |
| "ref_logps/rejected": -105.44302368164062, | |
| "rewards/accuracies": 0.9070001244544983, | |
| "rewards/chosen": 1.250647783279419, | |
| "rewards/margins": 1.8276008367538452, | |
| "rewards/rejected": -0.576953113079071, | |
| "step": 525 | |
| }, | |
| { | |
| "epoch": 2.66, | |
| "learning_rate": 6.115107913669064e-08, | |
| "logps/chosen": -140.52743530273438, | |
| "logps/rejected": -111.19217681884766, | |
| "loss": 0.253, | |
| "losses/dpo": 0.2362019419670105, | |
| "losses/sft": 1.219558835029602, | |
| "losses/total": 0.2362019419670105, | |
| "ref_logps/chosen": -153.1510009765625, | |
| "ref_logps/rejected": -104.75523376464844, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": 1.2623597383499146, | |
| "rewards/margins": 1.9060544967651367, | |
| "rewards/rejected": -0.6436949372291565, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 2.78, | |
| "learning_rate": 3.866906474820144e-08, | |
| "logps/chosen": -142.2875213623047, | |
| "logps/rejected": -113.81513977050781, | |
| "loss": 0.2529, | |
| "losses/dpo": 0.24827653169631958, | |
| "losses/sft": 1.2383555173873901, | |
| "losses/total": 0.24827653169631958, | |
| "ref_logps/chosen": -154.96365356445312, | |
| "ref_logps/rejected": -107.25000762939453, | |
| "rewards/accuracies": 0.9150000214576721, | |
| "rewards/chosen": 1.2676140069961548, | |
| "rewards/margins": 1.9241262674331665, | |
| "rewards/rejected": -0.656512439250946, | |
| "step": 575 | |
| }, | |
| { | |
| "epoch": 2.91, | |
| "learning_rate": 1.618705035971223e-08, | |
| "logps/chosen": -140.9930877685547, | |
| "logps/rejected": -113.36444091796875, | |
| "loss": 0.2651, | |
| "losses/dpo": 0.27665671706199646, | |
| "losses/sft": 1.2188746929168701, | |
| "losses/total": 0.27665671706199646, | |
| "ref_logps/chosen": -153.5058135986328, | |
| "ref_logps/rejected": -107.16682434082031, | |
| "rewards/accuracies": 0.9030001163482666, | |
| "rewards/chosen": 1.251274585723877, | |
| "rewards/margins": 1.871036410331726, | |
| "rewards/rejected": -0.6197616457939148, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 2.99, | |
| "step": 618, | |
| "total_flos": 0.0, | |
| "train_loss": 0.41181609237078326, | |
| "train_runtime": 7630.1245, | |
| "train_samples_per_second": 6.494, | |
| "train_steps_per_second": 0.081 | |
| } | |
| ], | |
| "logging_steps": 25, | |
| "max_steps": 618, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "stateful_callbacks": {}, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |