| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 7.5, |
| "eval_steps": 10, |
| "global_step": 30, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "completion_length": 25.59375, |
| "default/reward": 0.5906250178813934, |
| "default/reward_std": 0.5985057950019836, |
| "default/rewards/llm_judge_reward_function": 0.6000000089406967, |
| "epoch": 0.25, |
| "grad_norm": 4.563571929931641, |
| "kl": 0.0, |
| "learning_rate": 5e-06, |
| "loss": 0.0, |
| "step": 1 |
| }, |
| { |
| "completion_length": 29.46875, |
| "default/reward": 0.6906249821186066, |
| "default/reward_std": 0.8958407640457153, |
| "default/rewards/llm_judge_reward_function": 0.8499999940395355, |
| "epoch": 0.5, |
| "grad_norm": 2.090252637863159, |
| "kl": 0.00035738608858082443, |
| "learning_rate": 4.999143312438893e-06, |
| "loss": 0.0068, |
| "step": 2 |
| }, |
| { |
| "completion_length": 32.25, |
| "default/reward": 0.7562499940395355, |
| "default/reward_std": 0.6679378151893616, |
| "default/rewards/llm_judge_reward_function": 0.42500001192092896, |
| "epoch": 0.75, |
| "grad_norm": 3.4133903980255127, |
| "kl": 0.0035452890442684293, |
| "learning_rate": 4.9965738368864345e-06, |
| "loss": 0.0, |
| "step": 3 |
| }, |
| { |
| "completion_length": 23.416667938232422, |
| "default/reward": 0.734375, |
| "default/reward_std": 0.7354854047298431, |
| "default/rewards/llm_judge_reward_function": 0.5, |
| "epoch": 1.0, |
| "grad_norm": 4.797443389892578, |
| "kl": 0.003295191505458206, |
| "learning_rate": 4.992293334332821e-06, |
| "loss": 0.0, |
| "step": 4 |
| }, |
| { |
| "completion_length": 27.9375, |
| "default/reward": 0.546875, |
| "default/reward_std": 0.6988737881183624, |
| "default/rewards/llm_judge_reward_function": 0.925000011920929, |
| "epoch": 1.25, |
| "grad_norm": 2.957021713256836, |
| "kl": 0.013654953800141811, |
| "learning_rate": 4.986304738420684e-06, |
| "loss": 0.0001, |
| "step": 5 |
| }, |
| { |
| "completion_length": 33.71875, |
| "default/reward": 0.653124988079071, |
| "default/reward_std": 0.69887375831604, |
| "default/rewards/llm_judge_reward_function": 0.925000011920929, |
| "epoch": 1.5, |
| "grad_norm": 2.791567802429199, |
| "kl": 0.015739883296191692, |
| "learning_rate": 4.978612153434527e-06, |
| "loss": 0.0002, |
| "step": 6 |
| }, |
| { |
| "completion_length": 30.53125, |
| "default/reward": 0.768750011920929, |
| "default/reward_std": 0.8825825154781342, |
| "default/rewards/llm_judge_reward_function": 0.6749999970197678, |
| "epoch": 1.75, |
| "grad_norm": 2.669797658920288, |
| "kl": 0.021481021773070097, |
| "learning_rate": 4.9692208514878445e-06, |
| "loss": 0.0002, |
| "step": 7 |
| }, |
| { |
| "completion_length": 29.791666984558105, |
| "default/reward": 0.6937500238418579, |
| "default/reward_std": 0.7657932043075562, |
| "default/rewards/llm_judge_reward_function": 0.675000011920929, |
| "epoch": 2.0, |
| "grad_norm": 2.478447437286377, |
| "kl": 0.04025824833661318, |
| "learning_rate": 4.958137268909887e-06, |
| "loss": 0.0004, |
| "step": 8 |
| }, |
| { |
| "completion_length": 39.15625, |
| "default/reward": 0.737500011920929, |
| "default/reward_std": 0.6673097014427185, |
| "default/rewards/llm_judge_reward_function": 0.675000011920929, |
| "epoch": 2.25, |
| "grad_norm": 2.5778961181640625, |
| "kl": 0.026319073513150215, |
| "learning_rate": 4.9453690018345144e-06, |
| "loss": 0.0003, |
| "step": 9 |
| }, |
| { |
| "completion_length": 28.46875, |
| "default/reward": 0.7218749821186066, |
| "default/reward_std": 0.7424398064613342, |
| "default/rewards/llm_judge_reward_function": 0.5, |
| "epoch": 2.5, |
| "grad_norm": 4.112423896789551, |
| "kl": 0.09163418784737587, |
| "learning_rate": 4.930924800994192e-06, |
| "loss": 0.0009, |
| "step": 10 |
| }, |
| { |
| "epoch": 2.5, |
| "eval_default/reward": 0.8224999904632568, |
| "eval_default/reward_std": 0.6798096895217896, |
| "eval_default/rewards/llm_judge_reward_function": 0.7399999976158143, |
| "eval_loss": 0.0, |
| "eval_runtime": 432.2348, |
| "eval_samples_per_second": 0.046, |
| "eval_steps_per_second": 0.007, |
| "step": 10 |
| }, |
| { |
| "completion_length": 25.09375, |
| "default/reward": 0.893750011920929, |
| "default/reward_std": 0.5877601504325867, |
| "default/rewards/llm_judge_reward_function": 0.8500000238418579, |
| "epoch": 2.75, |
| "grad_norm": 2.8137168884277344, |
| "kl": 0.05960903689265251, |
| "learning_rate": 4.914814565722671e-06, |
| "loss": 0.0006, |
| "step": 11 |
| }, |
| { |
| "completion_length": 33.20833396911621, |
| "default/reward": 0.8531250059604645, |
| "default/reward_std": 0.8522747457027435, |
| "default/rewards/llm_judge_reward_function": 0.8499999940395355, |
| "epoch": 3.0, |
| "grad_norm": 1.4571622610092163, |
| "kl": 0.06877592019736767, |
| "learning_rate": 4.897049337170483e-06, |
| "loss": 0.0007, |
| "step": 12 |
| }, |
| { |
| "completion_length": 30.59375, |
| "default/reward": 0.7874999940395355, |
| "default/reward_std": 0.6849873661994934, |
| "default/rewards/llm_judge_reward_function": 0.925000011920929, |
| "epoch": 3.25, |
| "grad_norm": 2.8570544719696045, |
| "kl": 0.05570350959897041, |
| "learning_rate": 4.8776412907378845e-06, |
| "loss": 0.0006, |
| "step": 13 |
| }, |
| { |
| "completion_length": 39.25, |
| "default/reward": 0.7281250357627869, |
| "default/reward_std": 0.7159232497215271, |
| "default/rewards/llm_judge_reward_function": 0.925000011920929, |
| "epoch": 3.5, |
| "grad_norm": 2.8516972064971924, |
| "kl": 0.10984514653682709, |
| "learning_rate": 4.856603727730446e-06, |
| "loss": 0.0011, |
| "step": 14 |
| }, |
| { |
| "completion_length": 29.09375, |
| "default/reward": 0.824999988079071, |
| "default/reward_std": 0.9084708988666534, |
| "default/rewards/llm_judge_reward_function": 0.75, |
| "epoch": 3.75, |
| "grad_norm": 3.0547258853912354, |
| "kl": 0.177144393324852, |
| "learning_rate": 4.833951066243004e-06, |
| "loss": 0.0018, |
| "step": 15 |
| }, |
| { |
| "completion_length": 29.500000953674316, |
| "default/reward": 0.8812499940395355, |
| "default/reward_std": 0.8649048507213593, |
| "default/rewards/llm_judge_reward_function": 1.0, |
| "epoch": 4.0, |
| "grad_norm": 1.7537461519241333, |
| "kl": 0.08436176925897598, |
| "learning_rate": 4.809698831278217e-06, |
| "loss": 0.0008, |
| "step": 16 |
| }, |
| { |
| "completion_length": 35.625, |
| "default/reward": 0.8687499761581421, |
| "default/reward_std": 0.8200824856758118, |
| "default/rewards/llm_judge_reward_function": 1.0, |
| "epoch": 4.25, |
| "grad_norm": 3.1518936157226562, |
| "kl": 0.09947774186730385, |
| "learning_rate": 4.783863644106502e-06, |
| "loss": 0.001, |
| "step": 17 |
| }, |
| { |
| "completion_length": 43.34375, |
| "default/reward": 0.5843750238418579, |
| "default/reward_std": 0.715295135974884, |
| "default/rewards/llm_judge_reward_function": 0.5999999940395355, |
| "epoch": 4.5, |
| "grad_norm": 3.3874547481536865, |
| "kl": 0.11295061185956001, |
| "learning_rate": 4.7564632108746524e-06, |
| "loss": 0.0011, |
| "step": 18 |
| }, |
| { |
| "completion_length": 32.34375, |
| "default/reward": 0.734375, |
| "default/reward_std": 0.7973572611808777, |
| "default/rewards/llm_judge_reward_function": 1.0, |
| "epoch": 4.75, |
| "grad_norm": 2.7637128829956055, |
| "kl": 0.1756831854581833, |
| "learning_rate": 4.72751631047092e-06, |
| "loss": 0.0018, |
| "step": 19 |
| }, |
| { |
| "completion_length": 39.45833396911621, |
| "default/reward": 0.7562500238418579, |
| "default/reward_std": 0.8383883237838745, |
| "default/rewards/llm_judge_reward_function": 0.5249999910593033, |
| "epoch": 5.0, |
| "grad_norm": 1.7747427225112915, |
| "kl": 0.13021686673164368, |
| "learning_rate": 4.697042781654913e-06, |
| "loss": 0.0013, |
| "step": 20 |
| }, |
| { |
| "epoch": 5.0, |
| "eval_default/reward": 0.8600000143051147, |
| "eval_default/reward_std": 0.7343502879142761, |
| "eval_default/rewards/llm_judge_reward_function": 0.8799999952316284, |
| "eval_loss": 0.0, |
| "eval_runtime": 431.4917, |
| "eval_samples_per_second": 0.046, |
| "eval_steps_per_second": 0.007, |
| "step": 20 |
| }, |
| { |
| "completion_length": 28.53125, |
| "default/reward": 0.6343749910593033, |
| "default/reward_std": 0.6717291176319122, |
| "default/rewards/llm_judge_reward_function": 0.8500000238418579, |
| "epoch": 5.25, |
| "grad_norm": 3.542097330093384, |
| "kl": 0.15948720276355743, |
| "learning_rate": 4.665063509461098e-06, |
| "loss": 0.0016, |
| "step": 21 |
| }, |
| { |
| "completion_length": 31.21875, |
| "default/reward": 0.8375000059604645, |
| "default/reward_std": 0.8649048805236816, |
| "default/rewards/llm_judge_reward_function": 1.0, |
| "epoch": 5.5, |
| "grad_norm": 2.465200662612915, |
| "kl": 0.16199305281043053, |
| "learning_rate": 4.631600410885231e-06, |
| "loss": 0.0016, |
| "step": 22 |
| }, |
| { |
| "completion_length": 25.5625, |
| "default/reward": 0.934374988079071, |
| "default/reward_std": 0.9507582485675812, |
| "default/rewards/llm_judge_reward_function": 0.8499999940395355, |
| "epoch": 5.75, |
| "grad_norm": 0.9499327540397644, |
| "kl": 0.25974033027887344, |
| "learning_rate": 4.596676419863561e-06, |
| "loss": -0.015, |
| "step": 23 |
| }, |
| { |
| "completion_length": 24.625, |
| "default/reward": 0.7781250178813934, |
| "default/reward_std": 0.6982456147670746, |
| "default/rewards/llm_judge_reward_function": 0.5, |
| "epoch": 6.0, |
| "grad_norm": 3.659409999847412, |
| "kl": 0.317486435174942, |
| "learning_rate": 4.560315471555039e-06, |
| "loss": 0.0032, |
| "step": 24 |
| }, |
| { |
| "completion_length": 24.1875, |
| "default/reward": 0.7343750298023224, |
| "default/reward_std": 0.5915514230728149, |
| "default/rewards/llm_judge_reward_function": 0.925000011920929, |
| "epoch": 6.25, |
| "grad_norm": 9.055826187133789, |
| "kl": 0.19061780348420143, |
| "learning_rate": 4.522542485937369e-06, |
| "loss": 0.0019, |
| "step": 25 |
| }, |
| { |
| "completion_length": 34.84375, |
| "default/reward": 0.875, |
| "default/reward_std": 0.784727156162262, |
| "default/rewards/llm_judge_reward_function": 0.8499999940395355, |
| "epoch": 6.5, |
| "grad_norm": 3.1907670497894287, |
| "kl": 0.30184144526720047, |
| "learning_rate": 4.4833833507280884e-06, |
| "loss": 0.003, |
| "step": 26 |
| }, |
| { |
| "completion_length": 22.625, |
| "default/reward": 0.828125, |
| "default/reward_std": 0.8522747457027435, |
| "default/rewards/llm_judge_reward_function": 0.42500001192092896, |
| "epoch": 6.75, |
| "grad_norm": 2.9145781993865967, |
| "kl": 0.39268461614847183, |
| "learning_rate": 4.442864903642428e-06, |
| "loss": -0.0043, |
| "step": 27 |
| }, |
| { |
| "completion_length": 37.29166793823242, |
| "default/reward": 0.7187500298023224, |
| "default/reward_std": 0.6666815280914307, |
| "default/rewards/llm_judge_reward_function": 1.0, |
| "epoch": 7.0, |
| "grad_norm": 4.502452850341797, |
| "kl": 0.2312646582722664, |
| "learning_rate": 4.401014914000078e-06, |
| "loss": 0.0023, |
| "step": 28 |
| }, |
| { |
| "completion_length": 27.46875, |
| "default/reward": 0.796875, |
| "default/reward_std": 0.6988737881183624, |
| "default/rewards/llm_judge_reward_function": 0.925000011920929, |
| "epoch": 7.25, |
| "grad_norm": 3.6535165309906006, |
| "kl": 0.2524450868368149, |
| "learning_rate": 4.357862063693486e-06, |
| "loss": 0.0025, |
| "step": 29 |
| }, |
| { |
| "completion_length": 26.90625, |
| "default/reward": 0.84375, |
| "default/reward_std": 0.8030329942703247, |
| "default/rewards/llm_judge_reward_function": 0.42500001192092896, |
| "epoch": 7.5, |
| "grad_norm": 1.068318247795105, |
| "kl": 0.20408491045236588, |
| "learning_rate": 4.313435927530719e-06, |
| "loss": 0.002, |
| "step": 30 |
| }, |
| { |
| "epoch": 7.5, |
| "eval_default/reward": 0.9025000095367431, |
| "eval_default/reward_std": 0.7242461919784546, |
| "eval_default/rewards/llm_judge_reward_function": 0.9399999976158142, |
| "eval_loss": 0.0, |
| "eval_runtime": 422.8471, |
| "eval_samples_per_second": 0.047, |
| "eval_steps_per_second": 0.007, |
| "step": 30 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 120, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 30, |
| "save_steps": 10, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
