{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.0005, "eval_steps": 500, "global_step": 50, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9365.0, "completions/max_terminated_length": 9365.0, "completions/mean_length": 8020.65625, "completions/mean_terminated_length": 8020.65625, "completions/min_length": 4249.0, "completions/min_terminated_length": 4249.0, "entropy": 0.09263912006281316, "epoch": 1e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.3079737424850464, "kl": 0.0, "learning_rate": 0.0, "loss": -0.0363, "num_tokens": 283668.0, "reward": -0.4597676992416382, "reward_std": 0.3370218873023987, "rewards/rollout_reward_func/mean": -0.4597676992416382, "rewards/rollout_reward_func/std": 0.37389153242111206, "sampling/importance_sampling_ratio/max": 1.752305507659912, "sampling/importance_sampling_ratio/mean": 0.9996651411056519, "sampling/importance_sampling_ratio/min": 0.4149494767189026, "sampling/sampling_logp_difference/max": 0.8795984983444214, "sampling/sampling_logp_difference/mean": 0.009596828371286392, "step": 1, "step_time": 96.616219112002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.09263912006281316, "epoch": 2e-05, "grad_norm": 1.297884225845337, "kl": 0.0, "learning_rate": 2.8571428571428573e-06, "loss": -0.0363, "step": 2, "step_time": 28.28962996199698 }, { "clip_ratio/high_max": 0.005740102380514145, "clip_ratio/high_mean": 0.0028700511902570724, "clip_ratio/low_mean": 0.0018344554118812084, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004704506660345942, "completions/clipped_ratio": 0.0, "completions/max_length": 9371.0, "completions/max_terminated_length": 9371.0, "completions/mean_length": 7904.6875, "completions/mean_terminated_length": 7904.6875, "completions/min_length": 2043.0, "completions/min_terminated_length": 2043.0, "entropy": 0.11998538533225656, "epoch": 3e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.5289855003356934, "kl": 0.0012301416445552604, "learning_rate": 5.7142857142857145e-06, "loss": -0.0457, "num_tokens": 564087.0, "reward": -0.5441880226135254, "reward_std": 0.3924558162689209, "rewards/rollout_reward_func/mean": -0.5441880226135254, "rewards/rollout_reward_func/std": 0.42554038763046265, "sampling/importance_sampling_ratio/max": 1.5868604183197021, "sampling/importance_sampling_ratio/mean": 0.9994746446609497, "sampling/importance_sampling_ratio/min": 0.5226752758026123, "sampling/sampling_logp_difference/max": 0.6487948894500732, "sampling/sampling_logp_difference/mean": 0.011393455788493156, "step": 3, "step_time": 96.45885010200436 }, { "clip_ratio/high_max": 0.005765739129856229, "clip_ratio/high_mean": 0.0028828695649281144, "clip_ratio/low_mean": 0.0021901361469645053, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00507300574099645, "entropy": 0.12003705860115588, "epoch": 4e-05, "grad_norm": 1.2704079151153564, "kl": 0.001491897305186285, "learning_rate": 8.571428571428573e-06, "loss": -0.047, "step": 4, "step_time": 29.195899179991102 }, { "clip_ratio/high_max": 0.005963329633232206, "clip_ratio/high_mean": 0.002981664816616103, "clip_ratio/low_mean": 0.0014880952658131719, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004469760082429275, "completions/clipped_ratio": 0.0, "completions/max_length": 8882.0, "completions/max_terminated_length": 8882.0, "completions/mean_length": 7734.1875, "completions/mean_terminated_length": 7734.1875, "completions/min_length": 1783.0, "completions/min_terminated_length": 1783.0, "entropy": 0.09044346120208502, "epoch": 5e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.5453312397003174, "kl": 0.0007876164445406175, "learning_rate": 1.1428571428571429e-05, "loss": -0.0298, "num_tokens": 838350.0, "reward": -0.5028736591339111, "reward_std": 0.371159166097641, "rewards/rollout_reward_func/mean": -0.5028736591339111, "rewards/rollout_reward_func/std": 0.38670846819877625, "sampling/importance_sampling_ratio/max": 1.758213996887207, "sampling/importance_sampling_ratio/mean": 0.9994957447052002, "sampling/importance_sampling_ratio/min": 0.3028993308544159, "sampling/sampling_logp_difference/max": 1.194354772567749, "sampling/sampling_logp_difference/mean": 0.009732999838888645, "step": 5, "step_time": 93.71373151599619 }, { "clip_ratio/high_max": 0.006641630490776151, "clip_ratio/high_mean": 0.004027052520541474, "clip_ratio/low_mean": 0.002363648993195966, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00639070151373744, "entropy": 0.08915826864540577, "epoch": 6e-05, "grad_norm": 1.28467857837677, "kl": 0.0010278278514306294, "learning_rate": 1.4285714285714285e-05, "loss": -0.0305, "step": 6, "step_time": 27.6442580659932 }, { "clip_ratio/high_max": 0.006255197920836508, "clip_ratio/high_mean": 0.003127598960418254, "clip_ratio/low_mean": 0.0010739664430730045, "clip_ratio/low_min": 0.0006944444612599909, "clip_ratio/region_mean": 0.004201565403491259, "completions/clipped_ratio": 0.0, "completions/max_length": 9327.0, "completions/max_terminated_length": 9327.0, "completions/mean_length": 7944.875, "completions/mean_terminated_length": 7944.875, "completions/min_length": 1618.0, "completions/min_terminated_length": 1618.0, "entropy": 0.0936605577589944, "epoch": 7e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.6153287887573242, "kl": 0.001322501270351495, "learning_rate": 1.7142857142857145e-05, "loss": -0.04, "num_tokens": 1120382.0, "reward": -0.40193918347358704, "reward_std": 0.42798829078674316, "rewards/rollout_reward_func/mean": -0.40193918347358704, "rewards/rollout_reward_func/std": 0.45951688289642334, "sampling/importance_sampling_ratio/max": 2.626508951187134, "sampling/importance_sampling_ratio/mean": 0.9997774362564087, "sampling/importance_sampling_ratio/min": 0.4664866328239441, "sampling/sampling_logp_difference/max": 0.9656555652618408, "sampling/sampling_logp_difference/mean": 0.011292756535112858, "step": 7, "step_time": 93.59508053499303 }, { "clip_ratio/high_max": 0.004923702392261475, "clip_ratio/high_mean": 0.0028252233169041574, "clip_ratio/low_mean": 0.006011958757881075, "clip_ratio/low_min": 0.0007102272938936949, "clip_ratio/region_mean": 0.008837182191200554, "entropy": 0.08954936638474464, "epoch": 8e-05, "grad_norm": 1.4120911359786987, "kl": 0.0029542466588736715, "learning_rate": 2e-05, "loss": -0.0404, "step": 8, "step_time": 28.796026483003516 }, { "clip_ratio/high_max": 0.004531709826551378, "clip_ratio/high_mean": 0.00261697854148224, "clip_ratio/low_mean": 0.0036588753282558173, "clip_ratio/low_min": 0.0014044943964108825, "clip_ratio/region_mean": 0.006275853869738057, "completions/clipped_ratio": 0.0, "completions/max_length": 9217.0, "completions/max_terminated_length": 9217.0, "completions/mean_length": 7774.4375, "completions/mean_terminated_length": 7774.4375, "completions/min_length": 860.0, "completions/min_terminated_length": 860.0, "entropy": 0.09250054159201682, "epoch": 9e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.8319895267486572, "kl": 0.003183710745361168, "learning_rate": 2.2857142857142858e-05, "loss": -0.0399, "num_tokens": 1396422.0, "reward": -0.35600194334983826, "reward_std": 0.46349450945854187, "rewards/rollout_reward_func/mean": -0.35600194334983826, "rewards/rollout_reward_func/std": 0.4859367907047272, "sampling/importance_sampling_ratio/max": 2.4923911094665527, "sampling/importance_sampling_ratio/mean": 0.9995814561843872, "sampling/importance_sampling_ratio/min": 0.34728506207466125, "sampling/sampling_logp_difference/max": 1.0576093196868896, "sampling/sampling_logp_difference/mean": 0.011536622419953346, "step": 9, "step_time": 91.41392734301189 }, { "clip_ratio/high_max": 0.005798806145321578, "clip_ratio/high_mean": 0.0032545167196076363, "clip_ratio/low_mean": 0.005793917312985286, "clip_ratio/low_min": 0.0007022471982054412, "clip_ratio/region_mean": 0.009048434003489092, "entropy": 0.08986990223638713, "epoch": 0.0001, "grad_norm": 1.1754401922225952, "kl": 0.006458986033976544, "learning_rate": 2.5714285714285714e-05, "loss": -0.0398, "step": 10, "step_time": 27.723220527001104 }, { "clip_ratio/high_max": 0.002881916589103639, "clip_ratio/high_mean": 0.0014409582945518196, "clip_ratio/low_mean": 0.0003511235991027206, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017920818936545402, "completions/clipped_ratio": 0.0, "completions/max_length": 9155.0, "completions/max_terminated_length": 9155.0, "completions/mean_length": 7173.3125, "completions/mean_terminated_length": 7173.3125, "completions/min_length": 887.0, "completions/min_terminated_length": 887.0, "entropy": 0.06778168666642159, "epoch": 0.00011, "frac_reward_zero_std": 0.0, "grad_norm": 1.0174713134765625, "kl": 0.004350664900812262, "learning_rate": 2.857142857142857e-05, "loss": -0.2438, "num_tokens": 1653204.0, "reward": -0.4879741668701172, "reward_std": 0.49708276987075806, "rewards/rollout_reward_func/mean": -0.4879741668701172, "rewards/rollout_reward_func/std": 0.5053675770759583, "sampling/importance_sampling_ratio/max": 1.3829487562179565, "sampling/importance_sampling_ratio/mean": 1.0001708269119263, "sampling/importance_sampling_ratio/min": 0.6129343509674072, "sampling/sampling_logp_difference/max": 0.48949742317199707, "sampling/sampling_logp_difference/mean": 0.006062386557459831, "step": 11, "step_time": 88.94538035600272 }, { "clip_ratio/high_max": 0.0021720757358707488, "clip_ratio/high_mean": 0.0010860378679353744, "clip_ratio/low_mean": 0.005371870560338721, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006457908428274095, "entropy": 0.0648728254600428, "epoch": 0.00012, "grad_norm": 0.9108887910842896, "kl": 0.010094417615619022, "learning_rate": 3.142857142857143e-05, "loss": -0.2448, "step": 12, "step_time": 27.213586286001373 }, { "clip_ratio/high_max": 0.009980108763556927, "clip_ratio/high_mean": 0.004990054381778464, "clip_ratio/low_mean": 0.00396484971861355, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008954904100392014, "completions/clipped_ratio": 0.0, "completions/max_length": 9369.0, "completions/max_terminated_length": 9369.0, "completions/mean_length": 8193.15625, "completions/mean_terminated_length": 8193.15625, "completions/min_length": 960.0, "completions/min_terminated_length": 960.0, "entropy": 0.10193604091182351, "epoch": 0.00013, "frac_reward_zero_std": 0.0, "grad_norm": 1.485856294631958, "kl": 0.030039603414479643, "learning_rate": 3.428571428571429e-05, "loss": -0.1495, "num_tokens": 1943132.0, "reward": -0.5804459452629089, "reward_std": 0.27082452178001404, "rewards/rollout_reward_func/mean": -0.5804459452629089, "rewards/rollout_reward_func/std": 0.2952437400817871, "sampling/importance_sampling_ratio/max": 1.951629877090454, "sampling/importance_sampling_ratio/mean": 0.9995810985565186, "sampling/importance_sampling_ratio/min": 0.08306944370269775, "sampling/sampling_logp_difference/max": 2.4880783557891846, "sampling/sampling_logp_difference/mean": 0.016808483749628067, "step": 13, "step_time": 94.51923282000644 }, { "clip_ratio/high_max": 0.015696270624175668, "clip_ratio/high_mean": 0.007848135312087834, "clip_ratio/low_mean": 0.00827725778799504, "clip_ratio/low_min": 0.0021828330936841667, "clip_ratio/region_mean": 0.016125393158290535, "entropy": 0.10211672727018595, "epoch": 0.00014, "grad_norm": 1.529998779296875, "kl": 0.045463139831554145, "learning_rate": 3.7142857142857143e-05, "loss": -0.1549, "step": 14, "step_time": 27.938481579996733 }, { "clip_ratio/high_max": 0.002873942954465747, "clip_ratio/high_mean": 0.0017880951054394245, "clip_ratio/low_mean": 0.0017879052029456943, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003576000308385119, "completions/clipped_ratio": 0.0, "completions/max_length": 9252.0, "completions/max_terminated_length": 9252.0, "completions/mean_length": 7854.34375, "completions/mean_terminated_length": 7854.34375, "completions/min_length": 1084.0, "completions/min_terminated_length": 1084.0, "entropy": 0.064073825487867, "epoch": 0.00015, "frac_reward_zero_std": 0.0, "grad_norm": 2.1420814990997314, "kl": 0.013782255366095342, "learning_rate": 4e-05, "loss": 0.0498, "num_tokens": 2221867.0, "reward": -0.3640046715736389, "reward_std": 0.5706602334976196, "rewards/rollout_reward_func/mean": -0.3640046715736389, "rewards/rollout_reward_func/std": 0.5570085644721985, "sampling/importance_sampling_ratio/max": 2.0640242099761963, "sampling/importance_sampling_ratio/mean": 1.001070499420166, "sampling/importance_sampling_ratio/min": 0.4299520254135132, "sampling/sampling_logp_difference/max": 0.8440816402435303, "sampling/sampling_logp_difference/mean": 0.008871052414178848, "step": 15, "step_time": 95.68331051399582 }, { "clip_ratio/high_max": 0.009986817487515509, "clip_ratio/high_mean": 0.005344532371964306, "clip_ratio/low_mean": 0.005893140041735023, "clip_ratio/low_min": 0.0014044943964108825, "clip_ratio/region_mean": 0.01123767253011465, "entropy": 0.0662008763756603, "epoch": 0.00016, "grad_norm": 1.10405695438385, "kl": 0.022447960553108715, "learning_rate": 4.2857142857142856e-05, "loss": 0.045, "step": 16, "step_time": 27.621052484992106 }, { "clip_ratio/high_max": 0.006080338731408119, "clip_ratio/high_mean": 0.0030401693657040596, "clip_ratio/low_mean": 0.003163026151014492, "clip_ratio/low_min": 0.0007183908019214869, "clip_ratio/region_mean": 0.0062031955167185515, "completions/clipped_ratio": 0.0, "completions/max_length": 9357.0, "completions/max_terminated_length": 9357.0, "completions/mean_length": 7752.09375, "completions/mean_terminated_length": 7752.09375, "completions/min_length": 2165.0, "completions/min_terminated_length": 2165.0, "entropy": 0.07411375921219587, "epoch": 0.00017, "frac_reward_zero_std": 0.0, "grad_norm": 1.4593960046768188, "kl": 0.05461276904679835, "learning_rate": 4.5714285714285716e-05, "loss": 0.0396, "num_tokens": 2497538.0, "reward": -0.2799840569496155, "reward_std": 0.5249596834182739, "rewards/rollout_reward_func/mean": -0.2799840569496155, "rewards/rollout_reward_func/std": 0.5806505680084229, "sampling/importance_sampling_ratio/max": 1.9463616609573364, "sampling/importance_sampling_ratio/mean": 0.9976865649223328, "sampling/importance_sampling_ratio/min": 0.06184462085366249, "sampling/sampling_logp_difference/max": 2.783130168914795, "sampling/sampling_logp_difference/mean": 0.014597892761230469, "step": 17, "step_time": 93.46702274899872 }, { "clip_ratio/high_max": 0.01004306401591748, "clip_ratio/high_mean": 0.00502153200795874, "clip_ratio/low_mean": 0.009563877363689244, "clip_ratio/low_min": 0.001420638000126928, "clip_ratio/region_mean": 0.014585409400751814, "entropy": 0.07483723293989897, "epoch": 0.00018, "grad_norm": 1.2538055181503296, "kl": 0.17985437362222, "learning_rate": 4.8571428571428576e-05, "loss": 0.036, "step": 18, "step_time": 27.61892309099494 }, { "clip_ratio/high_max": 0.007797640864737332, "clip_ratio/high_mean": 0.004960351681802422, "clip_ratio/low_mean": 0.003771967807551846, "clip_ratio/low_min": 0.0013297871919348836, "clip_ratio/region_mean": 0.008732319489354268, "completions/clipped_ratio": 0.0, "completions/max_length": 9323.0, "completions/max_terminated_length": 9323.0, "completions/mean_length": 8057.5625, "completions/mean_terminated_length": 8057.5625, "completions/min_length": 4131.0, "completions/min_terminated_length": 4131.0, "entropy": 0.09318487590644509, "epoch": 0.00019, "frac_reward_zero_std": 0.0, "grad_norm": 1.8037179708480835, "kl": 0.08203080226667225, "learning_rate": 5.142857142857143e-05, "loss": -0.1161, "num_tokens": 2782910.0, "reward": -0.5081856846809387, "reward_std": 0.34735041856765747, "rewards/rollout_reward_func/mean": -0.5081856846809387, "rewards/rollout_reward_func/std": 0.34861209988594055, "sampling/importance_sampling_ratio/max": 2.6022117137908936, "sampling/importance_sampling_ratio/mean": 1.0005253553390503, "sampling/importance_sampling_ratio/min": 0.39166536927223206, "sampling/sampling_logp_difference/max": 0.9563617706298828, "sampling/sampling_logp_difference/mean": 0.016585635021328926, "step": 19, "step_time": 96.19064523599809 }, { "clip_ratio/high_max": 0.009210843476466835, "clip_ratio/high_mean": 0.0056830079411156476, "clip_ratio/low_mean": 0.01183583348756656, "clip_ratio/low_min": 0.005575570685323328, "clip_ratio/region_mean": 0.01751884145778604, "entropy": 0.08642621210310608, "epoch": 0.0002, "grad_norm": 1.6954180002212524, "kl": 0.1374246042687446, "learning_rate": 5.428571428571428e-05, "loss": -0.1201, "step": 20, "step_time": 27.602408401988214 }, { "clip_ratio/high_max": 0.005014301335904747, "clip_ratio/high_mean": 0.002883656619815156, "clip_ratio/low_mean": 0.0028673816996160895, "clip_ratio/low_min": 0.0006944444612599909, "clip_ratio/region_mean": 0.005751038319431245, "completions/clipped_ratio": 0.0, "completions/max_length": 9403.0, "completions/max_terminated_length": 9403.0, "completions/mean_length": 7930.5625, "completions/mean_terminated_length": 7930.5625, "completions/min_length": 634.0, "completions/min_terminated_length": 634.0, "entropy": 0.07444759807549417, "epoch": 0.00021, "frac_reward_zero_std": 0.0, "grad_norm": 1.1838347911834717, "kl": 0.09432762704091147, "learning_rate": 5.714285714285714e-05, "loss": -0.159, "num_tokens": 3064148.0, "reward": -0.3027535676956177, "reward_std": 0.49178069829940796, "rewards/rollout_reward_func/mean": -0.3027535676956177, "rewards/rollout_reward_func/std": 0.5297390818595886, "sampling/importance_sampling_ratio/max": 2.3851823806762695, "sampling/importance_sampling_ratio/mean": 1.0000920295715332, "sampling/importance_sampling_ratio/min": 0.3727108836174011, "sampling/sampling_logp_difference/max": 0.9869523048400879, "sampling/sampling_logp_difference/mean": 0.011310078203678131, "step": 21, "step_time": 91.98894712300535 }, { "clip_ratio/high_max": 0.011702421237714589, "clip_ratio/high_mean": 0.0065865046926774085, "clip_ratio/low_mean": 0.011994066560873762, "clip_ratio/low_min": 0.0024457787512801588, "clip_ratio/region_mean": 0.01858057125355117, "entropy": 0.07004613918252289, "epoch": 0.00022, "grad_norm": 1.0932140350341797, "kl": 0.06876913830637932, "learning_rate": 6e-05, "loss": -0.168, "step": 22, "step_time": 27.827071762007108 }, { "clip_ratio/high_max": 0.007137945271097124, "clip_ratio/high_mean": 0.003936619672458619, "clip_ratio/low_mean": 0.0028658679220825434, "clip_ratio/low_min": 0.0007267441833391786, "clip_ratio/region_mean": 0.006802487594541162, "completions/clipped_ratio": 0.0, "completions/max_length": 9274.0, "completions/max_terminated_length": 9274.0, "completions/mean_length": 8095.0625, "completions/mean_terminated_length": 8095.0625, "completions/min_length": 1533.0, "completions/min_terminated_length": 1533.0, "entropy": 0.0666239457204938, "epoch": 0.00023, "frac_reward_zero_std": 0.0, "grad_norm": 1.2763183116912842, "kl": 0.05845395056530833, "learning_rate": 6.285714285714286e-05, "loss": 0.3004, "num_tokens": 3350379.0, "reward": -0.3713611960411072, "reward_std": 0.44723719358444214, "rewards/rollout_reward_func/mean": -0.3713611960411072, "rewards/rollout_reward_func/std": 0.5041891932487488, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 1.0009047985076904, "sampling/importance_sampling_ratio/min": 0.4158317744731903, "sampling/sampling_logp_difference/max": 1.114884614944458, "sampling/sampling_logp_difference/mean": 0.01109105534851551, "step": 23, "step_time": 93.21978440000748 }, { "clip_ratio/high_max": 0.014727697416674346, "clip_ratio/high_mean": 0.00882161219487898, "clip_ratio/low_mean": 0.01083813124569133, "clip_ratio/low_min": 0.003661759605165571, "clip_ratio/region_mean": 0.01965974346967414, "entropy": 0.06382836727425456, "epoch": 0.00024, "grad_norm": 1.133302092552185, "kl": 0.07930020266212523, "learning_rate": 6.571428571428571e-05, "loss": 0.2957, "step": 24, "step_time": 28.035423179004283 }, { "clip_ratio/high_max": 0.004302181303501129, "clip_ratio/high_mean": 0.0021510906517505646, "clip_ratio/low_mean": 0.0007800387684255838, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0029311294783838093, "completions/clipped_ratio": 0.0, "completions/max_length": 8580.0, "completions/max_terminated_length": 8580.0, "completions/mean_length": 6758.75, "completions/mean_terminated_length": 6758.75, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.04931171424686909, "epoch": 0.00025, "frac_reward_zero_std": 0.0, "grad_norm": 1.291664719581604, "kl": 0.07758830557577312, "learning_rate": 6.857142857142858e-05, "loss": -0.2042, "num_tokens": 3593214.0, "reward": -0.5959107875823975, "reward_std": 0.35876262187957764, "rewards/rollout_reward_func/mean": -0.5959107875823975, "rewards/rollout_reward_func/std": 0.39302268624305725, "sampling/importance_sampling_ratio/max": 2.010491371154785, "sampling/importance_sampling_ratio/mean": 1.0006413459777832, "sampling/importance_sampling_ratio/min": 0.42770469188690186, "sampling/sampling_logp_difference/max": 0.8493223190307617, "sampling/sampling_logp_difference/mean": 0.008722992613911629, "step": 25, "step_time": 86.72631602899492 }, { "clip_ratio/high_max": 0.010786514729261398, "clip_ratio/high_mean": 0.005393257364630699, "clip_ratio/low_mean": 0.011662953183986247, "clip_ratio/low_min": 0.0019696029485203326, "clip_ratio/region_mean": 0.017056210490409285, "entropy": 0.05728732608258724, "epoch": 0.00026, "grad_norm": 2.188659906387329, "kl": 0.08259193820413202, "learning_rate": 7.142857142857143e-05, "loss": -0.2036, "step": 26, "step_time": 26.119606291009404 }, { "clip_ratio/high_max": 0.0061968869413249195, "clip_ratio/high_mean": 0.0030984434706624597, "clip_ratio/low_mean": 0.0010736910917330533, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0041721345914993435, "completions/clipped_ratio": 0.0, "completions/max_length": 8940.0, "completions/max_terminated_length": 8940.0, "completions/mean_length": 7771.3125, "completions/mean_terminated_length": 7771.3125, "completions/min_length": 1515.0, "completions/min_terminated_length": 1515.0, "entropy": 0.0601597810164094, "epoch": 0.00027, "frac_reward_zero_std": 0.0, "grad_norm": 1.141928791999817, "kl": 0.05808806075947359, "learning_rate": 7.428571428571429e-05, "loss": 0.1398, "num_tokens": 3868467.0, "reward": -0.36368632316589355, "reward_std": 0.5357357263565063, "rewards/rollout_reward_func/mean": -0.36368632316589355, "rewards/rollout_reward_func/std": 0.5756723880767822, "sampling/importance_sampling_ratio/max": 1.8575561046600342, "sampling/importance_sampling_ratio/mean": 1.0000970363616943, "sampling/importance_sampling_ratio/min": 0.3134206235408783, "sampling/sampling_logp_difference/max": 1.1602091789245605, "sampling/sampling_logp_difference/mean": 0.009103494696319103, "step": 27, "step_time": 92.96613138498651 }, { "clip_ratio/high_max": 0.0164202400483191, "clip_ratio/high_mean": 0.00821012002415955, "clip_ratio/low_mean": 0.013974322064314038, "clip_ratio/low_min": 0.0007183908019214869, "clip_ratio/region_mean": 0.022184442059369758, "entropy": 0.0649251660797745, "epoch": 0.00028, "grad_norm": 0.8147655725479126, "kl": 0.10712331312242895, "learning_rate": 7.714285714285715e-05, "loss": 0.1369, "step": 28, "step_time": 27.32312696599911 }, { "clip_ratio/high_max": 0.002155932132154703, "clip_ratio/high_mean": 0.0010779660660773516, "clip_ratio/low_mean": 0.0007102272938936949, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0017881933599710464, "completions/clipped_ratio": 0.0, "completions/max_length": 9165.0, "completions/max_terminated_length": 9165.0, "completions/mean_length": 7336.8125, "completions/mean_terminated_length": 7336.8125, "completions/min_length": 1667.0, "completions/min_terminated_length": 1667.0, "entropy": 0.06351292878389359, "epoch": 0.00029, "frac_reward_zero_std": 0.0, "grad_norm": 0.7936819195747375, "kl": 0.04702634894056246, "learning_rate": 8e-05, "loss": -0.2535, "num_tokens": 4129721.0, "reward": -0.4376564919948578, "reward_std": 0.39146924018859863, "rewards/rollout_reward_func/mean": -0.4376564919948578, "rewards/rollout_reward_func/std": 0.4162859320640564, "sampling/importance_sampling_ratio/max": 1.9884843826293945, "sampling/importance_sampling_ratio/mean": 1.0006194114685059, "sampling/importance_sampling_ratio/min": 0.5153306126594543, "sampling/sampling_logp_difference/max": 0.6873726844787598, "sampling/sampling_logp_difference/mean": 0.00740803312510252, "step": 29, "step_time": 91.82200075300352 }, { "clip_ratio/high_max": 0.005039906303863972, "clip_ratio/high_mean": 0.00323018038761802, "clip_ratio/low_mean": 0.010294274339685217, "clip_ratio/low_min": 0.004024621332064271, "clip_ratio/region_mean": 0.013524454698199406, "entropy": 0.06765187811106443, "epoch": 0.0003, "grad_norm": 0.7302473783493042, "kl": 0.050249568943399936, "learning_rate": 8.285714285714287e-05, "loss": -0.2607, "step": 30, "step_time": 28.11271668801055 }, { "clip_ratio/high_max": 0.00493967067450285, "clip_ratio/high_mean": 0.002469835337251425, "clip_ratio/low_mean": 0.0007143090479075909, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0031841443851590157, "completions/clipped_ratio": 0.0, "completions/max_length": 9146.0, "completions/max_terminated_length": 9146.0, "completions/mean_length": 7793.5625, "completions/mean_terminated_length": 7793.5625, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.08614894514903426, "epoch": 0.00031, "frac_reward_zero_std": 0.0, "grad_norm": 1.0549495220184326, "kl": 0.06513052270747721, "learning_rate": 8.571428571428571e-05, "loss": -0.0626, "num_tokens": 4405861.0, "reward": -0.37940478324890137, "reward_std": 0.45117825269699097, "rewards/rollout_reward_func/mean": -0.37940478324890137, "rewards/rollout_reward_func/std": 0.48283621668815613, "sampling/importance_sampling_ratio/max": 1.4535597562789917, "sampling/importance_sampling_ratio/mean": 0.9976645112037659, "sampling/importance_sampling_ratio/min": 0.5075654983520508, "sampling/sampling_logp_difference/max": 0.6781294345855713, "sampling/sampling_logp_difference/mean": 0.009683198295533657, "step": 31, "step_time": 92.44875282800785 }, { "clip_ratio/high_max": 0.011355375638231635, "clip_ratio/high_mean": 0.006735137314535677, "clip_ratio/low_mean": 0.012324601178988814, "clip_ratio/low_min": 0.0007022471982054412, "clip_ratio/region_mean": 0.019059738871874288, "entropy": 0.08400337002240121, "epoch": 0.00032, "grad_norm": 0.6007691621780396, "kl": 0.08906090061645955, "learning_rate": 8.857142857142857e-05, "loss": -0.0698, "step": 32, "step_time": 28.252189476006606 }, { "clip_ratio/high_max": 0.007888395455665886, "clip_ratio/high_mean": 0.003944197727832943, "clip_ratio/low_mean": 0.004384168569231406, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00832836632616818, "completions/clipped_ratio": 0.0, "completions/max_length": 9394.0, "completions/max_terminated_length": 9394.0, "completions/mean_length": 7661.125, "completions/mean_terminated_length": 7661.125, "completions/min_length": 1012.0, "completions/min_terminated_length": 1012.0, "entropy": 0.08313656039535999, "epoch": 0.00033, "frac_reward_zero_std": 0.0, "grad_norm": 1.035186767578125, "kl": 0.09069790283683687, "learning_rate": 9.142857142857143e-05, "loss": 0.0166, "num_tokens": 4677397.0, "reward": -0.3240296244621277, "reward_std": 0.4347441494464874, "rewards/rollout_reward_func/mean": -0.3240296244621277, "rewards/rollout_reward_func/std": 0.4854819178581238, "sampling/importance_sampling_ratio/max": 1.7231584787368774, "sampling/importance_sampling_ratio/mean": 0.9987027645111084, "sampling/importance_sampling_ratio/min": 0.06808681786060333, "sampling/sampling_logp_difference/max": 2.686971664428711, "sampling/sampling_logp_difference/mean": 0.010714043863117695, "step": 33, "step_time": 93.2897496730111 }, { "clip_ratio/high_max": 0.020736676699016243, "clip_ratio/high_mean": 0.01142766050179489, "clip_ratio/low_mean": 0.010279498674208298, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02170715932152234, "entropy": 0.07817789679393172, "epoch": 0.00034, "grad_norm": 0.6795229911804199, "kl": 0.110447074577678, "learning_rate": 9.428571428571429e-05, "loss": 0.0053, "step": 34, "step_time": 28.798946735994832 }, { "clip_ratio/high_max": 0.004271204466931522, "clip_ratio/high_mean": 0.002135602233465761, "clip_ratio/low_mean": 0.0010695144010242075, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032051166344899684, "completions/clipped_ratio": 0.0, "completions/max_length": 9245.0, "completions/max_terminated_length": 9245.0, "completions/mean_length": 8244.75, "completions/mean_terminated_length": 8244.75, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.06729014380834997, "epoch": 0.00035, "frac_reward_zero_std": 0.0, "grad_norm": 1.4786404371261597, "kl": 0.10362116701435298, "learning_rate": 9.714285714285715e-05, "loss": -0.2709, "num_tokens": 4968197.0, "reward": -0.3916197419166565, "reward_std": 0.35996532440185547, "rewards/rollout_reward_func/mean": -0.3916197419166565, "rewards/rollout_reward_func/std": 0.3666594326496124, "sampling/importance_sampling_ratio/max": 1.5868641138076782, "sampling/importance_sampling_ratio/mean": 1.0002307891845703, "sampling/importance_sampling_ratio/min": 0.6188454031944275, "sampling/sampling_logp_difference/max": 0.4798997640609741, "sampling/sampling_logp_difference/mean": 0.007114737294614315, "step": 35, "step_time": 92.1033384639959 }, { "clip_ratio/high_max": 0.017084260645788163, "clip_ratio/high_mean": 0.009252357733203098, "clip_ratio/low_mean": 0.012271191226318479, "clip_ratio/low_min": 0.0014044943964108825, "clip_ratio/region_mean": 0.021523548872210085, "entropy": 0.06766591034829617, "epoch": 0.00036, "grad_norm": 0.7554082870483398, "kl": 0.11198568157851696, "learning_rate": 0.0001, "loss": -0.2783, "step": 36, "step_time": 28.797730328002217 }, { "clip_ratio/high_max": 0.0014046717551536858, "clip_ratio/high_mean": 0.0007023358775768429, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0007023358775768429, "completions/clipped_ratio": 0.0, "completions/max_length": 9282.0, "completions/max_terminated_length": 9282.0, "completions/mean_length": 7701.8125, "completions/mean_terminated_length": 7701.8125, "completions/min_length": 1350.0, "completions/min_terminated_length": 1350.0, "entropy": 0.06278971070423722, "epoch": 0.00037, "frac_reward_zero_std": 0.0, "grad_norm": 1.0583816766738892, "kl": 0.12011543358676136, "learning_rate": 9.918053502751771e-05, "loss": -0.0297, "num_tokens": 5241387.0, "reward": -0.304426908493042, "reward_std": 0.4464525580406189, "rewards/rollout_reward_func/mean": -0.304426908493042, "rewards/rollout_reward_func/std": 0.4538337290287018, "sampling/importance_sampling_ratio/max": 1.6482372283935547, "sampling/importance_sampling_ratio/mean": 0.9995368719100952, "sampling/importance_sampling_ratio/min": 0.5574944615364075, "sampling/sampling_logp_difference/max": 0.5843026638031006, "sampling/sampling_logp_difference/mean": 0.007077127229422331, "step": 37, "step_time": 91.12368497201533 }, { "clip_ratio/high_max": 0.010138559562619776, "clip_ratio/high_mean": 0.006660482700681314, "clip_ratio/low_mean": 0.01362931149196811, "clip_ratio/low_min": 0.003907083009835333, "clip_ratio/region_mean": 0.020289794192649424, "entropy": 0.05679044884163886, "epoch": 0.00038, "grad_norm": 0.618403971195221, "kl": 0.118131990195252, "learning_rate": 9.675795466159755e-05, "loss": -0.0362, "step": 38, "step_time": 28.32057495099798 }, { "clip_ratio/high_max": 0.005428613570984453, "clip_ratio/high_mean": 0.0027143067854922265, "clip_ratio/low_mean": 0.0010573608451522887, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003771667630644515, "completions/clipped_ratio": 0.0, "completions/max_length": 9203.0, "completions/max_terminated_length": 9203.0, "completions/mean_length": 7459.09375, "completions/mean_terminated_length": 7459.09375, "completions/min_length": 647.0, "completions/min_terminated_length": 647.0, "entropy": 0.05742924939841032, "epoch": 0.00039, "frac_reward_zero_std": 0.0, "grad_norm": 1.1807667016983032, "kl": 0.11824507999699563, "learning_rate": 9.283813728906054e-05, "loss": -0.0685, "num_tokens": 5507267.0, "reward": -0.1265014261007309, "reward_std": 0.6104403734207153, "rewards/rollout_reward_func/mean": -0.1265014261007309, "rewards/rollout_reward_func/std": 0.6330578327178955, "sampling/importance_sampling_ratio/max": 1.6747703552246094, "sampling/importance_sampling_ratio/mean": 0.9992620944976807, "sampling/importance_sampling_ratio/min": 0.39614981412887573, "sampling/sampling_logp_difference/max": 0.9259628653526306, "sampling/sampling_logp_difference/mean": 0.006985975429415703, "step": 39, "step_time": 91.56473480099157 }, { "clip_ratio/high_max": 0.017474214837420732, "clip_ratio/high_mean": 0.008737107418710366, "clip_ratio/low_mean": 0.01038203525240533, "clip_ratio/low_min": 0.0021973324473947287, "clip_ratio/region_mean": 0.01911914284573868, "entropy": 0.05913685623090714, "epoch": 0.0004, "grad_norm": 11257.0869140625, "kl": 15902.953878552886, "learning_rate": 8.759239773845718e-05, "loss": 59.5637, "step": 40, "step_time": 28.143666776995815 }, { "clip_ratio/high_max": 0.0014367816038429737, "clip_ratio/high_mean": 0.00109489681199193, "clip_ratio/low_mean": 0.002375774463871494, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003470671275863424, "completions/clipped_ratio": 0.0, "completions/max_length": 9299.0, "completions/max_terminated_length": 9299.0, "completions/mean_length": 7399.59375, "completions/mean_terminated_length": 7399.59375, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.0707793568726629, "epoch": 0.00041, "frac_reward_zero_std": 0.0, "grad_norm": 0.779978334903717, "kl": 0.1064646802842617, "learning_rate": 8.125000000000001e-05, "loss": -0.006, "num_tokens": 5770826.0, "reward": -0.10253402590751648, "reward_std": 0.587022602558136, "rewards/rollout_reward_func/mean": -0.10253402590751648, "rewards/rollout_reward_func/std": 0.58503657579422, "sampling/importance_sampling_ratio/max": 1.7891288995742798, "sampling/importance_sampling_ratio/mean": 1.000353455543518, "sampling/importance_sampling_ratio/min": 0.33879733085632324, "sampling/sampling_logp_difference/max": 1.082353115081787, "sampling/sampling_logp_difference/mean": 0.008459154516458511, "step": 41, "step_time": 89.9564655849972 }, { "clip_ratio/high_max": 0.013633344613481313, "clip_ratio/high_mean": 0.0071758677077014, "clip_ratio/low_mean": 0.013401742442511022, "clip_ratio/low_min": 0.0028420357266440988, "clip_ratio/region_mean": 0.020577610324835405, "entropy": 0.07488202536478639, "epoch": 0.00042, "grad_norm": 0.6732333302497864, "kl": 0.11995455622673035, "learning_rate": 7.408813728906053e-05, "loss": -0.0139, "step": 42, "step_time": 27.865521725998406 }, { "clip_ratio/high_max": 0.001420638000126928, "clip_ratio/high_mean": 0.000710319000063464, "clip_ratio/low_mean": 0.002360046637477353, "clip_ratio/low_min": 0.0007022471982054412, "clip_ratio/region_mean": 0.0030703656375408173, "completions/clipped_ratio": 0.0, "completions/max_length": 8920.0, "completions/max_terminated_length": 8920.0, "completions/mean_length": 7293.5, "completions/mean_terminated_length": 7293.5, "completions/min_length": 876.0, "completions/min_terminated_length": 876.0, "entropy": 0.06767579750157893, "epoch": 0.00043, "frac_reward_zero_std": 0.0, "grad_norm": 0.9846507906913757, "kl": 0.11756309028714895, "learning_rate": 6.6419817372537e-05, "loss": -0.105, "num_tokens": 6030904.0, "reward": -0.17516297101974487, "reward_std": 0.6336148977279663, "rewards/rollout_reward_func/mean": -0.17516297101974487, "rewards/rollout_reward_func/std": 0.6060534715652466, "sampling/importance_sampling_ratio/max": 1.5995110273361206, "sampling/importance_sampling_ratio/mean": 1.0016493797302246, "sampling/importance_sampling_ratio/min": 0.6211615800857544, "sampling/sampling_logp_difference/max": 0.4761640429496765, "sampling/sampling_logp_difference/mean": 0.00795302540063858, "step": 43, "step_time": 88.68364224100515 }, { "clip_ratio/high_max": 0.011437574110459536, "clip_ratio/high_mean": 0.006488003156846389, "clip_ratio/low_mean": 0.011332564434269443, "clip_ratio/low_min": 0.002825683041010052, "clip_ratio/region_mean": 0.01782056744559668, "entropy": 0.06705769780091941, "epoch": 0.00044, "grad_norm": 0.5778977274894714, "kl": 0.13579032279085368, "learning_rate": 5.858018262746301e-05, "loss": -0.1148, "step": 44, "step_time": 26.98718312199344 }, { "clip_ratio/high_max": 0.0022492663701996207, "clip_ratio/high_mean": 0.0011246331850998104, "clip_ratio/low_mean": 0.0028377448907122016, "clip_ratio/low_min": 0.0006944444612599909, "clip_ratio/region_mean": 0.003962378075812012, "completions/clipped_ratio": 0.0, "completions/max_length": 9242.0, "completions/max_terminated_length": 9242.0, "completions/mean_length": 8146.5, "completions/mean_terminated_length": 8146.5, "completions/min_length": 1426.0, "completions/min_terminated_length": 1426.0, "entropy": 0.08103846106678247, "epoch": 0.00045, "frac_reward_zero_std": 0.0, "grad_norm": 1.409741759300232, "kl": 0.14359550992958248, "learning_rate": 5.0911862710939485e-05, "loss": -0.034, "num_tokens": 6318667.0, "reward": -0.12613427639007568, "reward_std": 0.5343547463417053, "rewards/rollout_reward_func/mean": -0.12613427639007568, "rewards/rollout_reward_func/std": 0.5416647791862488, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 1.000476598739624, "sampling/importance_sampling_ratio/min": 0.09892720729112625, "sampling/sampling_logp_difference/max": 2.313370943069458, "sampling/sampling_logp_difference/mean": 0.012712076306343079, "step": 45, "step_time": 93.79370664700036 }, { "clip_ratio/high_max": 0.011134565633255988, "clip_ratio/high_mean": 0.006269618694204837, "clip_ratio/low_mean": 0.009181194531265646, "clip_ratio/low_min": 0.0028904599603265524, "clip_ratio/region_mean": 0.015450813429197297, "entropy": 0.08389568043639883, "epoch": 0.00046, "grad_norm": 1.2059129476547241, "kl": 0.24020475335419178, "learning_rate": 4.375000000000001e-05, "loss": -0.0432, "step": 46, "step_time": 28.168583785994997 }, { "clip_ratio/high_max": 0.005004245089367032, "clip_ratio/high_mean": 0.002502122544683516, "clip_ratio/low_mean": 0.002046188630629331, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0045483111462090164, "completions/clipped_ratio": 0.0, "completions/max_length": 9056.0, "completions/max_terminated_length": 9056.0, "completions/mean_length": 7947.84375, "completions/mean_terminated_length": 7947.84375, "completions/min_length": 4376.0, "completions/min_terminated_length": 4376.0, "entropy": 0.07940921117551625, "epoch": 0.00047, "frac_reward_zero_std": 0.0, "grad_norm": 1.2504568099975586, "kl": 0.16516069741919637, "learning_rate": 3.740760226154283e-05, "loss": -0.0286, "num_tokens": 6600118.0, "reward": -0.12103764712810516, "reward_std": 0.5211879014968872, "rewards/rollout_reward_func/mean": -0.12103764712810516, "rewards/rollout_reward_func/std": 0.5456745028495789, "sampling/importance_sampling_ratio/max": 1.9942717552185059, "sampling/importance_sampling_ratio/mean": 0.9991114139556885, "sampling/importance_sampling_ratio/min": 0.11994559317827225, "sampling/sampling_logp_difference/max": 2.1207170486450195, "sampling/sampling_logp_difference/mean": 0.010312505066394806, "step": 47, "step_time": 94.76915054599158 }, { "clip_ratio/high_max": 0.00876080454327166, "clip_ratio/high_mean": 0.00438040227163583, "clip_ratio/low_mean": 0.002355890115723014, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006736292387358844, "entropy": 0.08134529506787658, "epoch": 0.00048, "grad_norm": 0.805917501449585, "kl": 0.16713016014546156, "learning_rate": 3.2161862710939476e-05, "loss": -0.0331, "step": 48, "step_time": 27.11353817199779 }, { "clip_ratio/high_max": 0.004328028007876128, "clip_ratio/high_mean": 0.002164014003938064, "clip_ratio/low_mean": 0.0010534594766795635, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003217473509721458, "completions/clipped_ratio": 0.0, "completions/max_length": 9558.0, "completions/max_terminated_length": 9558.0, "completions/mean_length": 8071.4375, "completions/mean_terminated_length": 8071.4375, "completions/min_length": 3757.0, "completions/min_terminated_length": 3757.0, "entropy": 0.09437589161098003, "epoch": 0.00049, "frac_reward_zero_std": 0.0, "grad_norm": 1.223923921585083, "kl": 0.17360988864675164, "learning_rate": 2.8242045338402468e-05, "loss": 0.0868, "num_tokens": 6885078.0, "reward": -0.17595598101615906, "reward_std": 0.574884295463562, "rewards/rollout_reward_func/mean": -0.17595598101615906, "rewards/rollout_reward_func/std": 0.617475152015686, "sampling/importance_sampling_ratio/max": 2.3295459747314453, "sampling/importance_sampling_ratio/mean": 0.9994887709617615, "sampling/importance_sampling_ratio/min": 0.21804022789001465, "sampling/sampling_logp_difference/max": 1.5230756998062134, "sampling/sampling_logp_difference/mean": 0.012081097811460495, "step": 49, "step_time": 92.44432610799413 }, { "clip_ratio/high_max": 0.009452320169657469, "clip_ratio/high_mean": 0.005073382286354899, "clip_ratio/low_mean": 0.00776787509676069, "clip_ratio/low_min": 0.002785580581985414, "clip_ratio/region_mean": 0.012841257266700268, "entropy": 0.09734066273085773, "epoch": 0.0005, "grad_norm": 1.335457682609558, "kl": 0.17287601297721267, "learning_rate": 2.5819464972482288e-05, "loss": 0.0771, "step": 50, "step_time": 28.072686168001383 } ], "logging_steps": 1.0, "max_steps": 50, "num_input_tokens_seen": 6885078, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }