{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.00034, "eval_steps": 500, "global_step": 34, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 74.0, "completions/max_terminated_length": 74.0, "completions/mean_length": 33.125, "completions/mean_terminated_length": 18.875, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.3827928453683853, "epoch": 1e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.3181266486644745, "kl": 0.0, "learning_rate": 0.0, "loss": -0.0678, "num_tokens": 16641.0, "reward": -0.06401677429676056, "reward_std": 0.7941610813140869, "rewards/rollout_reward_func/mean": -0.06401677429676056, "rewards/rollout_reward_func/std": 0.7941611409187317, "sampling/importance_sampling_ratio/max": 1.144827127456665, "sampling/importance_sampling_ratio/mean": 0.551578164100647, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.74828338623047, "sampling/sampling_logp_difference/mean": 1.3979711532592773, "step": 1, "step_time": 7.639951573002691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 63.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 22.5625, "completions/mean_terminated_length": 17.38461685180664, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.5964446440339088, "epoch": 2e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.34730541706085205, "kl": 0.0, "learning_rate": 1e-06, "loss": 0.0602, "num_tokens": 33557.0, "reward": 0.04738499969244003, "reward_std": 0.81049644947052, "rewards/rollout_reward_func/mean": 0.04738499969244003, "rewards/rollout_reward_func/std": 0.8104965090751648, "sampling/importance_sampling_ratio/max": 1.371529221534729, "sampling/importance_sampling_ratio/mean": 0.531771719455719, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.6956672668457, "sampling/sampling_logp_difference/mean": 2.163590908050537, "step": 2, "step_time": 6.844142468999053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 87.0, "completions/max_terminated_length": 56.0, "completions/mean_length": 34.6875, "completions/mean_terminated_length": 24.100000381469727, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.5396408289670944, "epoch": 3e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.40354248881340027, "kl": 0.0019364068793947808, "learning_rate": 2e-06, "loss": 0.0919, "num_tokens": 51093.0, "reward": 0.3400000035762787, "reward_std": 0.8063746094703674, "rewards/rollout_reward_func/mean": 0.3400000035762787, "rewards/rollout_reward_func/std": 0.8063746094703674, "sampling/importance_sampling_ratio/max": 1.2352397441864014, "sampling/importance_sampling_ratio/mean": 0.4642365574836731, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.2575569152832, "sampling/sampling_logp_difference/mean": 1.7650401592254639, "step": 3, "step_time": 8.551386018996709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 64.0, "completions/max_terminated_length": 62.0, "completions/mean_length": 32.125, "completions/mean_terminated_length": 24.166667938232422, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.3548607230186462, "epoch": 4e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.26631447672843933, "kl": 0.001303125623962842, "learning_rate": 3e-06, "loss": 0.052, "num_tokens": 67893.0, "reward": 0.11769548058509827, "reward_std": 0.8427314162254333, "rewards/rollout_reward_func/mean": 0.11769548058509827, "rewards/rollout_reward_func/std": 0.8427314758300781, "sampling/importance_sampling_ratio/max": 1.1722887754440308, "sampling/importance_sampling_ratio/mean": 0.4403129518032074, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.54515838623047, "sampling/sampling_logp_difference/mean": 1.7197456359863281, "step": 4, "step_time": 7.270715081998787 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 64.0, "completions/max_terminated_length": 37.0, "completions/mean_length": 26.6875, "completions/mean_terminated_length": 20.666667938232422, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2926968112587929, "epoch": 5e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.34337735176086426, "kl": 0.000681035679008346, "learning_rate": 4e-06, "loss": 0.035, "num_tokens": 84202.0, "reward": 0.03295581787824631, "reward_std": 0.8984397053718567, "rewards/rollout_reward_func/mean": 0.03295581787824631, "rewards/rollout_reward_func/std": 0.8984397649765015, "sampling/importance_sampling_ratio/max": 1.4395053386688232, "sampling/importance_sampling_ratio/mean": 0.5380887985229492, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.7639045715332, "sampling/sampling_logp_difference/mean": 1.5982863903045654, "step": 5, "step_time": 6.337428640003054 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 86.0, "completions/max_terminated_length": 86.0, "completions/mean_length": 32.6875, "completions/mean_terminated_length": 20.30000114440918, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.155207872390747, "epoch": 6e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.3781796097755432, "kl": 0.0006958278354431968, "learning_rate": 4.9999999999999996e-06, "loss": -0.0204, "num_tokens": 101026.0, "reward": -0.10519320517778397, "reward_std": 0.8153080344200134, "rewards/rollout_reward_func/mean": -0.10519320517778397, "rewards/rollout_reward_func/std": 0.8153082132339478, "sampling/importance_sampling_ratio/max": 1.2898168563842773, "sampling/importance_sampling_ratio/mean": 0.6265079379081726, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.706966400146484, "sampling/sampling_logp_difference/mean": 1.0843193531036377, "step": 6, "step_time": 7.546433894000074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 64.0, "completions/max_terminated_length": 32.0, "completions/mean_length": 26.1875, "completions/mean_terminated_length": 19.636363983154297, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.4203693270683289, "epoch": 7e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.4280190169811249, "kl": 0.0019115788309136406, "learning_rate": 6e-06, "loss": 0.0876, "num_tokens": 118090.0, "reward": 0.23219826817512512, "reward_std": 0.8312895894050598, "rewards/rollout_reward_func/mean": 0.23219826817512512, "rewards/rollout_reward_func/std": 0.8312896490097046, "sampling/importance_sampling_ratio/max": 1.3674085140228271, "sampling/importance_sampling_ratio/mean": 0.5275859236717224, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.18802261352539, "sampling/sampling_logp_difference/mean": 1.7722361087799072, "step": 7, "step_time": 6.381835996000518 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 64.0, "completions/max_terminated_length": 49.0, "completions/mean_length": 25.125, "completions/mean_terminated_length": 17.25, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2874730825424194, "epoch": 8e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.3058971166610718, "kl": 0.0010104302782565355, "learning_rate": 7e-06, "loss": 0.0967, "num_tokens": 134546.0, "reward": -0.08250001072883606, "reward_std": 0.8818729519844055, "rewards/rollout_reward_func/mean": -0.08250001072883606, "rewards/rollout_reward_func/std": 0.8818729519844055, "sampling/importance_sampling_ratio/max": 1.2262499332427979, "sampling/importance_sampling_ratio/mean": 0.6900386214256287, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.81074523925781, "sampling/sampling_logp_difference/mean": 1.485697627067566, "step": 8, "step_time": 7.5347441070043715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 64.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 21.9375, "completions/mean_terminated_length": 15.166666984558105, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.3974642157554626, "epoch": 9e-05, "frac_reward_zero_std": 0.0, "grad_norm": 0.29421234130859375, "kl": 0.0017157641450467054, "learning_rate": 8e-06, "loss": 0.0548, "num_tokens": 150906.0, "reward": 0.0730893611907959, "reward_std": 0.872587263584137, "rewards/rollout_reward_func/mean": 0.0730893611907959, "rewards/rollout_reward_func/std": 0.872587263584137, "sampling/importance_sampling_ratio/max": 1.183666467666626, "sampling/importance_sampling_ratio/mean": 0.511037290096283, "sampling/importance_sampling_ratio/min": 2.324053503082709e-41, "sampling/sampling_logp_difference/max": 34.89555358886719, "sampling/sampling_logp_difference/mean": 1.7038354873657227, "step": 9, "step_time": 6.773095810000086 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 57.0, "completions/max_terminated_length": 57.0, "completions/mean_length": 29.75, "completions/mean_terminated_length": 25.600000381469727, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.5960686951875687, "epoch": 0.0001, "frac_reward_zero_std": 0.0, "grad_norm": 0.3341529667377472, "kl": 0.0010894886290770955, "learning_rate": 9e-06, "loss": 0.0245, "num_tokens": 167587.0, "reward": 0.5254915952682495, "reward_std": 0.7537170052528381, "rewards/rollout_reward_func/mean": 0.5254915952682495, "rewards/rollout_reward_func/std": 0.7537170648574829, "sampling/importance_sampling_ratio/max": 1.0177525281906128, "sampling/importance_sampling_ratio/mean": 0.4141247272491455, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.73265838623047, "sampling/sampling_logp_difference/mean": 2.074547052383423, "step": 10, "step_time": 6.915056184001514 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 49.0, "completions/max_terminated_length": 46.0, "completions/mean_length": 22.125, "completions/mean_terminated_length": 18.076923370361328, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.4635429829359055, "epoch": 0.00011, "frac_reward_zero_std": 0.0, "grad_norm": 0.35319551825523376, "kl": 0.0016797076386865228, "learning_rate": 9.999999999999999e-06, "loss": 0.0826, "num_tokens": 183363.0, "reward": 0.005195483565330505, "reward_std": 0.837223470211029, "rewards/rollout_reward_func/mean": 0.005195483565330505, "rewards/rollout_reward_func/std": 0.8372235298156738, "sampling/importance_sampling_ratio/max": 1.2394757270812988, "sampling/importance_sampling_ratio/mean": 0.5569560527801514, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 36.220245361328125, "sampling/sampling_logp_difference/mean": 2.1369495391845703, "step": 11, "step_time": 5.859747936994609 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 96.0, "completions/max_terminated_length": 60.0, "completions/mean_length": 32.6875, "completions/mean_terminated_length": 26.923078536987305, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.6477473005652428, "epoch": 0.00012, "frac_reward_zero_std": 0.0, "grad_norm": 0.3246020972728729, "kl": 0.003157808539981488, "learning_rate": 1.1e-05, "loss": 0.0995, "num_tokens": 200806.0, "reward": -0.0572887659072876, "reward_std": 0.7941725850105286, "rewards/rollout_reward_func/mean": -0.0572887659072876, "rewards/rollout_reward_func/std": 0.7941725850105286, "sampling/importance_sampling_ratio/max": 1.1799664497375488, "sampling/importance_sampling_ratio/mean": 0.39019522070884705, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.21113204956055, "sampling/sampling_logp_difference/mean": 1.9028511047363281, "step": 12, "step_time": 7.54413384599502 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 72.0, "completions/max_terminated_length": 53.0, "completions/mean_length": 26.6875, "completions/mean_terminated_length": 19.923078536987305, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.274285078048706, "epoch": 0.00013, "frac_reward_zero_std": 0.0, "grad_norm": 0.32733649015426636, "kl": 0.001991469122003764, "learning_rate": 1.2e-05, "loss": -0.0647, "num_tokens": 217199.0, "reward": 0.31669238209724426, "reward_std": 0.7471501231193542, "rewards/rollout_reward_func/mean": 0.31669238209724426, "rewards/rollout_reward_func/std": 0.747150182723999, "sampling/importance_sampling_ratio/max": 1.236131191253662, "sampling/importance_sampling_ratio/mean": 0.6276021599769592, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 32.89448928833008, "sampling/sampling_logp_difference/mean": 1.8165452480316162, "step": 13, "step_time": 7.0487566130013874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 92.0, "completions/max_terminated_length": 92.0, "completions/mean_length": 27.25, "completions/mean_terminated_length": 21.83333396911621, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0698126628994942, "epoch": 0.00014, "frac_reward_zero_std": 0.0, "grad_norm": 0.3421112895011902, "kl": 0.005695157662557904, "learning_rate": 1.3000000000000001e-05, "loss": 0.0923, "num_tokens": 233569.0, "reward": 0.1585809737443924, "reward_std": 0.8103994727134705, "rewards/rollout_reward_func/mean": 0.1585809737443924, "rewards/rollout_reward_func/std": 0.8103994727134705, "sampling/importance_sampling_ratio/max": 1.1087441444396973, "sampling/importance_sampling_ratio/mean": 0.6734969615936279, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.04454803466797, "sampling/sampling_logp_difference/mean": 0.9640688300132751, "step": 14, "step_time": 7.728960284001005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 64.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 26.875, "completions/mean_terminated_length": 20.76923179626465, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.4536470770835876, "epoch": 0.00015, "frac_reward_zero_std": 0.0, "grad_norm": 0.402100533246994, "kl": 0.010122014762600884, "learning_rate": 1.4e-05, "loss": 0.0717, "num_tokens": 250395.0, "reward": 0.32499998807907104, "reward_std": 0.8999999761581421, "rewards/rollout_reward_func/mean": 0.32499998807907104, "rewards/rollout_reward_func/std": 0.9000000357627869, "sampling/importance_sampling_ratio/max": 1.1882612705230713, "sampling/importance_sampling_ratio/mean": 0.43665286898612976, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.10694122314453, "sampling/sampling_logp_difference/mean": 1.8059306144714355, "step": 15, "step_time": 6.667633953000404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 33.4375, "completions/mean_terminated_length": 24.22222328186035, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.5715003088116646, "epoch": 0.00016, "frac_reward_zero_std": 0.0, "grad_norm": 0.278489887714386, "kl": 0.010550641687586904, "learning_rate": 1.5e-05, "loss": -0.0207, "num_tokens": 266674.0, "reward": 0.34981411695480347, "reward_std": 0.8006922602653503, "rewards/rollout_reward_func/mean": 0.34981411695480347, "rewards/rollout_reward_func/std": 0.8006923198699951, "sampling/importance_sampling_ratio/max": 1.1930525302886963, "sampling/importance_sampling_ratio/mean": 0.37038883566856384, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 35.258827209472656, "sampling/sampling_logp_difference/mean": 2.096559524536133, "step": 16, "step_time": 7.509245967999959 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 70.0, "completions/max_terminated_length": 70.0, "completions/mean_length": 31.4375, "completions/mean_terminated_length": 23.88888931274414, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.1031887084245682, "epoch": 0.00017, "frac_reward_zero_std": 0.0, "grad_norm": 0.5009875893592834, "kl": 0.0252107827400323, "learning_rate": 1.4999946454054132e-05, "loss": -0.141, "num_tokens": 283142.0, "reward": -0.264902263879776, "reward_std": 0.7876402139663696, "rewards/rollout_reward_func/mean": -0.264902263879776, "rewards/rollout_reward_func/std": 0.7876402139663696, "sampling/importance_sampling_ratio/max": 1.2886087894439697, "sampling/importance_sampling_ratio/mean": 0.6680909395217896, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.088829040527344, "sampling/sampling_logp_difference/mean": 1.021271824836731, "step": 17, "step_time": 6.736764096998741 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 96.0, "completions/max_terminated_length": 53.0, "completions/mean_length": 40.0625, "completions/mean_terminated_length": 31.80000114440918, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2849810495972633, "epoch": 0.00018, "frac_reward_zero_std": 0.0, "grad_norm": 0.41720253229141235, "kl": 0.02835782650799956, "learning_rate": 1.4999785817235962e-05, "loss": -0.0753, "num_tokens": 300104.0, "reward": 0.27464818954467773, "reward_std": 0.7914400100708008, "rewards/rollout_reward_func/mean": 0.27464818954467773, "rewards/rollout_reward_func/std": 0.7914400696754456, "sampling/importance_sampling_ratio/max": 1.4827672243118286, "sampling/importance_sampling_ratio/mean": 0.40857595205307007, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.11698532104492, "sampling/sampling_logp_difference/mean": 1.3236627578735352, "step": 18, "step_time": 8.64123790499616 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 63.0, "completions/max_terminated_length": 63.0, "completions/mean_length": 29.625, "completions/mean_terminated_length": 24.545455932617188, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.4260313510894775, "epoch": 0.00019, "frac_reward_zero_std": 0.0, "grad_norm": 0.1461358666419983, "kl": 0.009397892834385857, "learning_rate": 1.499951809260379e-05, "loss": 0.0238, "num_tokens": 317197.0, "reward": 0.23086139559745789, "reward_std": 0.8318407535552979, "rewards/rollout_reward_func/mean": 0.23086139559745789, "rewards/rollout_reward_func/std": 0.8318406939506531, "sampling/importance_sampling_ratio/max": 1.0522925853729248, "sampling/importance_sampling_ratio/mean": 0.33937281370162964, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.15384292602539, "sampling/sampling_logp_difference/mean": 1.7843105792999268, "step": 19, "step_time": 7.4325521729988395 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 96.0, "completions/max_terminated_length": 62.0, "completions/mean_length": 29.5625, "completions/mean_terminated_length": 24.076923370361328, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.2045397385954857, "epoch": 0.0002, "frac_reward_zero_std": 0.0, "grad_norm": 0.3339298963546753, "kl": 0.03718238900182769, "learning_rate": 1.4999143285254702e-05, "loss": 0.0795, "num_tokens": 333462.0, "reward": 0.03039749711751938, "reward_std": 0.898818850517273, "rewards/rollout_reward_func/mean": 0.03039749711751938, "rewards/rollout_reward_func/std": 0.898818850517273, "sampling/importance_sampling_ratio/max": 1.158997654914856, "sampling/importance_sampling_ratio/mean": 0.577031135559082, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.222618103027344, "sampling/sampling_logp_difference/mean": 1.1813995838165283, "step": 20, "step_time": 7.158243223999307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 65.0, "completions/max_terminated_length": 65.0, "completions/mean_length": 32.8125, "completions/mean_terminated_length": 23.399999618530273, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.2111548259854317, "epoch": 0.00021, "frac_reward_zero_std": 0.0, "grad_norm": 0.4481445252895355, "kl": 0.022457067796494812, "learning_rate": 1.499866140232449e-05, "loss": -0.0678, "num_tokens": 349479.0, "reward": 0.257499635219574, "reward_std": 0.8863973021507263, "rewards/rollout_reward_func/mean": 0.257499635219574, "rewards/rollout_reward_func/std": 0.8863973617553711, "sampling/importance_sampling_ratio/max": 1.256410002708435, "sampling/importance_sampling_ratio/mean": 0.5914871692657471, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.9514045715332, "sampling/sampling_logp_difference/mean": 1.2352218627929688, "step": 21, "step_time": 7.391028447003919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 80.0, "completions/max_terminated_length": 80.0, "completions/mean_length": 40.0, "completions/mean_terminated_length": 32.22222137451172, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.5568143129348755, "epoch": 0.00022, "frac_reward_zero_std": 0.0, "grad_norm": 0.3243721127510071, "kl": 0.015402485732920468, "learning_rate": 1.4998072452987511e-05, "loss": 0.0085, "num_tokens": 366558.0, "reward": 0.032499995082616806, "reward_std": 0.8984987139701843, "rewards/rollout_reward_func/mean": 0.032499995082616806, "rewards/rollout_reward_func/std": 0.8984987735748291, "sampling/importance_sampling_ratio/max": 1.2086467742919922, "sampling/importance_sampling_ratio/mean": 0.3502025008201599, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.579566955566406, "sampling/sampling_logp_difference/mean": 1.8580657243728638, "step": 22, "step_time": 7.80719813199903 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 71.0, "completions/max_terminated_length": 71.0, "completions/mean_length": 34.25, "completions/mean_terminated_length": 24.72727394104004, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.4188472926616669, "epoch": 0.00023, "frac_reward_zero_std": 0.0, "grad_norm": 0.3766746520996094, "kl": 0.011834485281724483, "learning_rate": 1.499737644845651e-05, "loss": 0.1379, "num_tokens": 382526.0, "reward": 0.17003178596496582, "reward_std": 0.806688666343689, "rewards/rollout_reward_func/mean": 0.17003178596496582, "rewards/rollout_reward_func/std": 0.8066887855529785, "sampling/importance_sampling_ratio/max": 1.3684371709823608, "sampling/importance_sampling_ratio/mean": 0.40652528405189514, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.757999420166016, "sampling/sampling_logp_difference/mean": 1.753040075302124, "step": 23, "step_time": 7.566792009996789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 69.0, "completions/max_terminated_length": 54.0, "completions/mean_length": 31.9375, "completions/mean_terminated_length": 27.0, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.3168350234627724, "epoch": 0.00024, "frac_reward_zero_std": 0.0, "grad_norm": 0.22382819652557373, "kl": 0.04515724384691566, "learning_rate": 1.4996573401982415e-05, "loss": -0.1369, "num_tokens": 398534.0, "reward": 0.2118702381849289, "reward_std": 0.7696114778518677, "rewards/rollout_reward_func/mean": 0.2118702381849289, "rewards/rollout_reward_func/std": 0.7696115970611572, "sampling/importance_sampling_ratio/max": 1.3452194929122925, "sampling/importance_sampling_ratio/mean": 0.5481379628181458, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.12247085571289, "sampling/sampling_logp_difference/mean": 1.669603705406189, "step": 24, "step_time": 6.95720595599596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 50.0, "completions/max_terminated_length": 50.0, "completions/mean_length": 25.3125, "completions/mean_terminated_length": 25.3125, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.4437143057584763, "epoch": 0.00025, "frac_reward_zero_std": 0.0, "grad_norm": 0.2689191699028015, "kl": 0.024810370639897883, "learning_rate": 1.4995663328854067e-05, "loss": 0.1088, "num_tokens": 414685.0, "reward": 0.21249999105930328, "reward_std": 0.9222255349159241, "rewards/rollout_reward_func/mean": 0.21249999105930328, "rewards/rollout_reward_func/std": 0.9222256541252136, "sampling/importance_sampling_ratio/max": 1.4773107767105103, "sampling/importance_sampling_ratio/mean": 0.36113590002059937, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.24507141113281, "sampling/sampling_logp_difference/mean": 2.1210238933563232, "step": 25, "step_time": 6.867648501000076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 68.0, "completions/max_terminated_length": 68.0, "completions/mean_length": 38.875, "completions/mean_terminated_length": 32.400001525878906, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.3743861764669418, "epoch": 0.00026, "frac_reward_zero_std": 0.0, "grad_norm": 0.35105836391448975, "kl": 0.02209025900810957, "learning_rate": 1.499464624639795e-05, "loss": -0.0589, "num_tokens": 431852.0, "reward": 0.2815823554992676, "reward_std": 0.7883099317550659, "rewards/rollout_reward_func/mean": 0.2815823554992676, "rewards/rollout_reward_func/std": 0.7883099317550659, "sampling/importance_sampling_ratio/max": 1.2114486694335938, "sampling/importance_sampling_ratio/mean": 0.3262970447540283, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.25141906738281, "sampling/sampling_logp_difference/mean": 1.737898349761963, "step": 26, "step_time": 8.581968345997666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 78.0, "completions/max_terminated_length": 42.0, "completions/mean_length": 32.125, "completions/mean_terminated_length": 19.55555534362793, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.3159416913986206, "epoch": 0.00027, "frac_reward_zero_std": 0.0, "grad_norm": 0.43860021233558655, "kl": 0.027646008471492678, "learning_rate": 1.4993522173977848e-05, "loss": -0.0605, "num_tokens": 448841.0, "reward": 0.09299126267433167, "reward_std": 0.7795646786689758, "rewards/rollout_reward_func/mean": 0.09299126267433167, "rewards/rollout_reward_func/std": 0.7795647382736206, "sampling/importance_sampling_ratio/max": 1.3604545593261719, "sampling/importance_sampling_ratio/mean": 0.6032359004020691, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.211952209472656, "sampling/sampling_logp_difference/mean": 1.2441456317901611, "step": 27, "step_time": 6.943656498004202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 82.0, "completions/max_terminated_length": 53.0, "completions/mean_length": 33.0, "completions/mean_terminated_length": 22.18181800842285, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.314848430454731, "epoch": 0.00028, "frac_reward_zero_std": 0.0, "grad_norm": 0.41537225246429443, "kl": 0.2036892748437822, "learning_rate": 1.4992291132994477e-05, "loss": -0.1281, "num_tokens": 465235.0, "reward": -0.2775876820087433, "reward_std": 0.7095749974250793, "rewards/rollout_reward_func/mean": -0.2775876820087433, "rewards/rollout_reward_func/std": 0.7095749974250793, "sampling/importance_sampling_ratio/max": 1.490232229232788, "sampling/importance_sampling_ratio/mean": 0.5072295069694519, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.446311950683594, "sampling/sampling_logp_difference/mean": 2.0445632934570312, "step": 28, "step_time": 7.569796797002709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 54.0, "completions/max_terminated_length": 54.0, "completions/mean_length": 26.1875, "completions/mean_terminated_length": 23.545455932617188, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0906936302781105, "epoch": 0.00029, "frac_reward_zero_std": 0.0, "grad_norm": 0.3724343776702881, "kl": 0.019896890386007726, "learning_rate": 1.4990953146885087e-05, "loss": -0.1139, "num_tokens": 480859.0, "reward": 0.07259773463010788, "reward_std": 0.8727007508277893, "rewards/rollout_reward_func/mean": 0.07259773463010788, "rewards/rollout_reward_func/std": 0.8727008104324341, "sampling/importance_sampling_ratio/max": 1.3236075639724731, "sampling/importance_sampling_ratio/mean": 0.6240371465682983, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.669342041015625, "sampling/sampling_logp_difference/mean": 1.1278735399246216, "step": 29, "step_time": 6.49046915100007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 67.0, "completions/max_terminated_length": 35.0, "completions/mean_length": 34.5, "completions/mean_terminated_length": 18.30000114440918, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.3076796159148216, "epoch": 0.0003, "frac_reward_zero_std": 0.0, "grad_norm": 0.29738789796829224, "kl": 0.5647378874127753, "learning_rate": 1.4989508241123002e-05, "loss": -0.01, "num_tokens": 498003.0, "reward": 0.3218036890029907, "reward_std": 0.7443032264709473, "rewards/rollout_reward_func/mean": 0.3218036890029907, "rewards/rollout_reward_func/std": 0.744303286075592, "sampling/importance_sampling_ratio/max": 1.8216454982757568, "sampling/importance_sampling_ratio/mean": 0.3707486093044281, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.435035705566406, "sampling/sampling_logp_difference/mean": 1.7251830101013184, "step": 30, "step_time": 7.929032164998716 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 85.0, "completions/max_terminated_length": 67.0, "completions/mean_length": 37.125, "completions/mean_terminated_length": 28.545455932617188, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.4066748023033142, "epoch": 0.00031, "frac_reward_zero_std": 0.0, "grad_norm": 0.3455451428890228, "kl": 0.023082443978637457, "learning_rate": 1.4987956443217144e-05, "loss": 0.024, "num_tokens": 515167.0, "reward": 0.3658484220504761, "reward_std": 0.6925223469734192, "rewards/rollout_reward_func/mean": 0.3658484220504761, "rewards/rollout_reward_func/std": 0.692522406578064, "sampling/importance_sampling_ratio/max": 1.5549049377441406, "sampling/importance_sampling_ratio/mean": 0.4372093081474304, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.424922943115234, "sampling/sampling_logp_difference/mean": 1.8357561826705933, "step": 31, "step_time": 8.226191027002642 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 96.0, "completions/max_terminated_length": 56.0, "completions/mean_length": 28.8125, "completions/mean_terminated_length": 24.33333396911621, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 0.7501260340213776, "epoch": 0.00032, "frac_reward_zero_std": 0.0, "grad_norm": 0.3879266381263733, "kl": 0.2893356925342232, "learning_rate": 1.498629778271151e-05, "loss": -0.1897, "num_tokens": 531243.0, "reward": 0.5699800848960876, "reward_std": 0.6884264945983887, "rewards/rollout_reward_func/mean": 0.5699800848960876, "rewards/rollout_reward_func/std": 0.6884264945983887, "sampling/importance_sampling_ratio/max": 2.097933769226074, "sampling/importance_sampling_ratio/mean": 0.8604711890220642, "sampling/importance_sampling_ratio/min": 8.407790785948902e-45, "sampling/sampling_logp_difference/max": 29.5391845703125, "sampling/sampling_logp_difference/mean": 0.8243618607521057, "step": 32, "step_time": 6.536235716999727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 64.0, "completions/max_terminated_length": 53.0, "completions/mean_length": 33.0, "completions/mean_terminated_length": 23.363637924194336, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.5932165905833244, "epoch": 0.00033, "frac_reward_zero_std": 0.0, "grad_norm": 0.27561265230178833, "kl": 0.4444279205054045, "learning_rate": 1.4984532291184609e-05, "loss": -0.0436, "num_tokens": 548008.0, "reward": 0.4375, "reward_std": 0.8616843819618225, "rewards/rollout_reward_func/mean": 0.4375, "rewards/rollout_reward_func/std": 0.8616843819618225, "sampling/importance_sampling_ratio/max": 1.143114447593689, "sampling/importance_sampling_ratio/mean": 0.2715457081794739, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 33.419464111328125, "sampling/sampling_logp_difference/mean": 2.2192447185516357, "step": 33, "step_time": 6.985984274004295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 95.0, "completions/max_terminated_length": 95.0, "completions/mean_length": 37.8125, "completions/mean_terminated_length": 27.363637924194336, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.24718077480793, "epoch": 0.00034, "frac_reward_zero_std": 0.0, "grad_norm": 0.8303869962692261, "kl": 0.04481928865425289, "learning_rate": 1.4982660002248846e-05, "loss": -0.0245, "num_tokens": 565025.0, "reward": 0.0986979603767395, "reward_std": 0.778077244758606, "rewards/rollout_reward_func/mean": 0.0986979603767395, "rewards/rollout_reward_func/std": 0.7780773043632507, "sampling/importance_sampling_ratio/max": 2.4489076137542725, "sampling/importance_sampling_ratio/mean": 0.687353789806366, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 34.4277229309082, "sampling/sampling_logp_difference/mean": 1.5159074068069458, "step": 34, "step_time": 8.308581437999237 } ], "logging_steps": 1.0, "max_steps": 735, "num_input_tokens_seen": 565025, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }