| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.00034, |
| "eval_steps": 500, |
| "global_step": 34, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.5, |
| "completions/max_length": 74.0, |
| "completions/max_terminated_length": 74.0, |
| "completions/mean_length": 33.125, |
| "completions/mean_terminated_length": 18.875, |
| "completions/min_length": 6.0, |
| "completions/min_terminated_length": 6.0, |
| "entropy": 1.3827928453683853, |
| "epoch": 1e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3181266486644745, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": -0.0678, |
| "num_tokens": 16641.0, |
| "reward": -0.06401677429676056, |
| "reward_std": 0.7941610813140869, |
| "rewards/rollout_reward_func/mean": -0.06401677429676056, |
| "rewards/rollout_reward_func/std": 0.7941611409187317, |
| "sampling/importance_sampling_ratio/max": 1.144827127456665, |
| "sampling/importance_sampling_ratio/mean": 0.551578164100647, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 32.74828338623047, |
| "sampling/sampling_logp_difference/mean": 1.3979711532592773, |
| "step": 1, |
| "step_time": 7.639951573002691 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1875, |
| "completions/max_length": 63.0, |
| "completions/max_terminated_length": 46.0, |
| "completions/mean_length": 22.5625, |
| "completions/mean_terminated_length": 17.38461685180664, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.5964446440339088, |
| "epoch": 2e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.34730541706085205, |
| "kl": 0.0, |
| "learning_rate": 1e-06, |
| "loss": 0.0602, |
| "num_tokens": 33557.0, |
| "reward": 0.04738499969244003, |
| "reward_std": 0.81049644947052, |
| "rewards/rollout_reward_func/mean": 0.04738499969244003, |
| "rewards/rollout_reward_func/std": 0.8104965090751648, |
| "sampling/importance_sampling_ratio/max": 1.371529221534729, |
| "sampling/importance_sampling_ratio/mean": 0.531771719455719, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.6956672668457, |
| "sampling/sampling_logp_difference/mean": 2.163590908050537, |
| "step": 2, |
| "step_time": 6.844142468999053 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.375, |
| "completions/max_length": 87.0, |
| "completions/max_terminated_length": 56.0, |
| "completions/mean_length": 34.6875, |
| "completions/mean_terminated_length": 24.100000381469727, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.5396408289670944, |
| "epoch": 3e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.40354248881340027, |
| "kl": 0.0019364068793947808, |
| "learning_rate": 2e-06, |
| "loss": 0.0919, |
| "num_tokens": 51093.0, |
| "reward": 0.3400000035762787, |
| "reward_std": 0.8063746094703674, |
| "rewards/rollout_reward_func/mean": 0.3400000035762787, |
| "rewards/rollout_reward_func/std": 0.8063746094703674, |
| "sampling/importance_sampling_ratio/max": 1.2352397441864014, |
| "sampling/importance_sampling_ratio/mean": 0.4642365574836731, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.2575569152832, |
| "sampling/sampling_logp_difference/mean": 1.7650401592254639, |
| "step": 3, |
| "step_time": 8.551386018996709 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.25, |
| "completions/max_length": 64.0, |
| "completions/max_terminated_length": 62.0, |
| "completions/mean_length": 32.125, |
| "completions/mean_terminated_length": 24.166667938232422, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.3548607230186462, |
| "epoch": 4e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.26631447672843933, |
| "kl": 0.001303125623962842, |
| "learning_rate": 3e-06, |
| "loss": 0.052, |
| "num_tokens": 67893.0, |
| "reward": 0.11769548058509827, |
| "reward_std": 0.8427314162254333, |
| "rewards/rollout_reward_func/mean": 0.11769548058509827, |
| "rewards/rollout_reward_func/std": 0.8427314758300781, |
| "sampling/importance_sampling_ratio/max": 1.1722887754440308, |
| "sampling/importance_sampling_ratio/mean": 0.4403129518032074, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.54515838623047, |
| "sampling/sampling_logp_difference/mean": 1.7197456359863281, |
| "step": 4, |
| "step_time": 7.270715081998787 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.25, |
| "completions/max_length": 64.0, |
| "completions/max_terminated_length": 37.0, |
| "completions/mean_length": 26.6875, |
| "completions/mean_terminated_length": 20.666667938232422, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.2926968112587929, |
| "epoch": 5e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.34337735176086426, |
| "kl": 0.000681035679008346, |
| "learning_rate": 4e-06, |
| "loss": 0.035, |
| "num_tokens": 84202.0, |
| "reward": 0.03295581787824631, |
| "reward_std": 0.8984397053718567, |
| "rewards/rollout_reward_func/mean": 0.03295581787824631, |
| "rewards/rollout_reward_func/std": 0.8984397649765015, |
| "sampling/importance_sampling_ratio/max": 1.4395053386688232, |
| "sampling/importance_sampling_ratio/mean": 0.5380887985229492, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 33.7639045715332, |
| "sampling/sampling_logp_difference/mean": 1.5982863903045654, |
| "step": 5, |
| "step_time": 6.337428640003054 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.375, |
| "completions/max_length": 86.0, |
| "completions/max_terminated_length": 86.0, |
| "completions/mean_length": 32.6875, |
| "completions/mean_terminated_length": 20.30000114440918, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.155207872390747, |
| "epoch": 6e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3781796097755432, |
| "kl": 0.0006958278354431968, |
| "learning_rate": 4.9999999999999996e-06, |
| "loss": -0.0204, |
| "num_tokens": 101026.0, |
| "reward": -0.10519320517778397, |
| "reward_std": 0.8153080344200134, |
| "rewards/rollout_reward_func/mean": -0.10519320517778397, |
| "rewards/rollout_reward_func/std": 0.8153082132339478, |
| "sampling/importance_sampling_ratio/max": 1.2898168563842773, |
| "sampling/importance_sampling_ratio/mean": 0.6265079379081726, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 32.706966400146484, |
| "sampling/sampling_logp_difference/mean": 1.0843193531036377, |
| "step": 6, |
| "step_time": 7.546433894000074 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.3125, |
| "completions/max_length": 64.0, |
| "completions/max_terminated_length": 32.0, |
| "completions/mean_length": 26.1875, |
| "completions/mean_terminated_length": 19.636363983154297, |
| "completions/min_length": 7.0, |
| "completions/min_terminated_length": 7.0, |
| "entropy": 1.4203693270683289, |
| "epoch": 7e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4280190169811249, |
| "kl": 0.0019115788309136406, |
| "learning_rate": 6e-06, |
| "loss": 0.0876, |
| "num_tokens": 118090.0, |
| "reward": 0.23219826817512512, |
| "reward_std": 0.8312895894050598, |
| "rewards/rollout_reward_func/mean": 0.23219826817512512, |
| "rewards/rollout_reward_func/std": 0.8312896490097046, |
| "sampling/importance_sampling_ratio/max": 1.3674085140228271, |
| "sampling/importance_sampling_ratio/mean": 0.5275859236717224, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.18802261352539, |
| "sampling/sampling_logp_difference/mean": 1.7722361087799072, |
| "step": 7, |
| "step_time": 6.381835996000518 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.25, |
| "completions/max_length": 64.0, |
| "completions/max_terminated_length": 49.0, |
| "completions/mean_length": 25.125, |
| "completions/mean_terminated_length": 17.25, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.2874730825424194, |
| "epoch": 8e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3058971166610718, |
| "kl": 0.0010104302782565355, |
| "learning_rate": 7e-06, |
| "loss": 0.0967, |
| "num_tokens": 134546.0, |
| "reward": -0.08250001072883606, |
| "reward_std": 0.8818729519844055, |
| "rewards/rollout_reward_func/mean": -0.08250001072883606, |
| "rewards/rollout_reward_func/std": 0.8818729519844055, |
| "sampling/importance_sampling_ratio/max": 1.2262499332427979, |
| "sampling/importance_sampling_ratio/mean": 0.6900386214256287, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 33.81074523925781, |
| "sampling/sampling_logp_difference/mean": 1.485697627067566, |
| "step": 8, |
| "step_time": 7.5347441070043715 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.25, |
| "completions/max_length": 64.0, |
| "completions/max_terminated_length": 44.0, |
| "completions/mean_length": 21.9375, |
| "completions/mean_terminated_length": 15.166666984558105, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.3974642157554626, |
| "epoch": 9e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.29421234130859375, |
| "kl": 0.0017157641450467054, |
| "learning_rate": 8e-06, |
| "loss": 0.0548, |
| "num_tokens": 150906.0, |
| "reward": 0.0730893611907959, |
| "reward_std": 0.872587263584137, |
| "rewards/rollout_reward_func/mean": 0.0730893611907959, |
| "rewards/rollout_reward_func/std": 0.872587263584137, |
| "sampling/importance_sampling_ratio/max": 1.183666467666626, |
| "sampling/importance_sampling_ratio/mean": 0.511037290096283, |
| "sampling/importance_sampling_ratio/min": 2.324053503082709e-41, |
| "sampling/sampling_logp_difference/max": 34.89555358886719, |
| "sampling/sampling_logp_difference/mean": 1.7038354873657227, |
| "step": 9, |
| "step_time": 6.773095810000086 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.375, |
| "completions/max_length": 57.0, |
| "completions/max_terminated_length": 57.0, |
| "completions/mean_length": 29.75, |
| "completions/mean_terminated_length": 25.600000381469727, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.5960686951875687, |
| "epoch": 0.0001, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3341529667377472, |
| "kl": 0.0010894886290770955, |
| "learning_rate": 9e-06, |
| "loss": 0.0245, |
| "num_tokens": 167587.0, |
| "reward": 0.5254915952682495, |
| "reward_std": 0.7537170052528381, |
| "rewards/rollout_reward_func/mean": 0.5254915952682495, |
| "rewards/rollout_reward_func/std": 0.7537170648574829, |
| "sampling/importance_sampling_ratio/max": 1.0177525281906128, |
| "sampling/importance_sampling_ratio/mean": 0.4141247272491455, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 32.73265838623047, |
| "sampling/sampling_logp_difference/mean": 2.074547052383423, |
| "step": 10, |
| "step_time": 6.915056184001514 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1875, |
| "completions/max_length": 49.0, |
| "completions/max_terminated_length": 46.0, |
| "completions/mean_length": 22.125, |
| "completions/mean_terminated_length": 18.076923370361328, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.4635429829359055, |
| "epoch": 0.00011, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.35319551825523376, |
| "kl": 0.0016797076386865228, |
| "learning_rate": 9.999999999999999e-06, |
| "loss": 0.0826, |
| "num_tokens": 183363.0, |
| "reward": 0.005195483565330505, |
| "reward_std": 0.837223470211029, |
| "rewards/rollout_reward_func/mean": 0.005195483565330505, |
| "rewards/rollout_reward_func/std": 0.8372235298156738, |
| "sampling/importance_sampling_ratio/max": 1.2394757270812988, |
| "sampling/importance_sampling_ratio/mean": 0.5569560527801514, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 36.220245361328125, |
| "sampling/sampling_logp_difference/mean": 2.1369495391845703, |
| "step": 11, |
| "step_time": 5.859747936994609 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1875, |
| "completions/max_length": 96.0, |
| "completions/max_terminated_length": 60.0, |
| "completions/mean_length": 32.6875, |
| "completions/mean_terminated_length": 26.923078536987305, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.6477473005652428, |
| "epoch": 0.00012, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3246020972728729, |
| "kl": 0.003157808539981488, |
| "learning_rate": 1.1e-05, |
| "loss": 0.0995, |
| "num_tokens": 200806.0, |
| "reward": -0.0572887659072876, |
| "reward_std": 0.7941725850105286, |
| "rewards/rollout_reward_func/mean": -0.0572887659072876, |
| "rewards/rollout_reward_func/std": 0.7941725850105286, |
| "sampling/importance_sampling_ratio/max": 1.1799664497375488, |
| "sampling/importance_sampling_ratio/mean": 0.39019522070884705, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.21113204956055, |
| "sampling/sampling_logp_difference/mean": 1.9028511047363281, |
| "step": 12, |
| "step_time": 7.54413384599502 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1875, |
| "completions/max_length": 72.0, |
| "completions/max_terminated_length": 53.0, |
| "completions/mean_length": 26.6875, |
| "completions/mean_terminated_length": 19.923078536987305, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 1.274285078048706, |
| "epoch": 0.00013, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.32733649015426636, |
| "kl": 0.001991469122003764, |
| "learning_rate": 1.2e-05, |
| "loss": -0.0647, |
| "num_tokens": 217199.0, |
| "reward": 0.31669238209724426, |
| "reward_std": 0.7471501231193542, |
| "rewards/rollout_reward_func/mean": 0.31669238209724426, |
| "rewards/rollout_reward_func/std": 0.747150182723999, |
| "sampling/importance_sampling_ratio/max": 1.236131191253662, |
| "sampling/importance_sampling_ratio/mean": 0.6276021599769592, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 32.89448928833008, |
| "sampling/sampling_logp_difference/mean": 1.8165452480316162, |
| "step": 13, |
| "step_time": 7.0487566130013874 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.25, |
| "completions/max_length": 92.0, |
| "completions/max_terminated_length": 92.0, |
| "completions/mean_length": 27.25, |
| "completions/mean_terminated_length": 21.83333396911621, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.0698126628994942, |
| "epoch": 0.00014, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3421112895011902, |
| "kl": 0.005695157662557904, |
| "learning_rate": 1.3000000000000001e-05, |
| "loss": 0.0923, |
| "num_tokens": 233569.0, |
| "reward": 0.1585809737443924, |
| "reward_std": 0.8103994727134705, |
| "rewards/rollout_reward_func/mean": 0.1585809737443924, |
| "rewards/rollout_reward_func/std": 0.8103994727134705, |
| "sampling/importance_sampling_ratio/max": 1.1087441444396973, |
| "sampling/importance_sampling_ratio/mean": 0.6734969615936279, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.04454803466797, |
| "sampling/sampling_logp_difference/mean": 0.9640688300132751, |
| "step": 14, |
| "step_time": 7.728960284001005 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1875, |
| "completions/max_length": 64.0, |
| "completions/max_terminated_length": 35.0, |
| "completions/mean_length": 26.875, |
| "completions/mean_terminated_length": 20.76923179626465, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.4536470770835876, |
| "epoch": 0.00015, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.402100533246994, |
| "kl": 0.010122014762600884, |
| "learning_rate": 1.4e-05, |
| "loss": 0.0717, |
| "num_tokens": 250395.0, |
| "reward": 0.32499998807907104, |
| "reward_std": 0.8999999761581421, |
| "rewards/rollout_reward_func/mean": 0.32499998807907104, |
| "rewards/rollout_reward_func/std": 0.9000000357627869, |
| "sampling/importance_sampling_ratio/max": 1.1882612705230713, |
| "sampling/importance_sampling_ratio/mean": 0.43665286898612976, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.10694122314453, |
| "sampling/sampling_logp_difference/mean": 1.8059306144714355, |
| "step": 15, |
| "step_time": 6.667633953000404 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.4375, |
| "completions/max_length": 71.0, |
| "completions/max_terminated_length": 71.0, |
| "completions/mean_length": 33.4375, |
| "completions/mean_terminated_length": 24.22222328186035, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.5715003088116646, |
| "epoch": 0.00016, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.278489887714386, |
| "kl": 0.010550641687586904, |
| "learning_rate": 1.5e-05, |
| "loss": -0.0207, |
| "num_tokens": 266674.0, |
| "reward": 0.34981411695480347, |
| "reward_std": 0.8006922602653503, |
| "rewards/rollout_reward_func/mean": 0.34981411695480347, |
| "rewards/rollout_reward_func/std": 0.8006923198699951, |
| "sampling/importance_sampling_ratio/max": 1.1930525302886963, |
| "sampling/importance_sampling_ratio/mean": 0.37038883566856384, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 35.258827209472656, |
| "sampling/sampling_logp_difference/mean": 2.096559524536133, |
| "step": 16, |
| "step_time": 7.509245967999959 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.4375, |
| "completions/max_length": 70.0, |
| "completions/max_terminated_length": 70.0, |
| "completions/mean_length": 31.4375, |
| "completions/mean_terminated_length": 23.88888931274414, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 1.1031887084245682, |
| "epoch": 0.00017, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5009875893592834, |
| "kl": 0.0252107827400323, |
| "learning_rate": 1.4999946454054132e-05, |
| "loss": -0.141, |
| "num_tokens": 283142.0, |
| "reward": -0.264902263879776, |
| "reward_std": 0.7876402139663696, |
| "rewards/rollout_reward_func/mean": -0.264902263879776, |
| "rewards/rollout_reward_func/std": 0.7876402139663696, |
| "sampling/importance_sampling_ratio/max": 1.2886087894439697, |
| "sampling/importance_sampling_ratio/mean": 0.6680909395217896, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.088829040527344, |
| "sampling/sampling_logp_difference/mean": 1.021271824836731, |
| "step": 17, |
| "step_time": 6.736764096998741 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.375, |
| "completions/max_length": 96.0, |
| "completions/max_terminated_length": 53.0, |
| "completions/mean_length": 40.0625, |
| "completions/mean_terminated_length": 31.80000114440918, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.2849810495972633, |
| "epoch": 0.00018, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41720253229141235, |
| "kl": 0.02835782650799956, |
| "learning_rate": 1.4999785817235962e-05, |
| "loss": -0.0753, |
| "num_tokens": 300104.0, |
| "reward": 0.27464818954467773, |
| "reward_std": 0.7914400100708008, |
| "rewards/rollout_reward_func/mean": 0.27464818954467773, |
| "rewards/rollout_reward_func/std": 0.7914400696754456, |
| "sampling/importance_sampling_ratio/max": 1.4827672243118286, |
| "sampling/importance_sampling_ratio/mean": 0.40857595205307007, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.11698532104492, |
| "sampling/sampling_logp_difference/mean": 1.3236627578735352, |
| "step": 18, |
| "step_time": 8.64123790499616 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.3125, |
| "completions/max_length": 63.0, |
| "completions/max_terminated_length": 63.0, |
| "completions/mean_length": 29.625, |
| "completions/mean_terminated_length": 24.545455932617188, |
| "completions/min_length": 11.0, |
| "completions/min_terminated_length": 11.0, |
| "entropy": 1.4260313510894775, |
| "epoch": 0.00019, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1461358666419983, |
| "kl": 0.009397892834385857, |
| "learning_rate": 1.499951809260379e-05, |
| "loss": 0.0238, |
| "num_tokens": 317197.0, |
| "reward": 0.23086139559745789, |
| "reward_std": 0.8318407535552979, |
| "rewards/rollout_reward_func/mean": 0.23086139559745789, |
| "rewards/rollout_reward_func/std": 0.8318406939506531, |
| "sampling/importance_sampling_ratio/max": 1.0522925853729248, |
| "sampling/importance_sampling_ratio/mean": 0.33937281370162964, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.15384292602539, |
| "sampling/sampling_logp_difference/mean": 1.7843105792999268, |
| "step": 19, |
| "step_time": 7.4325521729988395 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1875, |
| "completions/max_length": 96.0, |
| "completions/max_terminated_length": 62.0, |
| "completions/mean_length": 29.5625, |
| "completions/mean_terminated_length": 24.076923370361328, |
| "completions/min_length": 6.0, |
| "completions/min_terminated_length": 6.0, |
| "entropy": 1.2045397385954857, |
| "epoch": 0.0002, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3339298963546753, |
| "kl": 0.03718238900182769, |
| "learning_rate": 1.4999143285254702e-05, |
| "loss": 0.0795, |
| "num_tokens": 333462.0, |
| "reward": 0.03039749711751938, |
| "reward_std": 0.898818850517273, |
| "rewards/rollout_reward_func/mean": 0.03039749711751938, |
| "rewards/rollout_reward_func/std": 0.898818850517273, |
| "sampling/importance_sampling_ratio/max": 1.158997654914856, |
| "sampling/importance_sampling_ratio/mean": 0.577031135559082, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 33.222618103027344, |
| "sampling/sampling_logp_difference/mean": 1.1813995838165283, |
| "step": 20, |
| "step_time": 7.158243223999307 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.375, |
| "completions/max_length": 65.0, |
| "completions/max_terminated_length": 65.0, |
| "completions/mean_length": 32.8125, |
| "completions/mean_terminated_length": 23.399999618530273, |
| "completions/min_length": 6.0, |
| "completions/min_terminated_length": 6.0, |
| "entropy": 1.2111548259854317, |
| "epoch": 0.00021, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4481445252895355, |
| "kl": 0.022457067796494812, |
| "learning_rate": 1.499866140232449e-05, |
| "loss": -0.0678, |
| "num_tokens": 349479.0, |
| "reward": 0.257499635219574, |
| "reward_std": 0.8863973021507263, |
| "rewards/rollout_reward_func/mean": 0.257499635219574, |
| "rewards/rollout_reward_func/std": 0.8863973617553711, |
| "sampling/importance_sampling_ratio/max": 1.256410002708435, |
| "sampling/importance_sampling_ratio/mean": 0.5914871692657471, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 33.9514045715332, |
| "sampling/sampling_logp_difference/mean": 1.2352218627929688, |
| "step": 21, |
| "step_time": 7.391028447003919 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.4375, |
| "completions/max_length": 80.0, |
| "completions/max_terminated_length": 80.0, |
| "completions/mean_length": 40.0, |
| "completions/mean_terminated_length": 32.22222137451172, |
| "completions/min_length": 11.0, |
| "completions/min_terminated_length": 11.0, |
| "entropy": 1.5568143129348755, |
| "epoch": 0.00022, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3243721127510071, |
| "kl": 0.015402485732920468, |
| "learning_rate": 1.4998072452987511e-05, |
| "loss": 0.0085, |
| "num_tokens": 366558.0, |
| "reward": 0.032499995082616806, |
| "reward_std": 0.8984987139701843, |
| "rewards/rollout_reward_func/mean": 0.032499995082616806, |
| "rewards/rollout_reward_func/std": 0.8984987735748291, |
| "sampling/importance_sampling_ratio/max": 1.2086467742919922, |
| "sampling/importance_sampling_ratio/mean": 0.3502025008201599, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 33.579566955566406, |
| "sampling/sampling_logp_difference/mean": 1.8580657243728638, |
| "step": 22, |
| "step_time": 7.80719813199903 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.3125, |
| "completions/max_length": 71.0, |
| "completions/max_terminated_length": 71.0, |
| "completions/mean_length": 34.25, |
| "completions/mean_terminated_length": 24.72727394104004, |
| "completions/min_length": 8.0, |
| "completions/min_terminated_length": 8.0, |
| "entropy": 1.4188472926616669, |
| "epoch": 0.00023, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3766746520996094, |
| "kl": 0.011834485281724483, |
| "learning_rate": 1.499737644845651e-05, |
| "loss": 0.1379, |
| "num_tokens": 382526.0, |
| "reward": 0.17003178596496582, |
| "reward_std": 0.806688666343689, |
| "rewards/rollout_reward_func/mean": 0.17003178596496582, |
| "rewards/rollout_reward_func/std": 0.8066887855529785, |
| "sampling/importance_sampling_ratio/max": 1.3684371709823608, |
| "sampling/importance_sampling_ratio/mean": 0.40652528405189514, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.757999420166016, |
| "sampling/sampling_logp_difference/mean": 1.753040075302124, |
| "step": 23, |
| "step_time": 7.566792009996789 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.3125, |
| "completions/max_length": 69.0, |
| "completions/max_terminated_length": 54.0, |
| "completions/mean_length": 31.9375, |
| "completions/mean_terminated_length": 27.0, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 1.3168350234627724, |
| "epoch": 0.00024, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.22382819652557373, |
| "kl": 0.04515724384691566, |
| "learning_rate": 1.4996573401982415e-05, |
| "loss": -0.1369, |
| "num_tokens": 398534.0, |
| "reward": 0.2118702381849289, |
| "reward_std": 0.7696114778518677, |
| "rewards/rollout_reward_func/mean": 0.2118702381849289, |
| "rewards/rollout_reward_func/std": 0.7696115970611572, |
| "sampling/importance_sampling_ratio/max": 1.3452194929122925, |
| "sampling/importance_sampling_ratio/mean": 0.5481379628181458, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.12247085571289, |
| "sampling/sampling_logp_difference/mean": 1.669603705406189, |
| "step": 24, |
| "step_time": 6.95720595599596 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 50.0, |
| "completions/max_terminated_length": 50.0, |
| "completions/mean_length": 25.3125, |
| "completions/mean_terminated_length": 25.3125, |
| "completions/min_length": 6.0, |
| "completions/min_terminated_length": 6.0, |
| "entropy": 1.4437143057584763, |
| "epoch": 0.00025, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.2689191699028015, |
| "kl": 0.024810370639897883, |
| "learning_rate": 1.4995663328854067e-05, |
| "loss": 0.1088, |
| "num_tokens": 414685.0, |
| "reward": 0.21249999105930328, |
| "reward_std": 0.9222255349159241, |
| "rewards/rollout_reward_func/mean": 0.21249999105930328, |
| "rewards/rollout_reward_func/std": 0.9222256541252136, |
| "sampling/importance_sampling_ratio/max": 1.4773107767105103, |
| "sampling/importance_sampling_ratio/mean": 0.36113590002059937, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.24507141113281, |
| "sampling/sampling_logp_difference/mean": 2.1210238933563232, |
| "step": 25, |
| "step_time": 6.867648501000076 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.375, |
| "completions/max_length": 68.0, |
| "completions/max_terminated_length": 68.0, |
| "completions/mean_length": 38.875, |
| "completions/mean_terminated_length": 32.400001525878906, |
| "completions/min_length": 6.0, |
| "completions/min_terminated_length": 6.0, |
| "entropy": 1.3743861764669418, |
| "epoch": 0.00026, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.35105836391448975, |
| "kl": 0.02209025900810957, |
| "learning_rate": 1.499464624639795e-05, |
| "loss": -0.0589, |
| "num_tokens": 431852.0, |
| "reward": 0.2815823554992676, |
| "reward_std": 0.7883099317550659, |
| "rewards/rollout_reward_func/mean": 0.2815823554992676, |
| "rewards/rollout_reward_func/std": 0.7883099317550659, |
| "sampling/importance_sampling_ratio/max": 1.2114486694335938, |
| "sampling/importance_sampling_ratio/mean": 0.3262970447540283, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.25141906738281, |
| "sampling/sampling_logp_difference/mean": 1.737898349761963, |
| "step": 26, |
| "step_time": 8.581968345997666 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.4375, |
| "completions/max_length": 78.0, |
| "completions/max_terminated_length": 42.0, |
| "completions/mean_length": 32.125, |
| "completions/mean_terminated_length": 19.55555534362793, |
| "completions/min_length": 6.0, |
| "completions/min_terminated_length": 6.0, |
| "entropy": 1.3159416913986206, |
| "epoch": 0.00027, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.43860021233558655, |
| "kl": 0.027646008471492678, |
| "learning_rate": 1.4993522173977848e-05, |
| "loss": -0.0605, |
| "num_tokens": 448841.0, |
| "reward": 0.09299126267433167, |
| "reward_std": 0.7795646786689758, |
| "rewards/rollout_reward_func/mean": 0.09299126267433167, |
| "rewards/rollout_reward_func/std": 0.7795647382736206, |
| "sampling/importance_sampling_ratio/max": 1.3604545593261719, |
| "sampling/importance_sampling_ratio/mean": 0.6032359004020691, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.211952209472656, |
| "sampling/sampling_logp_difference/mean": 1.2441456317901611, |
| "step": 27, |
| "step_time": 6.943656498004202 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.3125, |
| "completions/max_length": 82.0, |
| "completions/max_terminated_length": 53.0, |
| "completions/mean_length": 33.0, |
| "completions/mean_terminated_length": 22.18181800842285, |
| "completions/min_length": 9.0, |
| "completions/min_terminated_length": 9.0, |
| "entropy": 1.314848430454731, |
| "epoch": 0.00028, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.41537225246429443, |
| "kl": 0.2036892748437822, |
| "learning_rate": 1.4992291132994477e-05, |
| "loss": -0.1281, |
| "num_tokens": 465235.0, |
| "reward": -0.2775876820087433, |
| "reward_std": 0.7095749974250793, |
| "rewards/rollout_reward_func/mean": -0.2775876820087433, |
| "rewards/rollout_reward_func/std": 0.7095749974250793, |
| "sampling/importance_sampling_ratio/max": 1.490232229232788, |
| "sampling/importance_sampling_ratio/mean": 0.5072295069694519, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.446311950683594, |
| "sampling/sampling_logp_difference/mean": 2.0445632934570312, |
| "step": 28, |
| "step_time": 7.569796797002709 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.3125, |
| "completions/max_length": 54.0, |
| "completions/max_terminated_length": 54.0, |
| "completions/mean_length": 26.1875, |
| "completions/mean_terminated_length": 23.545455932617188, |
| "completions/min_length": 9.0, |
| "completions/min_terminated_length": 9.0, |
| "entropy": 1.0906936302781105, |
| "epoch": 0.00029, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3724343776702881, |
| "kl": 0.019896890386007726, |
| "learning_rate": 1.4990953146885087e-05, |
| "loss": -0.1139, |
| "num_tokens": 480859.0, |
| "reward": 0.07259773463010788, |
| "reward_std": 0.8727007508277893, |
| "rewards/rollout_reward_func/mean": 0.07259773463010788, |
| "rewards/rollout_reward_func/std": 0.8727008104324341, |
| "sampling/importance_sampling_ratio/max": 1.3236075639724731, |
| "sampling/importance_sampling_ratio/mean": 0.6240371465682983, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 33.669342041015625, |
| "sampling/sampling_logp_difference/mean": 1.1278735399246216, |
| "step": 29, |
| "step_time": 6.49046915100007 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.375, |
| "completions/max_length": 67.0, |
| "completions/max_terminated_length": 35.0, |
| "completions/mean_length": 34.5, |
| "completions/mean_terminated_length": 18.30000114440918, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.3076796159148216, |
| "epoch": 0.0003, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.29738789796829224, |
| "kl": 0.5647378874127753, |
| "learning_rate": 1.4989508241123002e-05, |
| "loss": -0.01, |
| "num_tokens": 498003.0, |
| "reward": 0.3218036890029907, |
| "reward_std": 0.7443032264709473, |
| "rewards/rollout_reward_func/mean": 0.3218036890029907, |
| "rewards/rollout_reward_func/std": 0.744303286075592, |
| "sampling/importance_sampling_ratio/max": 1.8216454982757568, |
| "sampling/importance_sampling_ratio/mean": 0.3707486093044281, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 33.435035705566406, |
| "sampling/sampling_logp_difference/mean": 1.7251830101013184, |
| "step": 30, |
| "step_time": 7.929032164998716 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.3125, |
| "completions/max_length": 85.0, |
| "completions/max_terminated_length": 67.0, |
| "completions/mean_length": 37.125, |
| "completions/mean_terminated_length": 28.545455932617188, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 1.4066748023033142, |
| "epoch": 0.00031, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3455451428890228, |
| "kl": 0.023082443978637457, |
| "learning_rate": 1.4987956443217144e-05, |
| "loss": 0.024, |
| "num_tokens": 515167.0, |
| "reward": 0.3658484220504761, |
| "reward_std": 0.6925223469734192, |
| "rewards/rollout_reward_func/mean": 0.3658484220504761, |
| "rewards/rollout_reward_func/std": 0.692522406578064, |
| "sampling/importance_sampling_ratio/max": 1.5549049377441406, |
| "sampling/importance_sampling_ratio/mean": 0.4372093081474304, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.424922943115234, |
| "sampling/sampling_logp_difference/mean": 1.8357561826705933, |
| "step": 31, |
| "step_time": 8.226191027002642 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0625, |
| "completions/max_length": 96.0, |
| "completions/max_terminated_length": 56.0, |
| "completions/mean_length": 28.8125, |
| "completions/mean_terminated_length": 24.33333396911621, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 0.7501260340213776, |
| "epoch": 0.00032, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3879266381263733, |
| "kl": 0.2893356925342232, |
| "learning_rate": 1.498629778271151e-05, |
| "loss": -0.1897, |
| "num_tokens": 531243.0, |
| "reward": 0.5699800848960876, |
| "reward_std": 0.6884264945983887, |
| "rewards/rollout_reward_func/mean": 0.5699800848960876, |
| "rewards/rollout_reward_func/std": 0.6884264945983887, |
| "sampling/importance_sampling_ratio/max": 2.097933769226074, |
| "sampling/importance_sampling_ratio/mean": 0.8604711890220642, |
| "sampling/importance_sampling_ratio/min": 8.407790785948902e-45, |
| "sampling/sampling_logp_difference/max": 29.5391845703125, |
| "sampling/sampling_logp_difference/mean": 0.8243618607521057, |
| "step": 32, |
| "step_time": 6.536235716999727 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.3125, |
| "completions/max_length": 64.0, |
| "completions/max_terminated_length": 53.0, |
| "completions/mean_length": 33.0, |
| "completions/mean_terminated_length": 23.363637924194336, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 1.5932165905833244, |
| "epoch": 0.00033, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.27561265230178833, |
| "kl": 0.4444279205054045, |
| "learning_rate": 1.4984532291184609e-05, |
| "loss": -0.0436, |
| "num_tokens": 548008.0, |
| "reward": 0.4375, |
| "reward_std": 0.8616843819618225, |
| "rewards/rollout_reward_func/mean": 0.4375, |
| "rewards/rollout_reward_func/std": 0.8616843819618225, |
| "sampling/importance_sampling_ratio/max": 1.143114447593689, |
| "sampling/importance_sampling_ratio/mean": 0.2715457081794739, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 33.419464111328125, |
| "sampling/sampling_logp_difference/mean": 2.2192447185516357, |
| "step": 33, |
| "step_time": 6.985984274004295 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.3125, |
| "completions/max_length": 95.0, |
| "completions/max_terminated_length": 95.0, |
| "completions/mean_length": 37.8125, |
| "completions/mean_terminated_length": 27.363637924194336, |
| "completions/min_length": 8.0, |
| "completions/min_terminated_length": 8.0, |
| "entropy": 1.24718077480793, |
| "epoch": 0.00034, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.8303869962692261, |
| "kl": 0.04481928865425289, |
| "learning_rate": 1.4982660002248846e-05, |
| "loss": -0.0245, |
| "num_tokens": 565025.0, |
| "reward": 0.0986979603767395, |
| "reward_std": 0.778077244758606, |
| "rewards/rollout_reward_func/mean": 0.0986979603767395, |
| "rewards/rollout_reward_func/std": 0.7780773043632507, |
| "sampling/importance_sampling_ratio/max": 2.4489076137542725, |
| "sampling/importance_sampling_ratio/mean": 0.687353789806366, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 34.4277229309082, |
| "sampling/sampling_logp_difference/mean": 1.5159074068069458, |
| "step": 34, |
| "step_time": 8.308581437999237 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 735, |
| "num_input_tokens_seen": 565025, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
