| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.24, | |
| "eval_steps": 500, | |
| "global_step": 60, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9999.0, | |
| "completions/max_terminated_length": 9999.0, | |
| "completions/mean_length": 4554.453125, | |
| "completions/mean_terminated_length": 4554.453125, | |
| "completions/min_length": 551.0, | |
| "completions/min_terminated_length": 551.0, | |
| "entropy": 0.31497739674523473, | |
| "epoch": 0.004, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.27671217918395996, | |
| "kl": 0.0, | |
| "learning_rate": 0.0, | |
| "loss": 0.1899, | |
| "num_tokens": 332413.0, | |
| "reward": 0.5768750309944153, | |
| "reward_std": 0.4988396167755127, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.5768749713897705, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5170210003852844, | |
| "sampling/importance_sampling_ratio/max": 2.406592845916748, | |
| "sampling/importance_sampling_ratio/mean": 0.6213822364807129, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 55.82071304321289, | |
| "sampling/sampling_logp_difference/mean": 0.023054132238030434, | |
| "step": 1, | |
| "step_time": 293.2645074960601 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 8320.0, | |
| "completions/max_terminated_length": 8320.0, | |
| "completions/mean_length": 5251.21875, | |
| "completions/mean_terminated_length": 5251.21875, | |
| "completions/min_length": 1861.0, | |
| "completions/min_terminated_length": 1861.0, | |
| "entropy": 0.3733639745041728, | |
| "epoch": 0.008, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.26362094283103943, | |
| "kl": 0.0, | |
| "learning_rate": 1.5e-06, | |
| "loss": 0.0961, | |
| "num_tokens": 704651.0, | |
| "reward": 0.13640625774860382, | |
| "reward_std": 0.43152421712875366, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.13640624284744263, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4294232726097107, | |
| "sampling/importance_sampling_ratio/max": 2.881873846054077, | |
| "sampling/importance_sampling_ratio/mean": 0.4364074170589447, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.0964527130126953, | |
| "sampling/sampling_logp_difference/mean": 0.019750293344259262, | |
| "step": 2, | |
| "step_time": 281.28423442596977 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 10722.0, | |
| "completions/max_terminated_length": 10722.0, | |
| "completions/mean_length": 6977.828125, | |
| "completions/mean_terminated_length": 6977.828125, | |
| "completions/min_length": 1154.0, | |
| "completions/min_terminated_length": 1154.0, | |
| "entropy": 0.3223036811687052, | |
| "epoch": 0.012, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.19866329431533813, | |
| "kl": 0.0017119673575507477, | |
| "learning_rate": 3e-06, | |
| "loss": 0.2902, | |
| "num_tokens": 1196416.0, | |
| "reward": 0.24703124165534973, | |
| "reward_std": 0.4671919345855713, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.24703125655651093, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4794430732727051, | |
| "sampling/importance_sampling_ratio/max": 2.9424545764923096, | |
| "sampling/importance_sampling_ratio/mean": 0.46759214997291565, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.1991941928863525, | |
| "sampling/sampling_logp_difference/mean": 0.019809413701295853, | |
| "step": 3, | |
| "step_time": 336.2479823809699 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9874.0, | |
| "completions/max_terminated_length": 9874.0, | |
| "completions/mean_length": 6289.03125, | |
| "completions/mean_terminated_length": 6289.03125, | |
| "completions/min_length": 3976.0, | |
| "completions/min_terminated_length": 3976.0, | |
| "entropy": 0.33153627486899495, | |
| "epoch": 0.016, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.20725910365581512, | |
| "kl": 0.0019219094938307535, | |
| "learning_rate": 2.9980747607565792e-06, | |
| "loss": -0.0184, | |
| "num_tokens": 1636162.0, | |
| "reward": 0.0007812529802322388, | |
| "reward_std": 0.17323368787765503, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.0007812529802322388, | |
| "rewards/alfworld_rollout_reward_func/std": 0.2191564291715622, | |
| "sampling/importance_sampling_ratio/max": 2.584482192993164, | |
| "sampling/importance_sampling_ratio/mean": 0.38795873522758484, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 4.953186511993408, | |
| "sampling/sampling_logp_difference/mean": 0.020250197499990463, | |
| "step": 4, | |
| "step_time": 313.9296533779707 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 11285.0, | |
| "completions/max_terminated_length": 11285.0, | |
| "completions/mean_length": 7613.421875, | |
| "completions/mean_terminated_length": 7613.421875, | |
| "completions/min_length": 2227.0, | |
| "completions/min_terminated_length": 2227.0, | |
| "entropy": 0.34567126259207726, | |
| "epoch": 0.02, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.17643821239471436, | |
| "kl": 0.0016085030692920554, | |
| "learning_rate": 2.9923039850878425e-06, | |
| "loss": 0.0963, | |
| "num_tokens": 2170909.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.4111865162849426, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.1875, | |
| "rewards/alfworld_rollout_reward_func/std": 0.445791631937027, | |
| "sampling/importance_sampling_ratio/max": 2.329904079437256, | |
| "sampling/importance_sampling_ratio/mean": 0.4071568250656128, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 51.395050048828125, | |
| "sampling/sampling_logp_difference/mean": 0.02332460880279541, | |
| "step": 5, | |
| "step_time": 396.88119790001656 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 10068.0, | |
| "completions/max_terminated_length": 10068.0, | |
| "completions/mean_length": 7365.96875, | |
| "completions/mean_terminated_length": 7365.96875, | |
| "completions/min_length": 2236.0, | |
| "completions/min_terminated_length": 2236.0, | |
| "entropy": 0.33807497285306454, | |
| "epoch": 0.024, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2105790078639984, | |
| "kl": 0.002014314475673018, | |
| "learning_rate": 2.982702486492167e-06, | |
| "loss": 0.1834, | |
| "num_tokens": 2690011.0, | |
| "reward": 0.18296873569488525, | |
| "reward_std": 0.34179380536079407, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.18296875059604645, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4501417279243469, | |
| "sampling/importance_sampling_ratio/max": 2.981616735458374, | |
| "sampling/importance_sampling_ratio/mean": 0.4046768844127655, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 5.697381019592285, | |
| "sampling/sampling_logp_difference/mean": 0.020046193152666092, | |
| "step": 6, | |
| "step_time": 348.1233048220165 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9846.0, | |
| "completions/max_terminated_length": 9846.0, | |
| "completions/mean_length": 6610.140625, | |
| "completions/mean_terminated_length": 6610.140625, | |
| "completions/min_length": 808.0, | |
| "completions/min_terminated_length": 808.0, | |
| "entropy": 0.3595086098648608, | |
| "epoch": 0.028, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.218948632478714, | |
| "kl": 0.0016301772157021333, | |
| "learning_rate": 2.969294911878742e-06, | |
| "loss": 0.0717, | |
| "num_tokens": 3155044.0, | |
| "reward": 0.171875, | |
| "reward_std": 0.41918015480041504, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.171875, | |
| "rewards/alfworld_rollout_reward_func/std": 0.42613014578819275, | |
| "sampling/importance_sampling_ratio/max": 2.5824413299560547, | |
| "sampling/importance_sampling_ratio/mean": 0.3877456784248352, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.626394271850586, | |
| "sampling/sampling_logp_difference/mean": 0.020398743450641632, | |
| "step": 7, | |
| "step_time": 351.79137006499514 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 11160.0, | |
| "completions/max_terminated_length": 11160.0, | |
| "completions/mean_length": 7036.953125, | |
| "completions/mean_terminated_length": 7036.953125, | |
| "completions/min_length": 1917.0, | |
| "completions/min_terminated_length": 1917.0, | |
| "entropy": 0.3341470160521567, | |
| "epoch": 0.032, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2093495875597, | |
| "kl": 0.0016681601082382258, | |
| "learning_rate": 2.9521156782993067e-06, | |
| "loss": -0.0439, | |
| "num_tokens": 3658497.0, | |
| "reward": 0.41718751192092896, | |
| "reward_std": 0.5516971349716187, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.41718751192092896, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5447016954421997, | |
| "sampling/importance_sampling_ratio/max": 2.776736259460449, | |
| "sampling/importance_sampling_ratio/mean": 0.4658258259296417, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 56.51785659790039, | |
| "sampling/sampling_logp_difference/mean": 0.022869806736707687, | |
| "step": 8, | |
| "step_time": 329.31609062396456 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9616.0, | |
| "completions/max_terminated_length": 9616.0, | |
| "completions/mean_length": 8430.71875, | |
| "completions/mean_terminated_length": 8430.71875, | |
| "completions/min_length": 4983.0, | |
| "completions/min_terminated_length": 4983.0, | |
| "entropy": 0.3272924036718905, | |
| "epoch": 0.036, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.5628678202629089, | |
| "kl": 0.0017659663353697397, | |
| "learning_rate": 2.9312088846000733e-06, | |
| "loss": -0.2308, | |
| "num_tokens": 4245423.0, | |
| "reward": 0.032499998807907104, | |
| "reward_std": 0.24380016326904297, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.032499998807907104, | |
| "rewards/alfworld_rollout_reward_func/std": 0.2736351490020752, | |
| "sampling/importance_sampling_ratio/max": 2.395029067993164, | |
| "sampling/importance_sampling_ratio/mean": 0.3855500817298889, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.2965643405914307, | |
| "sampling/sampling_logp_difference/mean": 0.020408470183610916, | |
| "step": 9, | |
| "step_time": 363.9418078730232 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 6815.0, | |
| "completions/max_terminated_length": 6815.0, | |
| "completions/mean_length": 4436.71875, | |
| "completions/mean_terminated_length": 4436.71875, | |
| "completions/min_length": 1645.0, | |
| "completions/min_terminated_length": 1645.0, | |
| "entropy": 0.3489171755500138, | |
| "epoch": 0.04, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.28452208638191223, | |
| "kl": 0.0015779180102981627, | |
| "learning_rate": 2.906628198220621e-06, | |
| "loss": 0.1568, | |
| "num_tokens": 4563965.0, | |
| "reward": 0.22546875476837158, | |
| "reward_std": 0.2872272729873657, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.22546875476837158, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4715692698955536, | |
| "sampling/importance_sampling_ratio/max": 2.9534051418304443, | |
| "sampling/importance_sampling_ratio/mean": 0.4985596835613251, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.560643196105957, | |
| "sampling/sampling_logp_difference/mean": 0.020065899938344955, | |
| "step": 10, | |
| "step_time": 263.016338995003 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9583.0, | |
| "completions/max_terminated_length": 9583.0, | |
| "completions/mean_length": 6498.203125, | |
| "completions/mean_terminated_length": 6498.203125, | |
| "completions/min_length": 630.0, | |
| "completions/min_terminated_length": 630.0, | |
| "entropy": 0.34817238664254546, | |
| "epoch": 0.044, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.218714639544487, | |
| "kl": 0.0016383687889174325, | |
| "learning_rate": 2.878436717430346e-06, | |
| "loss": 0.187, | |
| "num_tokens": 5023626.0, | |
| "reward": 0.18468749523162842, | |
| "reward_std": 0.4014522433280945, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.18468749523162842, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4519691467285156, | |
| "sampling/importance_sampling_ratio/max": 2.0133447647094727, | |
| "sampling/importance_sampling_ratio/mean": 0.48143666982650757, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.304382801055908, | |
| "sampling/sampling_logp_difference/mean": 0.020289719104766846, | |
| "step": 11, | |
| "step_time": 358.22142862899636 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 10214.0, | |
| "completions/max_terminated_length": 10214.0, | |
| "completions/mean_length": 8234.421875, | |
| "completions/mean_terminated_length": 8234.421875, | |
| "completions/min_length": 2923.0, | |
| "completions/min_terminated_length": 2923.0, | |
| "entropy": 0.2907608295790851, | |
| "epoch": 0.048, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.26984360814094543, | |
| "kl": 0.0016117684463097248, | |
| "learning_rate": 2.846706809356113e-06, | |
| "loss": 0.0146, | |
| "num_tokens": 5600485.0, | |
| "reward": 0.11812499165534973, | |
| "reward_std": 0.35846489667892456, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.11812499910593033, | |
| "rewards/alfworld_rollout_reward_func/std": 0.37784212827682495, | |
| "sampling/importance_sampling_ratio/max": 2.9857699871063232, | |
| "sampling/importance_sampling_ratio/mean": 0.5836184024810791, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 57.58735656738281, | |
| "sampling/sampling_logp_difference/mean": 0.01956798881292343, | |
| "step": 12, | |
| "step_time": 354.4028672409913 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 5514.0, | |
| "completions/max_terminated_length": 5514.0, | |
| "completions/mean_length": 2816.578125, | |
| "completions/mean_terminated_length": 2816.578125, | |
| "completions/min_length": 899.0, | |
| "completions/min_terminated_length": 899.0, | |
| "entropy": 0.37038205191493034, | |
| "epoch": 0.052, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.21182094514369965, | |
| "kl": 0.0029761453515675385, | |
| "learning_rate": 2.811519924216873e-06, | |
| "loss": -0.0192, | |
| "num_tokens": 5812938.0, | |
| "reward": 0.7221874594688416, | |
| "reward_std": 0.4521871507167816, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.7221875190734863, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4576561152935028, | |
| "sampling/importance_sampling_ratio/max": 2.8400075435638428, | |
| "sampling/importance_sampling_ratio/mean": 0.3720712661743164, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 54.35240936279297, | |
| "sampling/sampling_logp_difference/mean": 0.04739231616258621, | |
| "step": 13, | |
| "step_time": 176.69907861396496 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 11718.0, | |
| "completions/max_terminated_length": 11718.0, | |
| "completions/mean_length": 8725.5, | |
| "completions/mean_terminated_length": 8725.5, | |
| "completions/min_length": 2386.0, | |
| "completions/min_terminated_length": 2386.0, | |
| "entropy": 0.34669697750359774, | |
| "epoch": 0.056, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2509883940219879, | |
| "kl": 0.002024810473812977, | |
| "learning_rate": 2.7729663862421267e-06, | |
| "loss": 0.3618, | |
| "num_tokens": 6423274.0, | |
| "reward": 0.1317187398672104, | |
| "reward_std": 0.27824532985687256, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.13171875476837158, | |
| "rewards/alfworld_rollout_reward_func/std": 0.3909270763397217, | |
| "sampling/importance_sampling_ratio/max": 2.5765702724456787, | |
| "sampling/importance_sampling_ratio/mean": 0.4724542498588562, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.3789246082305908, | |
| "sampling/sampling_logp_difference/mean": 0.021418465301394463, | |
| "step": 14, | |
| "step_time": 375.7027179340075 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9823.0, | |
| "completions/max_terminated_length": 9823.0, | |
| "completions/mean_length": 3761.203125, | |
| "completions/mean_terminated_length": 3761.203125, | |
| "completions/min_length": 710.0, | |
| "completions/min_terminated_length": 710.0, | |
| "entropy": 0.2707831831648946, | |
| "epoch": 0.06, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.18436089158058167, | |
| "kl": 0.0015840599007788114, | |
| "learning_rate": 2.731145161810915e-06, | |
| "loss": 0.0515, | |
| "num_tokens": 6700631.0, | |
| "reward": 0.6873437166213989, | |
| "reward_std": 0.3868533968925476, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.6873437166213989, | |
| "rewards/alfworld_rollout_reward_func/std": 0.48384782671928406, | |
| "sampling/importance_sampling_ratio/max": 2.213698625564575, | |
| "sampling/importance_sampling_ratio/mean": 0.573298454284668, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 56.33928680419922, | |
| "sampling/sampling_logp_difference/mean": 0.019625678658485413, | |
| "step": 15, | |
| "step_time": 284.1389054669853 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 6007.0, | |
| "completions/max_terminated_length": 6007.0, | |
| "completions/mean_length": 4786.0, | |
| "completions/mean_terminated_length": 4786.0, | |
| "completions/min_length": 1835.0, | |
| "completions/min_terminated_length": 1835.0, | |
| "entropy": 0.42686293832957745, | |
| "epoch": 0.064, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.5065476298332214, | |
| "kl": 0.004679446868976811, | |
| "learning_rate": 2.6861636054065477e-06, | |
| "loss": -0.0551, | |
| "num_tokens": 7042679.0, | |
| "reward": 0.23718750476837158, | |
| "reward_std": 0.3366313576698303, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.23718750476837158, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4672325551509857, | |
| "sampling/importance_sampling_ratio/max": 1.987215518951416, | |
| "sampling/importance_sampling_ratio/mean": 0.3825452923774719, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 48.08681869506836, | |
| "sampling/sampling_logp_difference/mean": 0.02753990702331066, | |
| "step": 16, | |
| "step_time": 253.53596857101365 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 13026.0, | |
| "completions/max_terminated_length": 13026.0, | |
| "completions/mean_length": 7801.28125, | |
| "completions/mean_terminated_length": 7801.28125, | |
| "completions/min_length": 996.0, | |
| "completions/min_terminated_length": 996.0, | |
| "entropy": 0.38106839125975966, | |
| "epoch": 0.068, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.13401441276073456, | |
| "kl": 0.0019643659798020963, | |
| "learning_rate": 2.6381371840391863e-06, | |
| "loss": 0.2576, | |
| "num_tokens": 7587913.0, | |
| "reward": 0.08687499165534973, | |
| "reward_std": 0.31602421402931213, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.08687499165534973, | |
| "rewards/alfworld_rollout_reward_func/std": 0.37281376123428345, | |
| "sampling/importance_sampling_ratio/max": 1.947364330291748, | |
| "sampling/importance_sampling_ratio/mean": 0.346585750579834, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 50.33766174316406, | |
| "sampling/sampling_logp_difference/mean": 0.024903230369091034, | |
| "step": 17, | |
| "step_time": 438.1667110319977 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 7932.0, | |
| "completions/max_terminated_length": 7932.0, | |
| "completions/mean_length": 5135.984375, | |
| "completions/mean_terminated_length": 5135.984375, | |
| "completions/min_length": 2158.0, | |
| "completions/min_terminated_length": 2158.0, | |
| "entropy": 0.38822565879672766, | |
| "epoch": 0.072, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2989322245121002, | |
| "kl": 0.001747572790918639, | |
| "learning_rate": 2.58718918084368e-06, | |
| "loss": -0.1124, | |
| "num_tokens": 7953544.0, | |
| "reward": 0.277031272649765, | |
| "reward_std": 0.3521837890148163, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.2770312428474426, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4784753620624542, | |
| "sampling/importance_sampling_ratio/max": 2.394239902496338, | |
| "sampling/importance_sampling_ratio/mean": 0.37263223528862, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.304655075073242, | |
| "sampling/sampling_logp_difference/mean": 0.02233094722032547, | |
| "step": 18, | |
| "step_time": 256.8168452990067 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 14106.0, | |
| "completions/max_terminated_length": 14106.0, | |
| "completions/mean_length": 10424.546875, | |
| "completions/mean_terminated_length": 10424.546875, | |
| "completions/min_length": 2464.0, | |
| "completions/min_terminated_length": 2464.0, | |
| "entropy": 0.3585456218570471, | |
| "epoch": 0.076, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.16573461890220642, | |
| "kl": 0.0020609867096936796, | |
| "learning_rate": 2.53345037861353e-06, | |
| "loss": 0.1829, | |
| "num_tokens": 8681771.0, | |
| "reward": 0.17765626311302185, | |
| "reward_std": 0.4618713855743408, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.17765626311302185, | |
| "rewards/alfworld_rollout_reward_func/std": 0.46532371640205383, | |
| "sampling/importance_sampling_ratio/max": 2.4118423461914062, | |
| "sampling/importance_sampling_ratio/mean": 0.23810628056526184, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 54.374839782714844, | |
| "sampling/sampling_logp_difference/mean": 0.026143547147512436, | |
| "step": 19, | |
| "step_time": 470.0736003620259 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 6909.0, | |
| "completions/max_terminated_length": 6909.0, | |
| "completions/mean_length": 3685.25, | |
| "completions/mean_terminated_length": 3685.25, | |
| "completions/min_length": 621.0, | |
| "completions/min_terminated_length": 621.0, | |
| "entropy": 0.34135088324546814, | |
| "epoch": 0.08, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.1747477948665619, | |
| "kl": 0.001954720755747985, | |
| "learning_rate": 2.477058724083334e-06, | |
| "loss": 0.1728, | |
| "num_tokens": 8952283.0, | |
| "reward": 0.4632812738418579, | |
| "reward_std": 0.3338242769241333, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.4632812738418579, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5396590828895569, | |
| "sampling/importance_sampling_ratio/max": 2.4462101459503174, | |
| "sampling/importance_sampling_ratio/mean": 0.5562223792076111, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 57.48880386352539, | |
| "sampling/sampling_logp_difference/mean": 0.025760911405086517, | |
| "step": 20, | |
| "step_time": 243.09781286600628 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9257.0, | |
| "completions/max_terminated_length": 9257.0, | |
| "completions/mean_length": 5892.0, | |
| "completions/mean_terminated_length": 5892.0, | |
| "completions/min_length": 2008.0, | |
| "completions/min_terminated_length": 2008.0, | |
| "entropy": 0.317400851752609, | |
| "epoch": 0.084, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2818284034729004, | |
| "kl": 0.0034878615733759943, | |
| "learning_rate": 2.4181589738214946e-06, | |
| "loss": -0.1166, | |
| "num_tokens": 9367611.0, | |
| "reward": 0.13109375536441803, | |
| "reward_std": 0.4177722930908203, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.13109375536441803, | |
| "rewards/alfworld_rollout_reward_func/std": 0.41239187121391296, | |
| "sampling/importance_sampling_ratio/max": 2.3060288429260254, | |
| "sampling/importance_sampling_ratio/mean": 0.40336811542510986, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.0395994186401367, | |
| "sampling/sampling_logp_difference/mean": 0.01938227005302906, | |
| "step": 21, | |
| "step_time": 322.3771183890203 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 5404.0, | |
| "completions/max_terminated_length": 5404.0, | |
| "completions/mean_length": 2908.671875, | |
| "completions/mean_terminated_length": 2908.671875, | |
| "completions/min_length": 504.0, | |
| "completions/min_terminated_length": 504.0, | |
| "entropy": 0.33527113823220134, | |
| "epoch": 0.088, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.28914618492126465, | |
| "kl": 0.0015201076712401118, | |
| "learning_rate": 2.3569023226421886e-06, | |
| "loss": 0.4174, | |
| "num_tokens": 9586310.0, | |
| "reward": 0.5940625071525574, | |
| "reward_std": 0.4888940155506134, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.5940624475479126, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5188471078872681, | |
| "sampling/importance_sampling_ratio/max": 2.96110200881958, | |
| "sampling/importance_sampling_ratio/mean": 0.5440493822097778, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.2356529235839844, | |
| "sampling/sampling_logp_difference/mean": 0.02087477594614029, | |
| "step": 22, | |
| "step_time": 220.2282471570361 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 11264.0, | |
| "completions/max_terminated_length": 11264.0, | |
| "completions/mean_length": 4416.765625, | |
| "completions/mean_terminated_length": 4416.765625, | |
| "completions/min_length": 548.0, | |
| "completions/min_terminated_length": 548.0, | |
| "entropy": 0.37216851580888033, | |
| "epoch": 0.092, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.21720609068870544, | |
| "kl": 0.0015780055618961342, | |
| "learning_rate": 2.2934460154904436e-06, | |
| "loss": -0.1388, | |
| "num_tokens": 9904855.0, | |
| "reward": 0.44140625, | |
| "reward_std": 0.5448532104492188, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.44140625, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5440409779548645, | |
| "sampling/importance_sampling_ratio/max": 2.4073739051818848, | |
| "sampling/importance_sampling_ratio/mean": 0.5275839567184448, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.4305299520492554, | |
| "sampling/sampling_logp_difference/mean": 0.020915336906909943, | |
| "step": 23, | |
| "step_time": 276.8828063480614 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 8603.0, | |
| "completions/max_terminated_length": 8603.0, | |
| "completions/mean_length": 5866.765625, | |
| "completions/mean_terminated_length": 5866.765625, | |
| "completions/min_length": 638.0, | |
| "completions/min_terminated_length": 638.0, | |
| "entropy": 0.38163668289780617, | |
| "epoch": 0.096, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.1996874064207077, | |
| "kl": 0.0022600124138989486, | |
| "learning_rate": 2.227952943796622e-06, | |
| "loss": 0.0203, | |
| "num_tokens": 10321992.0, | |
| "reward": 0.3020312488079071, | |
| "reward_std": 0.32399436831474304, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.3020312488079071, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4859981834888458, | |
| "sampling/importance_sampling_ratio/max": 2.0370097160339355, | |
| "sampling/importance_sampling_ratio/mean": 0.322752445936203, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.6053457260131836, | |
| "sampling/sampling_logp_difference/mean": 0.023195665329694748, | |
| "step": 24, | |
| "step_time": 326.98625825899944 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.015625, | |
| "completions/max_length": 11175.0, | |
| "completions/max_terminated_length": 11175.0, | |
| "completions/mean_length": 8415.28125, | |
| "completions/mean_terminated_length": 8421.2705078125, | |
| "completions/min_length": 3084.0, | |
| "completions/min_terminated_length": 3084.0, | |
| "entropy": 0.3829211392439902, | |
| "epoch": 0.1, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.15396606922149658, | |
| "kl": 0.0019763640011660755, | |
| "learning_rate": 2.160591227336452e-06, | |
| "loss": 0.0503, | |
| "num_tokens": 10909370.0, | |
| "reward": 0.10234375298023224, | |
| "reward_std": 0.32216501235961914, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.10234374552965164, | |
| "rewards/alfworld_rollout_reward_func/std": 0.3721825182437897, | |
| "sampling/importance_sampling_ratio/max": 2.063018560409546, | |
| "sampling/importance_sampling_ratio/mean": 0.26939719915390015, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 6.286721229553223, | |
| "sampling/sampling_logp_difference/mean": 0.021831313148140907, | |
| "step": 25, | |
| "step_time": 399.5593527499732 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 6961.0, | |
| "completions/max_terminated_length": 6961.0, | |
| "completions/mean_length": 4205.625, | |
| "completions/mean_terminated_length": 4205.625, | |
| "completions/min_length": 601.0, | |
| "completions/min_terminated_length": 601.0, | |
| "entropy": 0.3060068585909903, | |
| "epoch": 0.104, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.26218128204345703, | |
| "kl": 0.0015425982546730665, | |
| "learning_rate": 2.091533782669978e-06, | |
| "loss": 0.2487, | |
| "num_tokens": 11216162.0, | |
| "reward": 0.43156248331069946, | |
| "reward_std": 0.4275887608528137, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.43156248331069946, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5302896499633789, | |
| "sampling/importance_sampling_ratio/max": 2.158494472503662, | |
| "sampling/importance_sampling_ratio/mean": 0.5278518795967102, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.369166374206543, | |
| "sampling/sampling_logp_difference/mean": 0.019138336181640625, | |
| "step": 26, | |
| "step_time": 264.540892480014 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 6184.0, | |
| "completions/max_terminated_length": 6184.0, | |
| "completions/mean_length": 4046.3125, | |
| "completions/mean_terminated_length": 4046.3125, | |
| "completions/min_length": 1216.0, | |
| "completions/min_terminated_length": 1216.0, | |
| "entropy": 0.35598616674542427, | |
| "epoch": 0.108, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.568981409072876, | |
| "kl": 0.001767738020134857, | |
| "learning_rate": 2.0209578792672304e-06, | |
| "loss": 0.764, | |
| "num_tokens": 11507734.0, | |
| "reward": 0.2854687571525574, | |
| "reward_std": 0.27528271079063416, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.2854687571525574, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4897940456867218, | |
| "sampling/importance_sampling_ratio/max": 2.9929444789886475, | |
| "sampling/importance_sampling_ratio/mean": 0.5946164131164551, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 7.642696380615234, | |
| "sampling/sampling_logp_difference/mean": 0.021483708173036575, | |
| "step": 27, | |
| "step_time": 238.4292279400688 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 11353.0, | |
| "completions/max_terminated_length": 11353.0, | |
| "completions/mean_length": 6085.71875, | |
| "completions/mean_terminated_length": 6085.71875, | |
| "completions/min_length": 442.0, | |
| "completions/min_terminated_length": 442.0, | |
| "entropy": 0.34554752754047513, | |
| "epoch": 0.112, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.20419467985630035, | |
| "kl": 0.0022789983468101127, | |
| "learning_rate": 1.9490446844600373e-06, | |
| "loss": 0.0168, | |
| "num_tokens": 11941124.0, | |
| "reward": 0.3648437559604645, | |
| "reward_std": 0.3334695100784302, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.36484378576278687, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5057353973388672, | |
| "sampling/importance_sampling_ratio/max": 2.2758615016937256, | |
| "sampling/importance_sampling_ratio/mean": 0.6538110375404358, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.1021443605422974, | |
| "sampling/sampling_logp_difference/mean": 0.02113598957657814, | |
| "step": 28, | |
| "step_time": 396.3338410100696 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 11143.0, | |
| "completions/max_terminated_length": 11143.0, | |
| "completions/mean_length": 8093.265625, | |
| "completions/mean_terminated_length": 8093.265625, | |
| "completions/min_length": 2100.0, | |
| "completions/min_terminated_length": 2100.0, | |
| "entropy": 0.30172505462542176, | |
| "epoch": 0.116, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.23860569298267365, | |
| "kl": 0.0015739148620923515, | |
| "learning_rate": 1.875978798388081e-06, | |
| "loss": 0.1597, | |
| "num_tokens": 12510485.0, | |
| "reward": 0.2574999928474426, | |
| "reward_std": 0.27106666564941406, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.2574999928474426, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4856676161289215, | |
| "sampling/importance_sampling_ratio/max": 2.100985527038574, | |
| "sampling/importance_sampling_ratio/mean": 0.4662199318408966, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.5363552570343018, | |
| "sampling/sampling_logp_difference/mean": 0.0185923483222723, | |
| "step": 29, | |
| "step_time": 398.5051951840287 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 11407.0, | |
| "completions/max_terminated_length": 11407.0, | |
| "completions/mean_length": 7339.609375, | |
| "completions/mean_terminated_length": 7339.609375, | |
| "completions/min_length": 3248.0, | |
| "completions/min_terminated_length": 3248.0, | |
| "entropy": 0.30220934841781855, | |
| "epoch": 0.12, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2525518834590912, | |
| "kl": 0.0015497524218517356, | |
| "learning_rate": 1.8019477801329903e-06, | |
| "loss": 0.0827, | |
| "num_tokens": 13022268.0, | |
| "reward": 0.15546876192092896, | |
| "reward_std": 0.3895842730998993, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.15546876192092896, | |
| "rewards/alfworld_rollout_reward_func/std": 0.398253858089447, | |
| "sampling/importance_sampling_ratio/max": 2.3416731357574463, | |
| "sampling/importance_sampling_ratio/mean": 0.5051580667495728, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.9200248718261719, | |
| "sampling/sampling_logp_difference/mean": 0.01827174611389637, | |
| "step": 30, | |
| "step_time": 324.4753994320199 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 12941.0, | |
| "completions/max_terminated_length": 12941.0, | |
| "completions/mean_length": 8962.4375, | |
| "completions/mean_terminated_length": 8962.4375, | |
| "completions/min_length": 2241.0, | |
| "completions/min_terminated_length": 2241.0, | |
| "entropy": 0.31774380756542087, | |
| "epoch": 0.124, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.22262242436408997, | |
| "kl": 0.0023503736083512194, | |
| "learning_rate": 1.7271416662568652e-06, | |
| "loss": -0.2728, | |
| "num_tokens": 13653560.0, | |
| "reward": 0.3187499940395355, | |
| "reward_std": 0.46891656517982483, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.3187500238418579, | |
| "rewards/alfworld_rollout_reward_func/std": 0.49869832396507263, | |
| "sampling/importance_sampling_ratio/max": 2.161463975906372, | |
| "sampling/importance_sampling_ratio/mean": 0.36309731006622314, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.3483941555023193, | |
| "sampling/sampling_logp_difference/mean": 0.02068442292511463, | |
| "step": 31, | |
| "step_time": 430.4778445490374 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 7700.0, | |
| "completions/max_terminated_length": 7700.0, | |
| "completions/mean_length": 4740.34375, | |
| "completions/mean_terminated_length": 4740.34375, | |
| "completions/min_length": 751.0, | |
| "completions/min_terminated_length": 751.0, | |
| "entropy": 0.3920795237645507, | |
| "epoch": 0.128, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.3321034610271454, | |
| "kl": 0.0023466993261536118, | |
| "learning_rate": 1.6517524829811483e-06, | |
| "loss": 0.2419, | |
| "num_tokens": 13993934.0, | |
| "reward": 0.25953125953674316, | |
| "reward_std": 0.27672427892684937, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.25953125953674316, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4774650037288666, | |
| "sampling/importance_sampling_ratio/max": 2.515002965927124, | |
| "sampling/importance_sampling_ratio/mean": 0.5241488218307495, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.658240556716919, | |
| "sampling/sampling_logp_difference/mean": 0.023516817018389702, | |
| "step": 32, | |
| "step_time": 244.8374374089908 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9641.0, | |
| "completions/max_terminated_length": 9641.0, | |
| "completions/mean_length": 5815.34375, | |
| "completions/mean_terminated_length": 5815.34375, | |
| "completions/min_length": 1387.0, | |
| "completions/min_terminated_length": 1387.0, | |
| "entropy": 0.3165481761097908, | |
| "epoch": 0.132, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2114800661802292, | |
| "kl": 0.0015353642320405925, | |
| "learning_rate": 1.5759737532580691e-06, | |
| "loss": 0.3504, | |
| "num_tokens": 14409316.0, | |
| "reward": 0.27562499046325684, | |
| "reward_std": 0.48886263370513916, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.27562499046325684, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5106294751167297, | |
| "sampling/importance_sampling_ratio/max": 1.6374565362930298, | |
| "sampling/importance_sampling_ratio/mean": 0.5097586512565613, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.133441925048828, | |
| "sampling/sampling_logp_difference/mean": 0.01870131492614746, | |
| "step": 33, | |
| "step_time": 331.69739585091884 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 10654.0, | |
| "completions/max_terminated_length": 10654.0, | |
| "completions/mean_length": 7535.609375, | |
| "completions/mean_terminated_length": 7535.609375, | |
| "completions/min_length": 2351.0, | |
| "completions/min_terminated_length": 2351.0, | |
| "entropy": 0.33530362136662006, | |
| "epoch": 0.136, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.263776034116745, | |
| "kl": 0.0015736176555947168, | |
| "learning_rate": 1.5e-06, | |
| "loss": 0.1335, | |
| "num_tokens": 14939979.0, | |
| "reward": 0.18921875953674316, | |
| "reward_std": 0.44627484679222107, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.18921875953674316, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4631647765636444, | |
| "sampling/importance_sampling_ratio/max": 2.777794122695923, | |
| "sampling/importance_sampling_ratio/mean": 0.5164031982421875, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 55.2677001953125, | |
| "sampling/sampling_logp_difference/mean": 0.021179549396038055, | |
| "step": 34, | |
| "step_time": 358.0179667470511 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 10147.0, | |
| "completions/max_terminated_length": 10147.0, | |
| "completions/mean_length": 5750.734375, | |
| "completions/mean_terminated_length": 5750.734375, | |
| "completions/min_length": 2291.0, | |
| "completions/min_terminated_length": 2291.0, | |
| "entropy": 0.32300999434664845, | |
| "epoch": 0.14, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.19500228762626648, | |
| "kl": 0.0016171741226571612, | |
| "learning_rate": 1.4240262467419312e-06, | |
| "loss": 0.1209, | |
| "num_tokens": 15353146.0, | |
| "reward": 0.2917187511920929, | |
| "reward_std": 0.40162450075149536, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.2917187809944153, | |
| "rewards/alfworld_rollout_reward_func/std": 0.48963722586631775, | |
| "sampling/importance_sampling_ratio/max": 1.8978744745254517, | |
| "sampling/importance_sampling_ratio/mean": 0.44630223512649536, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 53.83928298950195, | |
| "sampling/sampling_logp_difference/mean": 0.021219002082943916, | |
| "step": 35, | |
| "step_time": 295.46302833205846 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 10553.0, | |
| "completions/max_terminated_length": 10553.0, | |
| "completions/mean_length": 7239.625, | |
| "completions/mean_terminated_length": 7239.625, | |
| "completions/min_length": 2030.0, | |
| "completions/min_terminated_length": 2030.0, | |
| "entropy": 0.31878431234508753, | |
| "epoch": 0.144, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.21166330575942993, | |
| "kl": 0.0018054660904454067, | |
| "learning_rate": 1.348247517018852e-06, | |
| "loss": 0.2801, | |
| "num_tokens": 15866850.0, | |
| "reward": 0.2901562452316284, | |
| "reward_std": 0.47076618671417236, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.2901562452316284, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5097291469573975, | |
| "sampling/importance_sampling_ratio/max": 2.524914503097534, | |
| "sampling/importance_sampling_ratio/mean": 0.5144106149673462, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.984360694885254, | |
| "sampling/sampling_logp_difference/mean": 0.020332355052232742, | |
| "step": 36, | |
| "step_time": 350.01136623701314 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 5913.0, | |
| "completions/max_terminated_length": 5913.0, | |
| "completions/mean_length": 4780.84375, | |
| "completions/mean_terminated_length": 4780.84375, | |
| "completions/min_length": 579.0, | |
| "completions/min_terminated_length": 579.0, | |
| "entropy": 0.36987574864178896, | |
| "epoch": 0.148, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.4266482889652252, | |
| "kl": 0.002036058562225662, | |
| "learning_rate": 1.2728583337431355e-06, | |
| "loss": -0.1097, | |
| "num_tokens": 16207096.0, | |
| "reward": 0.07484374940395355, | |
| "reward_std": 0.2557505965232849, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.07484374940395355, | |
| "rewards/alfworld_rollout_reward_func/std": 0.34834298491477966, | |
| "sampling/importance_sampling_ratio/max": 2.214252233505249, | |
| "sampling/importance_sampling_ratio/mean": 0.5370051860809326, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.3917155265808105, | |
| "sampling/sampling_logp_difference/mean": 0.02142323926091194, | |
| "step": 37, | |
| "step_time": 247.20503475998703 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 6967.0, | |
| "completions/max_terminated_length": 6967.0, | |
| "completions/mean_length": 4629.875, | |
| "completions/mean_terminated_length": 4629.875, | |
| "completions/min_length": 999.0, | |
| "completions/min_terminated_length": 999.0, | |
| "entropy": 0.3442773185670376, | |
| "epoch": 0.152, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.31260499358177185, | |
| "kl": 0.0025437602398596937, | |
| "learning_rate": 1.1980522198670096e-06, | |
| "loss": 0.0576, | |
| "num_tokens": 16541488.0, | |
| "reward": 0.44578126072883606, | |
| "reward_std": 0.5412331819534302, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.44578126072883606, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5354713201522827, | |
| "sampling/importance_sampling_ratio/max": 2.806023597717285, | |
| "sampling/importance_sampling_ratio/mean": 0.517693281173706, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.0272293090820312, | |
| "sampling/sampling_logp_difference/mean": 0.020449213683605194, | |
| "step": 38, | |
| "step_time": 244.387926218973 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 13688.0, | |
| "completions/max_terminated_length": 13688.0, | |
| "completions/mean_length": 8324.359375, | |
| "completions/mean_terminated_length": 8324.359375, | |
| "completions/min_length": 3154.0, | |
| "completions/min_terminated_length": 3154.0, | |
| "entropy": 0.3674225895665586, | |
| "epoch": 0.156, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.36787140369415283, | |
| "kl": 0.0025533495027048048, | |
| "learning_rate": 1.1240212016119191e-06, | |
| "loss": 0.3241, | |
| "num_tokens": 17125223.0, | |
| "reward": 0.09515625238418579, | |
| "reward_std": 0.3095976710319519, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.09515625238418579, | |
| "rewards/alfworld_rollout_reward_func/std": 0.365721732378006, | |
| "sampling/importance_sampling_ratio/max": 2.9900293350219727, | |
| "sampling/importance_sampling_ratio/mean": 0.3794456422328949, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 46.29445266723633, | |
| "sampling/sampling_logp_difference/mean": 0.027921315282583237, | |
| "step": 39, | |
| "step_time": 463.61854148590646 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 6432.0, | |
| "completions/max_terminated_length": 6432.0, | |
| "completions/mean_length": 4293.5625, | |
| "completions/mean_terminated_length": 4293.5625, | |
| "completions/min_length": 552.0, | |
| "completions/min_terminated_length": 552.0, | |
| "entropy": 0.38137416914105415, | |
| "epoch": 0.16, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.4678648114204407, | |
| "kl": 0.001712255700113019, | |
| "learning_rate": 1.050955315539963e-06, | |
| "loss": -0.2122, | |
| "num_tokens": 17436395.0, | |
| "reward": 0.31953126192092896, | |
| "reward_std": 0.3283727169036865, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.31953126192092896, | |
| "rewards/alfworld_rollout_reward_func/std": 0.510062038898468, | |
| "sampling/importance_sampling_ratio/max": 2.993114948272705, | |
| "sampling/importance_sampling_ratio/mean": 0.6158585548400879, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 48.348148345947266, | |
| "sampling/sampling_logp_difference/mean": 0.023211471736431122, | |
| "step": 40, | |
| "step_time": 256.8888109070249 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 10452.0, | |
| "completions/max_terminated_length": 10452.0, | |
| "completions/mean_length": 7975.96875, | |
| "completions/mean_terminated_length": 7975.96875, | |
| "completions/min_length": 1643.0, | |
| "completions/min_terminated_length": 1643.0, | |
| "entropy": 0.3266838057897985, | |
| "epoch": 0.164, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2977025508880615, | |
| "kl": 0.0014619105677411426, | |
| "learning_rate": 9.790421207327699e-07, | |
| "loss": 0.0016, | |
| "num_tokens": 17994953.0, | |
| "reward": 0.12093749642372131, | |
| "reward_std": 0.26076871156692505, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.12093749642372131, | |
| "rewards/alfworld_rollout_reward_func/std": 0.37234872579574585, | |
| "sampling/importance_sampling_ratio/max": 2.8854687213897705, | |
| "sampling/importance_sampling_ratio/mean": 0.5439540147781372, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.1864867210388184, | |
| "sampling/sampling_logp_difference/mean": 0.01889047771692276, | |
| "step": 41, | |
| "step_time": 425.9135665790236 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 10013.0, | |
| "completions/max_terminated_length": 10013.0, | |
| "completions/mean_length": 5837.21875, | |
| "completions/mean_terminated_length": 5837.21875, | |
| "completions/min_length": 812.0, | |
| "completions/min_terminated_length": 812.0, | |
| "entropy": 0.3718068078160286, | |
| "epoch": 0.168, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.27091944217681885, | |
| "kl": 0.0016485519881825894, | |
| "learning_rate": 9.084662173300225e-07, | |
| "loss": -0.1063, | |
| "num_tokens": 18409943.0, | |
| "reward": 0.32343751192092896, | |
| "reward_std": 0.5115345120429993, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.32343751192092896, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5089429616928101, | |
| "sampling/importance_sampling_ratio/max": 2.761387586593628, | |
| "sampling/importance_sampling_ratio/mean": 0.5661083459854126, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.2433140277862549, | |
| "sampling/sampling_logp_difference/mean": 0.020694797858595848, | |
| "step": 42, | |
| "step_time": 350.4842578980315 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9433.0, | |
| "completions/max_terminated_length": 9433.0, | |
| "completions/mean_length": 7310.640625, | |
| "completions/mean_terminated_length": 7310.640625, | |
| "completions/min_length": 2144.0, | |
| "completions/min_terminated_length": 2144.0, | |
| "entropy": 0.3453982062637806, | |
| "epoch": 0.172, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.3637518882751465, | |
| "kl": 0.002296481339726597, | |
| "learning_rate": 8.394087726635485e-07, | |
| "loss": -0.1358, | |
| "num_tokens": 18925600.0, | |
| "reward": 0.2357812523841858, | |
| "reward_std": 0.44348853826522827, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.2357812523841858, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4680924117565155, | |
| "sampling/importance_sampling_ratio/max": 2.9779891967773438, | |
| "sampling/importance_sampling_ratio/mean": 0.5728085041046143, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.363124132156372, | |
| "sampling/sampling_logp_difference/mean": 0.021319666877388954, | |
| "step": 43, | |
| "step_time": 331.5770332479151 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 8984.0, | |
| "completions/max_terminated_length": 8984.0, | |
| "completions/mean_length": 6977.109375, | |
| "completions/mean_terminated_length": 6977.109375, | |
| "completions/min_length": 3575.0, | |
| "completions/min_terminated_length": 3575.0, | |
| "entropy": 0.36568512208759785, | |
| "epoch": 0.176, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.20232941210269928, | |
| "kl": 0.0016346415723091923, | |
| "learning_rate": 7.720470562033787e-07, | |
| "loss": -0.0301, | |
| "num_tokens": 19414471.0, | |
| "reward": 0.04781249910593033, | |
| "reward_std": 0.2049415409564972, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.04781249910593033, | |
| "rewards/alfworld_rollout_reward_func/std": 0.2761898934841156, | |
| "sampling/importance_sampling_ratio/max": 2.573187828063965, | |
| "sampling/importance_sampling_ratio/mean": 0.40001991391181946, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.278357982635498, | |
| "sampling/sampling_logp_difference/mean": 0.02066657692193985, | |
| "step": 44, | |
| "step_time": 335.4528454130341 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 15070.0, | |
| "completions/max_terminated_length": 15070.0, | |
| "completions/mean_length": 8883.3125, | |
| "completions/mean_terminated_length": 8883.3125, | |
| "completions/min_length": 4182.0, | |
| "completions/min_terminated_length": 4182.0, | |
| "entropy": 0.31764299469068646, | |
| "epoch": 0.18, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.1879281848669052, | |
| "kl": 0.0016870554245542735, | |
| "learning_rate": 7.065539845095568e-07, | |
| "loss": 0.205, | |
| "num_tokens": 20034107.0, | |
| "reward": 0.10062500834465027, | |
| "reward_std": 0.3260836601257324, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.10062500089406967, | |
| "rewards/alfworld_rollout_reward_func/std": 0.3762761652469635, | |
| "sampling/importance_sampling_ratio/max": 2.5225887298583984, | |
| "sampling/importance_sampling_ratio/mean": 0.5068827271461487, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.621875524520874, | |
| "sampling/sampling_logp_difference/mean": 0.019450519233942032, | |
| "step": 45, | |
| "step_time": 501.09887950192206 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9683.0, | |
| "completions/max_terminated_length": 9683.0, | |
| "completions/mean_length": 6441.609375, | |
| "completions/mean_terminated_length": 6441.609375, | |
| "completions/min_length": 2937.0, | |
| "completions/min_terminated_length": 2937.0, | |
| "entropy": 0.3502948391251266, | |
| "epoch": 0.184, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.1859540343284607, | |
| "kl": 0.0018878296577895526, | |
| "learning_rate": 6.430976773578113e-07, | |
| "loss": 0.0278, | |
| "num_tokens": 20485858.0, | |
| "reward": 0.0010937508195638657, | |
| "reward_std": 0.23143835365772247, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.0010937508195638657, | |
| "rewards/alfworld_rollout_reward_func/std": 0.2593160569667816, | |
| "sampling/importance_sampling_ratio/max": 2.3244879245758057, | |
| "sampling/importance_sampling_ratio/mean": 0.34531980752944946, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.0072948932647705, | |
| "sampling/sampling_logp_difference/mean": 0.020865384489297867, | |
| "step": 46, | |
| "step_time": 366.95768191007664 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 5392.0, | |
| "completions/max_terminated_length": 5392.0, | |
| "completions/mean_length": 3591.0625, | |
| "completions/mean_terminated_length": 3591.0625, | |
| "completions/min_length": 517.0, | |
| "completions/min_terminated_length": 517.0, | |
| "entropy": 0.3532397085800767, | |
| "epoch": 0.188, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.26849785447120667, | |
| "kl": 0.0017250235250685364, | |
| "learning_rate": 5.818410261785057e-07, | |
| "loss": 0.1737, | |
| "num_tokens": 20748838.0, | |
| "reward": 0.41593751311302185, | |
| "reward_std": 0.5181204080581665, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.41593748331069946, | |
| "rewards/alfworld_rollout_reward_func/std": 0.5215962529182434, | |
| "sampling/importance_sampling_ratio/max": 2.7028884887695312, | |
| "sampling/importance_sampling_ratio/mean": 0.5614103078842163, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.718446731567383, | |
| "sampling/sampling_logp_difference/mean": 0.021135177463293076, | |
| "step": 47, | |
| "step_time": 194.4773051920347 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 5232.0, | |
| "completions/max_terminated_length": 5232.0, | |
| "completions/mean_length": 2274.453125, | |
| "completions/mean_terminated_length": 2274.453125, | |
| "completions/min_length": 498.0, | |
| "completions/min_terminated_length": 498.0, | |
| "entropy": 0.3223302629776299, | |
| "epoch": 0.192, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.20919622480869293, | |
| "kl": 0.001539721906738123, | |
| "learning_rate": 5.22941275916667e-07, | |
| "loss": 0.3358, | |
| "num_tokens": 20925091.0, | |
| "reward": 0.6996874809265137, | |
| "reward_std": 0.4721481502056122, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.6996875405311584, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4819766581058502, | |
| "sampling/importance_sampling_ratio/max": 2.5296757221221924, | |
| "sampling/importance_sampling_ratio/mean": 0.6417877674102783, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.344658613204956, | |
| "sampling/sampling_logp_difference/mean": 0.019674479961395264, | |
| "step": 48, | |
| "step_time": 168.07057143107522 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 6957.0, | |
| "completions/max_terminated_length": 6957.0, | |
| "completions/mean_length": 5750.375, | |
| "completions/mean_terminated_length": 5750.375, | |
| "completions/min_length": 3849.0, | |
| "completions/min_terminated_length": 3849.0, | |
| "entropy": 0.31901704519987106, | |
| "epoch": 0.196, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.28379565477371216, | |
| "kl": 0.0021226651278993813, | |
| "learning_rate": 4.6654962138647007e-07, | |
| "loss": 0.0489, | |
| "num_tokens": 21330971.0, | |
| "reward": 0.02437499538064003, | |
| "reward_std": 0.186305969953537, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.02437499724328518, | |
| "rewards/alfworld_rollout_reward_func/std": 0.2512942850589752, | |
| "sampling/importance_sampling_ratio/max": 2.9523849487304688, | |
| "sampling/importance_sampling_ratio/mean": 0.4695657193660736, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 53.46038055419922, | |
| "sampling/sampling_logp_difference/mean": 0.019282279536128044, | |
| "step": 49, | |
| "step_time": 266.79126943598385 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9327.0, | |
| "completions/max_terminated_length": 9327.0, | |
| "completions/mean_length": 5568.34375, | |
| "completions/mean_terminated_length": 5568.34375, | |
| "completions/min_length": 544.0, | |
| "completions/min_terminated_length": 544.0, | |
| "entropy": 0.3215307700447738, | |
| "epoch": 0.2, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.319042831659317, | |
| "kl": 0.0018373212151345797, | |
| "learning_rate": 4.1281081915632036e-07, | |
| "loss": 0.3955, | |
| "num_tokens": 21725489.0, | |
| "reward": 0.22624999284744263, | |
| "reward_std": 0.40438205003738403, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.22624999284744263, | |
| "rewards/alfworld_rollout_reward_func/std": 0.4729659855365753, | |
| "sampling/importance_sampling_ratio/max": 2.5651261806488037, | |
| "sampling/importance_sampling_ratio/mean": 0.5718743801116943, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.8857594728469849, | |
| "sampling/sampling_logp_difference/mean": 0.019382823258638382, | |
| "step": 50, | |
| "step_time": 338.1545116149937 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 11776.0, | |
| "completions/max_terminated_length": 11776.0, | |
| "completions/mean_length": 8995.296875, | |
| "completions/mean_terminated_length": 8995.296875, | |
| "completions/min_length": 2626.0, | |
| "completions/min_terminated_length": 2626.0, | |
| "entropy": 0.35890309792011976, | |
| "epoch": 0.204, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.16795441508293152, | |
| "kl": 0.002312422751856502, | |
| "learning_rate": 3.618628159608137e-07, | |
| "loss": 0.1271, | |
| "num_tokens": 22352804.0, | |
| "reward": 0.06656250357627869, | |
| "reward_std": 0.3205060660839081, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.06656249612569809, | |
| "rewards/alfworld_rollout_reward_func/std": 0.32436618208885193, | |
| "sampling/importance_sampling_ratio/max": 2.878086566925049, | |
| "sampling/importance_sampling_ratio/mean": 0.4224788546562195, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.016578197479248, | |
| "sampling/sampling_logp_difference/mean": 0.022725660353899002, | |
| "step": 51, | |
| "step_time": 381.1358364249754 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 6659.0, | |
| "completions/max_terminated_length": 6659.0, | |
| "completions/mean_length": 4975.1875, | |
| "completions/mean_terminated_length": 4975.1875, | |
| "completions/min_length": 2482.0, | |
| "completions/min_terminated_length": 2482.0, | |
| "entropy": 0.3571747145615518, | |
| "epoch": 0.208, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.19965247809886932, | |
| "kl": 0.001949786972545553, | |
| "learning_rate": 3.1383639459345236e-07, | |
| "loss": -0.0391, | |
| "num_tokens": 22705936.0, | |
| "reward": 0.10843749344348907, | |
| "reward_std": 0.2537260055541992, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.10843749344348907, | |
| "rewards/alfworld_rollout_reward_func/std": 0.3537294268608093, | |
| "sampling/importance_sampling_ratio/max": 2.9369618892669678, | |
| "sampling/importance_sampling_ratio/mean": 0.33827975392341614, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 4.243539333343506, | |
| "sampling/sampling_logp_difference/mean": 0.020905133336782455, | |
| "step": 52, | |
| "step_time": 256.6296185880783 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9607.0, | |
| "completions/max_terminated_length": 9607.0, | |
| "completions/mean_length": 5973.78125, | |
| "completions/mean_terminated_length": 5973.78125, | |
| "completions/min_length": 1849.0, | |
| "completions/min_terminated_length": 1849.0, | |
| "entropy": 0.3052480248734355, | |
| "epoch": 0.212, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.27389758825302124, | |
| "kl": 0.0034342061553616077, | |
| "learning_rate": 2.688548381890859e-07, | |
| "loss": 0.0311, | |
| "num_tokens": 23130466.0, | |
| "reward": 0.19437500834465027, | |
| "reward_std": 0.4533996284008026, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.19437500834465027, | |
| "rewards/alfworld_rollout_reward_func/std": 0.45724809169769287, | |
| "sampling/importance_sampling_ratio/max": 2.6141345500946045, | |
| "sampling/importance_sampling_ratio/mean": 0.41237422823905945, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 3.5475544929504395, | |
| "sampling/sampling_logp_difference/mean": 0.020619163289666176, | |
| "step": 53, | |
| "step_time": 327.58865647405037 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9492.0, | |
| "completions/max_terminated_length": 9492.0, | |
| "completions/mean_length": 6306.28125, | |
| "completions/mean_terminated_length": 6306.28125, | |
| "completions/min_length": 1341.0, | |
| "completions/min_terminated_length": 1341.0, | |
| "entropy": 0.3394437450915575, | |
| "epoch": 0.216, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.19828611612319946, | |
| "kl": 0.0016102603804029059, | |
| "learning_rate": 2.2703361375787346e-07, | |
| "loss": 0.3936, | |
| "num_tokens": 23577428.0, | |
| "reward": 0.16374999284744263, | |
| "reward_std": 0.4402611553668976, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.16374999284744263, | |
| "rewards/alfworld_rollout_reward_func/std": 0.44003063440322876, | |
| "sampling/importance_sampling_ratio/max": 2.7105014324188232, | |
| "sampling/importance_sampling_ratio/mean": 0.5148346424102783, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.6567630767822266, | |
| "sampling/sampling_logp_difference/mean": 0.019922319799661636, | |
| "step": 54, | |
| "step_time": 330.50816768797813 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 5940.0, | |
| "completions/max_terminated_length": 5940.0, | |
| "completions/mean_length": 3369.0, | |
| "completions/mean_terminated_length": 3369.0, | |
| "completions/min_length": 498.0, | |
| "completions/min_terminated_length": 498.0, | |
| "entropy": 0.36095575196668506, | |
| "epoch": 0.22, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.3427432179450989, | |
| "kl": 0.0036608128975785803, | |
| "learning_rate": 1.8848007578312686e-07, | |
| "loss": 0.4046, | |
| "num_tokens": 23825684.0, | |
| "reward": 0.5928125381469727, | |
| "reward_std": 0.5227715969085693, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.5928125381469727, | |
| "rewards/alfworld_rollout_reward_func/std": 0.511535108089447, | |
| "sampling/importance_sampling_ratio/max": 2.6692662239074707, | |
| "sampling/importance_sampling_ratio/mean": 0.5836673378944397, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 4.313566207885742, | |
| "sampling/sampling_logp_difference/mean": 0.021921399980783463, | |
| "step": 55, | |
| "step_time": 209.76535455693374 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 7371.0, | |
| "completions/max_terminated_length": 7371.0, | |
| "completions/mean_length": 5376.625, | |
| "completions/mean_terminated_length": 5376.625, | |
| "completions/min_length": 992.0, | |
| "completions/min_terminated_length": 992.0, | |
| "entropy": 0.335981338750571, | |
| "epoch": 0.224, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2279304414987564, | |
| "kl": 0.001375907209876459, | |
| "learning_rate": 1.5329319064388763e-07, | |
| "loss": 0.3754, | |
| "num_tokens": 24206844.0, | |
| "reward": 0.08656249940395355, | |
| "reward_std": 0.34256887435913086, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.08656249940395355, | |
| "rewards/alfworld_rollout_reward_func/std": 0.34272125363349915, | |
| "sampling/importance_sampling_ratio/max": 2.9141616821289062, | |
| "sampling/importance_sampling_ratio/mean": 0.5004754066467285, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 2.75748872756958, | |
| "sampling/sampling_logp_difference/mean": 0.018594371154904366, | |
| "step": 56, | |
| "step_time": 289.04723902978003 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 10048.0, | |
| "completions/max_terminated_length": 10048.0, | |
| "completions/mean_length": 8312.671875, | |
| "completions/mean_terminated_length": 8312.671875, | |
| "completions/min_length": 2348.0, | |
| "completions/min_terminated_length": 2348.0, | |
| "entropy": 0.3852003728970885, | |
| "epoch": 0.228, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.22347784042358398, | |
| "kl": 0.0018401142369839363, | |
| "learning_rate": 1.215632825696541e-07, | |
| "loss": 0.2461, | |
| "num_tokens": 24789191.0, | |
| "reward": 0.11437500268220901, | |
| "reward_std": 0.3912443518638611, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.11437500268220901, | |
| "rewards/alfworld_rollout_reward_func/std": 0.403114378452301, | |
| "sampling/importance_sampling_ratio/max": 2.5475013256073, | |
| "sampling/importance_sampling_ratio/mean": 0.45833104848861694, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.6942596435546875, | |
| "sampling/sampling_logp_difference/mean": 0.022461190819740295, | |
| "step": 57, | |
| "step_time": 388.85974016188993 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 6720.0, | |
| "completions/max_terminated_length": 6720.0, | |
| "completions/mean_length": 5543.359375, | |
| "completions/mean_terminated_length": 5543.359375, | |
| "completions/min_length": 2344.0, | |
| "completions/min_terminated_length": 2344.0, | |
| "entropy": 0.32870627008378506, | |
| "epoch": 0.232, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.4650394320487976, | |
| "kl": 0.001981915997021133, | |
| "learning_rate": 9.337180177937954e-08, | |
| "loss": 0.7758, | |
| "num_tokens": 25182878.0, | |
| "reward": 0.08546875417232513, | |
| "reward_std": 0.2914150357246399, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.08546875417232513, | |
| "rewards/alfworld_rollout_reward_func/std": 0.34556111693382263, | |
| "sampling/importance_sampling_ratio/max": 2.719653367996216, | |
| "sampling/importance_sampling_ratio/mean": 0.5657415390014648, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 5.367229461669922, | |
| "sampling/sampling_logp_difference/mean": 0.0186595655977726, | |
| "step": 58, | |
| "step_time": 247.89911623601802 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 11541.0, | |
| "completions/max_terminated_length": 11541.0, | |
| "completions/mean_length": 5263.265625, | |
| "completions/mean_terminated_length": 5263.265625, | |
| "completions/min_length": 1418.0, | |
| "completions/min_terminated_length": 1418.0, | |
| "entropy": 0.361502249725163, | |
| "epoch": 0.236, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2132435142993927, | |
| "kl": 0.0016867622362042312, | |
| "learning_rate": 6.879111539992677e-08, | |
| "loss": 0.0722, | |
| "num_tokens": 25557423.0, | |
| "reward": 0.2345312535762787, | |
| "reward_std": 0.4755549430847168, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.2345312535762787, | |
| "rewards/alfworld_rollout_reward_func/std": 0.486049622297287, | |
| "sampling/importance_sampling_ratio/max": 2.7570416927337646, | |
| "sampling/importance_sampling_ratio/mean": 0.4710940420627594, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.1428396701812744, | |
| "sampling/sampling_logp_difference/mean": 0.021931620314717293, | |
| "step": 59, | |
| "step_time": 334.8811481029552 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.0, | |
| "completions/max_length": 9499.0, | |
| "completions/max_terminated_length": 9499.0, | |
| "completions/mean_length": 5835.5625, | |
| "completions/mean_terminated_length": 5835.5625, | |
| "completions/min_length": 1881.0, | |
| "completions/min_terminated_length": 1881.0, | |
| "entropy": 0.28580303583294153, | |
| "epoch": 0.24, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2324402630329132, | |
| "kl": 0.0014885484506521607, | |
| "learning_rate": 4.788432170069373e-08, | |
| "loss": 0.0045, | |
| "num_tokens": 25969907.0, | |
| "reward": 0.2704687714576721, | |
| "reward_std": 0.48869654536247253, | |
| "rewards/alfworld_rollout_reward_func/mean": 0.27046874165534973, | |
| "rewards/alfworld_rollout_reward_func/std": 0.482570618391037, | |
| "sampling/importance_sampling_ratio/max": 2.4066574573516846, | |
| "sampling/importance_sampling_ratio/mean": 0.4883001446723938, | |
| "sampling/importance_sampling_ratio/min": 0.0, | |
| "sampling/sampling_logp_difference/max": 1.3303287029266357, | |
| "sampling/sampling_logp_difference/mean": 0.016779117286205292, | |
| "step": 60, | |
| "step_time": 319.99202060801326 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 64, | |
| "num_input_tokens_seen": 25969907, | |
| "num_train_epochs": 1, | |
| "save_steps": 20, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |