| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.04631578947368421, |
| "eval_steps": 500, |
| "global_step": 44, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 39.0, |
| "completions/max_terminated_length": 39.0, |
| "completions/mean_length": 38.0, |
| "completions/mean_terminated_length": 38.0, |
| "completions/min_length": 37.0, |
| "completions/min_terminated_length": 37.0, |
| "entropy": 0.4614375829696655, |
| "epoch": 0.0010526315789473684, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 23.0, |
| "kl": 0.002436438575387001, |
| "learning_rate": 0.0, |
| "loss": 0.1571, |
| "num_tokens": 8138.0, |
| "reward": -0.10500000417232513, |
| "reward_std": 0.021213199943304062, |
| "rewards/alfworld_rollout_reward_func/mean": -0.10500000417232513, |
| "rewards/alfworld_rollout_reward_func/std": 0.021213199943304062, |
| "sampling/importance_sampling_ratio/max": 0.9498974680900574, |
| "sampling/importance_sampling_ratio/mean": 0.7463880777359009, |
| "sampling/importance_sampling_ratio/min": 0.5428786873817444, |
| "sampling/sampling_logp_difference/max": 0.2791634798049927, |
| "sampling/sampling_logp_difference/mean": 0.02213391289114952, |
| "step": 1, |
| "step_time": 21.471763861 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 54.0, |
| "completions/max_terminated_length": 54.0, |
| "completions/mean_length": 49.5, |
| "completions/mean_terminated_length": 49.5, |
| "completions/min_length": 45.0, |
| "completions/min_terminated_length": 45.0, |
| "entropy": 0.6891850829124451, |
| "epoch": 0.002105263157894737, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 13.4375, |
| "kl": 0.0023963379207998514, |
| "learning_rate": 2.0000000000000002e-07, |
| "loss": -0.0442, |
| "num_tokens": 16245.0, |
| "reward": -0.08500000089406967, |
| "reward_std": 0.007071071770042181, |
| "rewards/alfworld_rollout_reward_func/mean": -0.08500000089406967, |
| "rewards/alfworld_rollout_reward_func/std": 0.007071071770042181, |
| "sampling/importance_sampling_ratio/max": 0.6407822966575623, |
| "sampling/importance_sampling_ratio/mean": 0.5291908383369446, |
| "sampling/importance_sampling_ratio/min": 0.4175994098186493, |
| "sampling/sampling_logp_difference/max": 0.26871776580810547, |
| "sampling/sampling_logp_difference/mean": 0.0313858687877655, |
| "step": 2, |
| "step_time": 19.41586231600013 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 35.0, |
| "completions/max_terminated_length": 35.0, |
| "completions/mean_length": 32.5, |
| "completions/mean_terminated_length": 32.5, |
| "completions/min_length": 30.0, |
| "completions/min_terminated_length": 30.0, |
| "entropy": 0.2943471372127533, |
| "epoch": 0.003157894736842105, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 9.0625, |
| "kl": 0.00032700574956834316, |
| "learning_rate": 4.0000000000000003e-07, |
| "loss": -0.0682, |
| "num_tokens": 24330.0, |
| "reward": -0.07000000029802322, |
| "reward_std": 0.05656854063272476, |
| "rewards/alfworld_rollout_reward_func/mean": -0.07000000029802322, |
| "rewards/alfworld_rollout_reward_func/std": 0.05656854063272476, |
| "sampling/importance_sampling_ratio/max": 1.0007613897323608, |
| "sampling/importance_sampling_ratio/mean": 0.8396192193031311, |
| "sampling/importance_sampling_ratio/min": 0.6784770488739014, |
| "sampling/sampling_logp_difference/max": 0.22780990600585938, |
| "sampling/sampling_logp_difference/mean": 0.013574070297181606, |
| "step": 3, |
| "step_time": 17.228165518000196 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 45.0, |
| "completions/max_terminated_length": 45.0, |
| "completions/mean_length": 37.5, |
| "completions/mean_terminated_length": 37.5, |
| "completions/min_length": 30.0, |
| "completions/min_terminated_length": 30.0, |
| "entropy": 0.4852295517921448, |
| "epoch": 0.004210526315789474, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 22.75, |
| "kl": 0.0009841235587373376, |
| "learning_rate": 6.000000000000001e-07, |
| "loss": 0.0262, |
| "num_tokens": 32171.0, |
| "reward": -0.04500000178813934, |
| "reward_std": 0.0353553406894207, |
| "rewards/alfworld_rollout_reward_func/mean": -0.04500000178813934, |
| "rewards/alfworld_rollout_reward_func/std": 0.0353553369641304, |
| "sampling/importance_sampling_ratio/max": 0.9375013709068298, |
| "sampling/importance_sampling_ratio/mean": 0.7503011226654053, |
| "sampling/importance_sampling_ratio/min": 0.5631008148193359, |
| "sampling/sampling_logp_difference/max": 0.28363165259361267, |
| "sampling/sampling_logp_difference/mean": 0.02271696925163269, |
| "step": 4, |
| "step_time": 17.49860281700012 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 51.0, |
| "completions/max_terminated_length": 51.0, |
| "completions/mean_length": 27.5, |
| "completions/mean_terminated_length": 27.5, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 0.5301113724708557, |
| "epoch": 0.005263157894736842, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 28.125, |
| "kl": 0.0012831644853577018, |
| "learning_rate": 8.000000000000001e-07, |
| "loss": 0.4587, |
| "num_tokens": 40277.0, |
| "reward": -0.004999999888241291, |
| "reward_std": 0.007071067579090595, |
| "rewards/alfworld_rollout_reward_func/mean": -0.004999999888241291, |
| "rewards/alfworld_rollout_reward_func/std": 0.007071067579090595, |
| "sampling/importance_sampling_ratio/max": 1.0000011920928955, |
| "sampling/importance_sampling_ratio/mean": 0.893904447555542, |
| "sampling/importance_sampling_ratio/min": 0.7878076434135437, |
| "sampling/sampling_logp_difference/max": 0.21004503965377808, |
| "sampling/sampling_logp_difference/mean": 0.02911142073571682, |
| "step": 5, |
| "step_time": 12.491938435000066 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 26.0, |
| "completions/max_terminated_length": 26.0, |
| "completions/mean_length": 16.5, |
| "completions/mean_terminated_length": 16.5, |
| "completions/min_length": 7.0, |
| "completions/min_terminated_length": 7.0, |
| "entropy": 0.1027420163154602, |
| "epoch": 0.00631578947368421, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 18.875, |
| "kl": 4.723216625279747e-05, |
| "learning_rate": 1.0000000000000002e-06, |
| "loss": -0.397, |
| "num_tokens": 48155.0, |
| "reward": -0.029999999329447746, |
| "reward_std": 0.01414213515818119, |
| "rewards/alfworld_rollout_reward_func/mean": -0.029999999329447746, |
| "rewards/alfworld_rollout_reward_func/std": 0.01414213515818119, |
| "sampling/importance_sampling_ratio/max": 1.0009899139404297, |
| "sampling/importance_sampling_ratio/mean": 0.9940400719642639, |
| "sampling/importance_sampling_ratio/min": 0.9870902299880981, |
| "sampling/sampling_logp_difference/max": 0.15823769569396973, |
| "sampling/sampling_logp_difference/mean": 0.009266156703233719, |
| "step": 6, |
| "step_time": 23.18857599299986 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 37.0, |
| "completions/max_terminated_length": 37.0, |
| "completions/mean_length": 24.0, |
| "completions/mean_terminated_length": 24.0, |
| "completions/min_length": 11.0, |
| "completions/min_terminated_length": 11.0, |
| "entropy": 0.45579978823661804, |
| "epoch": 0.007368421052631579, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 46.75, |
| "kl": 0.0014351233839988708, |
| "learning_rate": 1.2000000000000002e-06, |
| "loss": 0.607, |
| "num_tokens": 55983.0, |
| "reward": -0.019999999552965164, |
| "reward_std": 0.02828427031636238, |
| "rewards/alfworld_rollout_reward_func/mean": -0.019999999552965164, |
| "rewards/alfworld_rollout_reward_func/std": 0.02828427031636238, |
| "sampling/importance_sampling_ratio/max": 1.4146320819854736, |
| "sampling/importance_sampling_ratio/mean": 1.2070283889770508, |
| "sampling/importance_sampling_ratio/min": 0.9994246959686279, |
| "sampling/sampling_logp_difference/max": 0.1907503604888916, |
| "sampling/sampling_logp_difference/mean": 0.028974320739507675, |
| "step": 7, |
| "step_time": 18.246992883999837 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 47.0, |
| "completions/max_terminated_length": 47.0, |
| "completions/mean_length": 28.0, |
| "completions/mean_terminated_length": 28.0, |
| "completions/min_length": 9.0, |
| "completions/min_terminated_length": 9.0, |
| "entropy": 0.5107156038284302, |
| "epoch": 0.008421052631578947, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 58.75, |
| "kl": 0.0012022192822769284, |
| "learning_rate": 1.4000000000000001e-06, |
| "loss": -0.6365, |
| "num_tokens": 64025.0, |
| "reward": -0.08500000089406967, |
| "reward_std": 0.007071071770042181, |
| "rewards/alfworld_rollout_reward_func/mean": -0.08500000089406967, |
| "rewards/alfworld_rollout_reward_func/std": 0.007071071770042181, |
| "sampling/importance_sampling_ratio/max": 1.279162883758545, |
| "sampling/importance_sampling_ratio/mean": 1.1397144794464111, |
| "sampling/importance_sampling_ratio/min": 1.0002660751342773, |
| "sampling/sampling_logp_difference/max": 0.5383334159851074, |
| "sampling/sampling_logp_difference/mean": 0.034098681062459946, |
| "step": 8, |
| "step_time": 14.293913408000208 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 56.0, |
| "completions/max_terminated_length": 56.0, |
| "completions/mean_length": 44.5, |
| "completions/mean_terminated_length": 44.5, |
| "completions/min_length": 33.0, |
| "completions/min_terminated_length": 33.0, |
| "entropy": 0.4052967131137848, |
| "epoch": 0.009473684210526316, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 16.625, |
| "kl": 0.0016700377454981208, |
| "learning_rate": 1.6000000000000001e-06, |
| "loss": -0.2022, |
| "num_tokens": 71652.0, |
| "reward": 0.4650000035762787, |
| "reward_std": 0.6576092839241028, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4650000035762787, |
| "rewards/alfworld_rollout_reward_func/std": 0.6576092839241028, |
| "sampling/importance_sampling_ratio/max": 0.7853229641914368, |
| "sampling/importance_sampling_ratio/mean": 0.6733799576759338, |
| "sampling/importance_sampling_ratio/min": 0.5614369511604309, |
| "sampling/sampling_logp_difference/max": 0.27764952182769775, |
| "sampling/sampling_logp_difference/mean": 0.01565438136458397, |
| "step": 9, |
| "step_time": 16.68387536299997 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 35.0, |
| "completions/max_terminated_length": 35.0, |
| "completions/mean_length": 30.5, |
| "completions/mean_terminated_length": 30.5, |
| "completions/min_length": 26.0, |
| "completions/min_terminated_length": 26.0, |
| "entropy": 0.33000221848487854, |
| "epoch": 0.010526315789473684, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.013671875, |
| "kl": 0.0005048786988481879, |
| "learning_rate": 1.8000000000000001e-06, |
| "loss": 0.0, |
| "num_tokens": 79470.0, |
| "reward": -0.029999999329447746, |
| "reward_std": 0.0, |
| "rewards/alfworld_rollout_reward_func/mean": -0.029999999329447746, |
| "rewards/alfworld_rollout_reward_func/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 0.9141315221786499, |
| "sampling/importance_sampling_ratio/mean": 0.7841682434082031, |
| "sampling/importance_sampling_ratio/min": 0.6542050242424011, |
| "sampling/sampling_logp_difference/max": 0.24437618255615234, |
| "sampling/sampling_logp_difference/mean": 0.01810554973781109, |
| "step": 10, |
| "step_time": 17.949971448000042 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 45.0, |
| "completions/max_terminated_length": 45.0, |
| "completions/mean_length": 41.5, |
| "completions/mean_terminated_length": 41.5, |
| "completions/min_length": 38.0, |
| "completions/min_terminated_length": 38.0, |
| "entropy": 0.5000134706497192, |
| "epoch": 0.011578947368421053, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 14.125, |
| "kl": 0.001739501953125, |
| "learning_rate": 2.0000000000000003e-06, |
| "loss": 0.0346, |
| "num_tokens": 87509.0, |
| "reward": -0.044999998062849045, |
| "reward_std": 0.04949747398495674, |
| "rewards/alfworld_rollout_reward_func/mean": -0.044999998062849045, |
| "rewards/alfworld_rollout_reward_func/std": 0.04949747398495674, |
| "sampling/importance_sampling_ratio/max": 0.6740682721138, |
| "sampling/importance_sampling_ratio/mean": 0.6668994426727295, |
| "sampling/importance_sampling_ratio/min": 0.6597306132316589, |
| "sampling/sampling_logp_difference/max": 0.35384368896484375, |
| "sampling/sampling_logp_difference/mean": 0.021013759076595306, |
| "step": 11, |
| "step_time": 15.37352415700002 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 45.0, |
| "completions/max_terminated_length": 45.0, |
| "completions/mean_length": 29.0, |
| "completions/mean_terminated_length": 29.0, |
| "completions/min_length": 13.0, |
| "completions/min_terminated_length": 13.0, |
| "entropy": 0.37302640080451965, |
| "epoch": 0.01263157894736842, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 39.0, |
| "kl": 0.0027003493160009384, |
| "learning_rate": 2.2e-06, |
| "loss": -0.5275, |
| "num_tokens": 95262.0, |
| "reward": -0.06000000238418579, |
| "reward_std": 0.04242640733718872, |
| "rewards/alfworld_rollout_reward_func/mean": -0.06000000238418579, |
| "rewards/alfworld_rollout_reward_func/std": 0.04242641106247902, |
| "sampling/importance_sampling_ratio/max": 1.2358118295669556, |
| "sampling/importance_sampling_ratio/mean": 1.088789463043213, |
| "sampling/importance_sampling_ratio/min": 0.9417669773101807, |
| "sampling/sampling_logp_difference/max": 0.3406403064727783, |
| "sampling/sampling_logp_difference/mean": 0.028803091496229172, |
| "step": 12, |
| "step_time": 20.290826388999903 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 23.0, |
| "completions/max_terminated_length": 23.0, |
| "completions/mean_length": 15.0, |
| "completions/mean_terminated_length": 15.0, |
| "completions/min_length": 7.0, |
| "completions/min_terminated_length": 7.0, |
| "entropy": 0.07970059663057327, |
| "epoch": 0.01368421052631579, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 25.125, |
| "kl": 0.0003672480524983257, |
| "learning_rate": 2.4000000000000003e-06, |
| "loss": -0.3785, |
| "num_tokens": 102993.0, |
| "reward": -0.004999999888241291, |
| "reward_std": 0.007071067579090595, |
| "rewards/alfworld_rollout_reward_func/mean": -0.004999999888241291, |
| "rewards/alfworld_rollout_reward_func/std": 0.007071067579090595, |
| "sampling/importance_sampling_ratio/max": 1.012474536895752, |
| "sampling/importance_sampling_ratio/mean": 1.006240963935852, |
| "sampling/importance_sampling_ratio/min": 1.0000073909759521, |
| "sampling/sampling_logp_difference/max": 0.011289931833744049, |
| "sampling/sampling_logp_difference/mean": 0.00044049008283764124, |
| "step": 13, |
| "step_time": 18.20201583400012 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 36.0, |
| "completions/max_terminated_length": 36.0, |
| "completions/mean_length": 24.5, |
| "completions/mean_terminated_length": 24.5, |
| "completions/min_length": 13.0, |
| "completions/min_terminated_length": 13.0, |
| "entropy": 0.21557021141052246, |
| "epoch": 0.014736842105263158, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 28.375, |
| "kl": 0.005430301651358604, |
| "learning_rate": 2.6e-06, |
| "loss": -0.3004, |
| "num_tokens": 111031.0, |
| "reward": 0.42500001192092896, |
| "reward_std": 0.7141778469085693, |
| "rewards/alfworld_rollout_reward_func/mean": 0.42500001192092896, |
| "rewards/alfworld_rollout_reward_func/std": 0.7141778469085693, |
| "sampling/importance_sampling_ratio/max": 0.8329165577888489, |
| "sampling/importance_sampling_ratio/mean": 0.7687587738037109, |
| "sampling/importance_sampling_ratio/min": 0.7046010494232178, |
| "sampling/sampling_logp_difference/max": 0.3576321601867676, |
| "sampling/sampling_logp_difference/mean": 0.016674496233463287, |
| "step": 14, |
| "step_time": 11.272263890000204 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 23.0, |
| "completions/max_terminated_length": 23.0, |
| "completions/mean_length": 15.0, |
| "completions/mean_terminated_length": 15.0, |
| "completions/min_length": 7.0, |
| "completions/min_terminated_length": 7.0, |
| "entropy": 0.18071487545967102, |
| "epoch": 0.015789473684210527, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 16.625, |
| "kl": 0.00020178158592898399, |
| "learning_rate": 2.8000000000000003e-06, |
| "loss": 0.4877, |
| "num_tokens": 118867.0, |
| "reward": -0.004999999888241291, |
| "reward_std": 0.007071067579090595, |
| "rewards/alfworld_rollout_reward_func/mean": -0.004999999888241291, |
| "rewards/alfworld_rollout_reward_func/std": 0.007071067579090595, |
| "sampling/importance_sampling_ratio/max": 1.216752290725708, |
| "sampling/importance_sampling_ratio/mean": 1.1083970069885254, |
| "sampling/importance_sampling_ratio/min": 1.0000418424606323, |
| "sampling/sampling_logp_difference/max": 0.11035466194152832, |
| "sampling/sampling_logp_difference/mean": 0.007579161319881678, |
| "step": 15, |
| "step_time": 20.160193882000158 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12.0, |
| "completions/max_terminated_length": 12.0, |
| "completions/mean_length": 11.5, |
| "completions/mean_terminated_length": 11.5, |
| "completions/min_length": 11.0, |
| "completions/min_terminated_length": 11.0, |
| "entropy": 0.07483004778623581, |
| "epoch": 0.016842105263157894, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 29.625, |
| "kl": 0.00041633585351519287, |
| "learning_rate": 3e-06, |
| "loss": -0.0959, |
| "num_tokens": 126279.0, |
| "reward": 0.429999977350235, |
| "reward_std": 0.7495331764221191, |
| "rewards/alfworld_rollout_reward_func/mean": 0.429999977350235, |
| "rewards/alfworld_rollout_reward_func/std": 0.7495331764221191, |
| "sampling/importance_sampling_ratio/max": 1.1467688083648682, |
| "sampling/importance_sampling_ratio/mean": 1.057037591934204, |
| "sampling/importance_sampling_ratio/min": 0.9673064351081848, |
| "sampling/sampling_logp_difference/max": 0.11966276168823242, |
| "sampling/sampling_logp_difference/mean": 0.007402568124234676, |
| "step": 16, |
| "step_time": 10.745571063999932 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 36.0, |
| "completions/max_terminated_length": 36.0, |
| "completions/mean_length": 33.0, |
| "completions/mean_terminated_length": 33.0, |
| "completions/min_length": 30.0, |
| "completions/min_terminated_length": 30.0, |
| "entropy": 0.5801213383674622, |
| "epoch": 0.017894736842105262, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 30.125, |
| "kl": 0.001962649170309305, |
| "learning_rate": 3.2000000000000003e-06, |
| "loss": 0.1349, |
| "num_tokens": 134227.0, |
| "reward": 0.4749999940395355, |
| "reward_std": 0.6858935952186584, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4749999940395355, |
| "rewards/alfworld_rollout_reward_func/std": 0.6858935952186584, |
| "sampling/importance_sampling_ratio/max": 1.1694220304489136, |
| "sampling/importance_sampling_ratio/mean": 1.0765215158462524, |
| "sampling/importance_sampling_ratio/min": 0.9836210608482361, |
| "sampling/sampling_logp_difference/max": 0.17440319061279297, |
| "sampling/sampling_logp_difference/mean": 0.022032134234905243, |
| "step": 17, |
| "step_time": 12.921286662000057 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 41.0, |
| "completions/max_terminated_length": 41.0, |
| "completions/mean_length": 39.5, |
| "completions/mean_terminated_length": 39.5, |
| "completions/min_length": 38.0, |
| "completions/min_terminated_length": 38.0, |
| "entropy": 0.48242872953414917, |
| "epoch": 0.018947368421052633, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 19.5, |
| "kl": 0.0010035536251962185, |
| "learning_rate": 3.4000000000000005e-06, |
| "loss": -0.0587, |
| "num_tokens": 142157.0, |
| "reward": -0.04999999701976776, |
| "reward_std": 0.04242640733718872, |
| "rewards/alfworld_rollout_reward_func/mean": -0.04999999701976776, |
| "rewards/alfworld_rollout_reward_func/std": 0.04242640733718872, |
| "sampling/importance_sampling_ratio/max": 1.0037174224853516, |
| "sampling/importance_sampling_ratio/mean": 0.9568833112716675, |
| "sampling/importance_sampling_ratio/min": 0.9100492596626282, |
| "sampling/sampling_logp_difference/max": 0.17302274703979492, |
| "sampling/sampling_logp_difference/mean": 0.02212933637201786, |
| "step": 18, |
| "step_time": 23.997690939999984 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 46.0, |
| "completions/max_terminated_length": 46.0, |
| "completions/mean_length": 44.0, |
| "completions/mean_terminated_length": 44.0, |
| "completions/min_length": 42.0, |
| "completions/min_terminated_length": 42.0, |
| "entropy": 0.44313955307006836, |
| "epoch": 0.02, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 19.5, |
| "kl": 0.0014339183690026402, |
| "learning_rate": 3.6000000000000003e-06, |
| "loss": -0.1594, |
| "num_tokens": 150213.0, |
| "reward": -0.05999999865889549, |
| "reward_std": 0.0707106813788414, |
| "rewards/alfworld_rollout_reward_func/mean": -0.05999999865889549, |
| "rewards/alfworld_rollout_reward_func/std": 0.0707106739282608, |
| "sampling/importance_sampling_ratio/max": 1.1801482439041138, |
| "sampling/importance_sampling_ratio/mean": 0.9998883008956909, |
| "sampling/importance_sampling_ratio/min": 0.8196282982826233, |
| "sampling/sampling_logp_difference/max": 0.21845340728759766, |
| "sampling/sampling_logp_difference/mean": 0.022122323513031006, |
| "step": 19, |
| "step_time": 15.372385936 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 39.0, |
| "completions/max_terminated_length": 39.0, |
| "completions/mean_length": 36.5, |
| "completions/mean_terminated_length": 36.5, |
| "completions/min_length": 34.0, |
| "completions/min_terminated_length": 34.0, |
| "entropy": 0.26607948541641235, |
| "epoch": 0.021052631578947368, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 21.125, |
| "kl": 0.0008539336849935353, |
| "learning_rate": 3.8000000000000005e-06, |
| "loss": -0.0343, |
| "num_tokens": 157583.0, |
| "reward": 0.45500001311302185, |
| "reward_std": 0.742462158203125, |
| "rewards/alfworld_rollout_reward_func/mean": 0.45500001311302185, |
| "rewards/alfworld_rollout_reward_func/std": 0.7424620985984802, |
| "sampling/importance_sampling_ratio/max": 0.9992303848266602, |
| "sampling/importance_sampling_ratio/mean": 0.8897002339363098, |
| "sampling/importance_sampling_ratio/min": 0.7801700830459595, |
| "sampling/sampling_logp_difference/max": 0.2817434072494507, |
| "sampling/sampling_logp_difference/mean": 0.01816781423985958, |
| "step": 20, |
| "step_time": 21.228662558999986 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 26.0, |
| "completions/max_terminated_length": 26.0, |
| "completions/mean_length": 19.5, |
| "completions/mean_terminated_length": 19.5, |
| "completions/min_length": 13.0, |
| "completions/min_terminated_length": 13.0, |
| "entropy": 0.28872889280319214, |
| "epoch": 0.022105263157894735, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 18.75, |
| "kl": 0.00026875274488702416, |
| "learning_rate": 4.000000000000001e-06, |
| "loss": 0.191, |
| "num_tokens": 162725.0, |
| "reward": 0.9650000333786011, |
| "reward_std": 0.04949747025966644, |
| "rewards/alfworld_rollout_reward_func/mean": 0.9650000333786011, |
| "rewards/alfworld_rollout_reward_func/std": 0.04949747025966644, |
| "sampling/importance_sampling_ratio/max": 0.9999924898147583, |
| "sampling/importance_sampling_ratio/mean": 0.9529882669448853, |
| "sampling/importance_sampling_ratio/min": 0.905984103679657, |
| "sampling/sampling_logp_difference/max": 0.13952183723449707, |
| "sampling/sampling_logp_difference/mean": 0.012355787679553032, |
| "step": 21, |
| "step_time": 8.206261441999914 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 41.0, |
| "completions/max_terminated_length": 41.0, |
| "completions/mean_length": 27.0, |
| "completions/mean_terminated_length": 27.0, |
| "completions/min_length": 13.0, |
| "completions/min_terminated_length": 13.0, |
| "entropy": 0.4186505675315857, |
| "epoch": 0.023157894736842106, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 70.0, |
| "kl": 0.0038073172327131033, |
| "learning_rate": 4.2000000000000004e-06, |
| "loss": 0.6479, |
| "num_tokens": 170762.0, |
| "reward": 0.4650000035762787, |
| "reward_std": 0.7566042542457581, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4650000035762787, |
| "rewards/alfworld_rollout_reward_func/std": 0.7566042542457581, |
| "sampling/importance_sampling_ratio/max": 1.66000235080719, |
| "sampling/importance_sampling_ratio/mean": 1.544608235359192, |
| "sampling/importance_sampling_ratio/min": 1.4292141199111938, |
| "sampling/sampling_logp_difference/max": 0.23494195938110352, |
| "sampling/sampling_logp_difference/mean": 0.03184577822685242, |
| "step": 22, |
| "step_time": 17.66746830000011 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 44.0, |
| "completions/max_terminated_length": 44.0, |
| "completions/mean_length": 34.5, |
| "completions/mean_terminated_length": 34.5, |
| "completions/min_length": 25.0, |
| "completions/min_terminated_length": 25.0, |
| "entropy": 0.3005879819393158, |
| "epoch": 0.024210526315789474, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 20.5, |
| "kl": 0.02588764950633049, |
| "learning_rate": 4.4e-06, |
| "loss": 0.1134, |
| "num_tokens": 178722.0, |
| "reward": 0.48000001907348633, |
| "reward_std": 0.6929646730422974, |
| "rewards/alfworld_rollout_reward_func/mean": 0.48000001907348633, |
| "rewards/alfworld_rollout_reward_func/std": 0.6929646730422974, |
| "sampling/importance_sampling_ratio/max": 0.9999445676803589, |
| "sampling/importance_sampling_ratio/mean": 0.6585712432861328, |
| "sampling/importance_sampling_ratio/min": 0.3171979784965515, |
| "sampling/sampling_logp_difference/max": 1.331534504890442, |
| "sampling/sampling_logp_difference/mean": 0.03551221266388893, |
| "step": 23, |
| "step_time": 14.36746498499997 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 35.0, |
| "completions/max_terminated_length": 35.0, |
| "completions/mean_length": 23.5, |
| "completions/mean_terminated_length": 23.5, |
| "completions/min_length": 12.0, |
| "completions/min_terminated_length": 12.0, |
| "entropy": 0.3918939232826233, |
| "epoch": 0.02526315789473684, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 43.25, |
| "kl": 0.0027948389761149883, |
| "learning_rate": 4.600000000000001e-06, |
| "loss": 0.5853, |
| "num_tokens": 185227.0, |
| "reward": 0.4650000035762787, |
| "reward_std": 0.7424620389938354, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4650000035762787, |
| "rewards/alfworld_rollout_reward_func/std": 0.7424620389938354, |
| "sampling/importance_sampling_ratio/max": 1.4053035974502563, |
| "sampling/importance_sampling_ratio/mean": 1.130873203277588, |
| "sampling/importance_sampling_ratio/min": 0.8564428091049194, |
| "sampling/sampling_logp_difference/max": 0.16124820709228516, |
| "sampling/sampling_logp_difference/mean": 0.016462432220578194, |
| "step": 24, |
| "step_time": 11.062920657999939 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 31.0, |
| "completions/max_terminated_length": 31.0, |
| "completions/mean_length": 29.0, |
| "completions/mean_terminated_length": 29.0, |
| "completions/min_length": 27.0, |
| "completions/min_terminated_length": 27.0, |
| "entropy": 0.2529997229576111, |
| "epoch": 0.02631578947368421, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.018310546875, |
| "kl": 0.0004380304308142513, |
| "learning_rate": 4.800000000000001e-06, |
| "loss": 0.0, |
| "num_tokens": 193142.0, |
| "reward": -0.019999999552965164, |
| "reward_std": 0.0, |
| "rewards/alfworld_rollout_reward_func/mean": -0.019999999552965164, |
| "rewards/alfworld_rollout_reward_func/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.0052870512008667, |
| "sampling/importance_sampling_ratio/mean": 0.8423590660095215, |
| "sampling/importance_sampling_ratio/min": 0.6794310212135315, |
| "sampling/sampling_logp_difference/max": 0.18489933013916016, |
| "sampling/sampling_logp_difference/mean": 0.014295091852545738, |
| "step": 25, |
| "step_time": 17.847312069000054 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 49.0, |
| "completions/max_terminated_length": 49.0, |
| "completions/mean_length": 31.0, |
| "completions/mean_terminated_length": 31.0, |
| "completions/min_length": 13.0, |
| "completions/min_terminated_length": 13.0, |
| "entropy": 0.3845043182373047, |
| "epoch": 0.02736842105263158, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 50.0, |
| "kl": 0.002633685013279319, |
| "learning_rate": 5e-06, |
| "loss": -0.273, |
| "num_tokens": 200881.0, |
| "reward": -0.10999999940395355, |
| "reward_std": 0.014142133295536041, |
| "rewards/alfworld_rollout_reward_func/mean": -0.10999999940395355, |
| "rewards/alfworld_rollout_reward_func/std": 0.014142133295536041, |
| "sampling/importance_sampling_ratio/max": 1.0188355445861816, |
| "sampling/importance_sampling_ratio/mean": 0.8905454874038696, |
| "sampling/importance_sampling_ratio/min": 0.7622554302215576, |
| "sampling/sampling_logp_difference/max": 0.29382169246673584, |
| "sampling/sampling_logp_difference/mean": 0.03841578587889671, |
| "step": 26, |
| "step_time": 22.61463799199987 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 26.0, |
| "completions/max_terminated_length": 26.0, |
| "completions/mean_length": 17.0, |
| "completions/mean_terminated_length": 17.0, |
| "completions/min_length": 8.0, |
| "completions/min_terminated_length": 8.0, |
| "entropy": 0.32032543420791626, |
| "epoch": 0.028421052631578948, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 33.75, |
| "kl": 0.0024931079242378473, |
| "learning_rate": 5.2e-06, |
| "loss": 0.3737, |
| "num_tokens": 208999.0, |
| "reward": 0.4950000047683716, |
| "reward_std": 0.7141778469085693, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4950000047683716, |
| "rewards/alfworld_rollout_reward_func/std": 0.7141778469085693, |
| "sampling/importance_sampling_ratio/max": 1.1281967163085938, |
| "sampling/importance_sampling_ratio/mean": 1.083227276802063, |
| "sampling/importance_sampling_ratio/min": 1.0382578372955322, |
| "sampling/sampling_logp_difference/max": 0.17356586456298828, |
| "sampling/sampling_logp_difference/mean": 0.017684968188405037, |
| "step": 27, |
| "step_time": 11.613979460999872 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 14.0, |
| "completions/max_terminated_length": 14.0, |
| "completions/mean_length": 11.0, |
| "completions/mean_terminated_length": 11.0, |
| "completions/min_length": 8.0, |
| "completions/min_terminated_length": 8.0, |
| "entropy": 0.05829498916864395, |
| "epoch": 0.029473684210526315, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 32.25, |
| "kl": 0.0006212808657437563, |
| "learning_rate": 5.400000000000001e-06, |
| "loss": -0.2479, |
| "num_tokens": 214157.0, |
| "reward": 0.4650000035762787, |
| "reward_std": 0.7424620389938354, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4650000035762787, |
| "rewards/alfworld_rollout_reward_func/std": 0.7424620389938354, |
| "sampling/importance_sampling_ratio/max": 0.9999884366989136, |
| "sampling/importance_sampling_ratio/mean": 0.8929275274276733, |
| "sampling/importance_sampling_ratio/min": 0.7858666181564331, |
| "sampling/sampling_logp_difference/max": 0.21855998039245605, |
| "sampling/sampling_logp_difference/mean": 0.01126509066671133, |
| "step": 28, |
| "step_time": 9.151633475999915 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 37.0, |
| "completions/max_terminated_length": 37.0, |
| "completions/mean_length": 36.5, |
| "completions/mean_terminated_length": 36.5, |
| "completions/min_length": 36.0, |
| "completions/min_terminated_length": 36.0, |
| "entropy": 0.46773216128349304, |
| "epoch": 0.030526315789473683, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 86.0, |
| "kl": 0.015533313155174255, |
| "learning_rate": 5.600000000000001e-06, |
| "loss": -0.5057, |
| "num_tokens": 222111.0, |
| "reward": -0.03500000014901161, |
| "reward_std": 0.02121320366859436, |
| "rewards/alfworld_rollout_reward_func/mean": -0.03500000014901161, |
| "rewards/alfworld_rollout_reward_func/std": 0.02121320366859436, |
| "sampling/importance_sampling_ratio/max": 2.2050247192382812, |
| "sampling/importance_sampling_ratio/mean": 1.4661935567855835, |
| "sampling/importance_sampling_ratio/min": 0.727362334728241, |
| "sampling/sampling_logp_difference/max": 1.248981237411499, |
| "sampling/sampling_logp_difference/mean": 0.042725156992673874, |
| "step": 29, |
| "step_time": 20.633877447999794 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 44.0, |
| "completions/max_terminated_length": 44.0, |
| "completions/mean_length": 28.0, |
| "completions/mean_terminated_length": 28.0, |
| "completions/min_length": 12.0, |
| "completions/min_terminated_length": 12.0, |
| "entropy": 0.18183040618896484, |
| "epoch": 0.031578947368421054, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 29.625, |
| "kl": 0.0012305846903473139, |
| "learning_rate": 5.8e-06, |
| "loss": 0.3004, |
| "num_tokens": 229439.0, |
| "reward": 0.44999998807907104, |
| "reward_std": 0.7778174877166748, |
| "rewards/alfworld_rollout_reward_func/mean": 0.44999998807907104, |
| "rewards/alfworld_rollout_reward_func/std": 0.7778174877166748, |
| "sampling/importance_sampling_ratio/max": 0.9991921782493591, |
| "sampling/importance_sampling_ratio/mean": 0.9062168598175049, |
| "sampling/importance_sampling_ratio/min": 0.8132414817810059, |
| "sampling/sampling_logp_difference/max": 0.1453406810760498, |
| "sampling/sampling_logp_difference/mean": 0.007052628789097071, |
| "step": 30, |
| "step_time": 15.40591485699997 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 56.0, |
| "completions/max_terminated_length": 56.0, |
| "completions/mean_length": 49.5, |
| "completions/mean_terminated_length": 49.5, |
| "completions/min_length": 43.0, |
| "completions/min_terminated_length": 43.0, |
| "entropy": 0.3357900381088257, |
| "epoch": 0.03263157894736842, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 17.875, |
| "kl": 0.004221746232360601, |
| "learning_rate": 6e-06, |
| "loss": -0.0807, |
| "num_tokens": 237387.0, |
| "reward": -0.05999999865889549, |
| "reward_std": 0.02828427031636238, |
| "rewards/alfworld_rollout_reward_func/mean": -0.05999999865889549, |
| "rewards/alfworld_rollout_reward_func/std": 0.02828427031636238, |
| "sampling/importance_sampling_ratio/max": 0.8237836956977844, |
| "sampling/importance_sampling_ratio/mean": 0.8164201974868774, |
| "sampling/importance_sampling_ratio/min": 0.8090566992759705, |
| "sampling/sampling_logp_difference/max": 0.2831292152404785, |
| "sampling/sampling_logp_difference/mean": 0.024411508813500404, |
| "step": 31, |
| "step_time": 19.268974757000024 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 33.0, |
| "completions/max_terminated_length": 33.0, |
| "completions/mean_length": 32.0, |
| "completions/mean_terminated_length": 32.0, |
| "completions/min_length": 31.0, |
| "completions/min_terminated_length": 31.0, |
| "entropy": 0.2340346872806549, |
| "epoch": 0.03368421052631579, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.049560546875, |
| "kl": 0.0027104970067739487, |
| "learning_rate": 6.200000000000001e-06, |
| "loss": 0.0, |
| "num_tokens": 245449.0, |
| "reward": -0.05000000074505806, |
| "reward_std": 0.0, |
| "rewards/alfworld_rollout_reward_func/mean": -0.05000000074505806, |
| "rewards/alfworld_rollout_reward_func/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 2.6312272548675537, |
| "sampling/importance_sampling_ratio/mean": 1.798392653465271, |
| "sampling/importance_sampling_ratio/min": 0.9655579924583435, |
| "sampling/sampling_logp_difference/max": 0.5748621225357056, |
| "sampling/sampling_logp_difference/mean": 0.029400669038295746, |
| "step": 32, |
| "step_time": 24.658211100000017 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 31.0, |
| "completions/max_terminated_length": 31.0, |
| "completions/mean_length": 20.5, |
| "completions/mean_terminated_length": 20.5, |
| "completions/min_length": 10.0, |
| "completions/min_terminated_length": 10.0, |
| "entropy": 0.050181157886981964, |
| "epoch": 0.034736842105263156, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 19.125, |
| "kl": 0.003014294197782874, |
| "learning_rate": 6.4000000000000006e-06, |
| "loss": -0.3172, |
| "num_tokens": 253476.0, |
| "reward": -0.014999999664723873, |
| "reward_std": 0.007071067579090595, |
| "rewards/alfworld_rollout_reward_func/mean": -0.014999999664723873, |
| "rewards/alfworld_rollout_reward_func/std": 0.007071067579090595, |
| "sampling/importance_sampling_ratio/max": 1.2256042957305908, |
| "sampling/importance_sampling_ratio/mean": 1.1113841533660889, |
| "sampling/importance_sampling_ratio/min": 0.9971638917922974, |
| "sampling/sampling_logp_difference/max": 0.23240363597869873, |
| "sampling/sampling_logp_difference/mean": 0.006807921454310417, |
| "step": 33, |
| "step_time": 12.894395297000074 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 13.0, |
| "completions/max_terminated_length": 13.0, |
| "completions/mean_length": 12.0, |
| "completions/mean_terminated_length": 12.0, |
| "completions/min_length": 11.0, |
| "completions/min_terminated_length": 11.0, |
| "entropy": 8.056841761572286e-05, |
| "epoch": 0.035789473684210524, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.031494140625, |
| "kl": 9.934107758624577e-09, |
| "learning_rate": 6.600000000000001e-06, |
| "loss": -0.0589, |
| "num_tokens": 259774.0, |
| "reward": 0.49000000953674316, |
| "reward_std": 0.7212488651275635, |
| "rewards/alfworld_rollout_reward_func/mean": 0.49000000953674316, |
| "rewards/alfworld_rollout_reward_func/std": 0.7212488651275635, |
| "sampling/importance_sampling_ratio/max": 1.0000007152557373, |
| "sampling/importance_sampling_ratio/mean": 0.9999703764915466, |
| "sampling/importance_sampling_ratio/min": 0.999940037727356, |
| "sampling/sampling_logp_difference/max": 5.8182922657579184e-05, |
| "sampling/sampling_logp_difference/mean": 2.677608563317335e-06, |
| "step": 34, |
| "step_time": 13.956647035999822 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 62.0, |
| "completions/max_terminated_length": 62.0, |
| "completions/mean_length": 49.0, |
| "completions/mean_terminated_length": 49.0, |
| "completions/min_length": 36.0, |
| "completions/min_terminated_length": 36.0, |
| "entropy": 0.489044189453125, |
| "epoch": 0.03684210526315789, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 46.0, |
| "kl": 0.004022765904664993, |
| "learning_rate": 6.800000000000001e-06, |
| "loss": -0.8836, |
| "num_tokens": 267539.0, |
| "reward": 0.9549999833106995, |
| "reward_std": 0.007071061059832573, |
| "rewards/alfworld_rollout_reward_func/mean": 0.9549999833106995, |
| "rewards/alfworld_rollout_reward_func/std": 0.007071061059832573, |
| "sampling/importance_sampling_ratio/max": 2.5752766132354736, |
| "sampling/importance_sampling_ratio/mean": 1.780219316482544, |
| "sampling/importance_sampling_ratio/min": 0.985162079334259, |
| "sampling/sampling_logp_difference/max": 0.38198375701904297, |
| "sampling/sampling_logp_difference/mean": 0.03108775056898594, |
| "step": 35, |
| "step_time": 13.362450941999896 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 29.0, |
| "completions/max_terminated_length": 29.0, |
| "completions/mean_length": 28.5, |
| "completions/mean_terminated_length": 28.5, |
| "completions/min_length": 28.0, |
| "completions/min_terminated_length": 28.0, |
| "entropy": 0.03428112342953682, |
| "epoch": 0.037894736842105266, |
| "frac_reward_zero_std": 1.0, |
| "grad_norm": 0.005950927734375, |
| "kl": 9.688996215118095e-05, |
| "learning_rate": 7e-06, |
| "loss": 0.0, |
| "num_tokens": 275461.0, |
| "reward": 0.0, |
| "reward_std": 0.0, |
| "rewards/alfworld_rollout_reward_func/mean": 0.0, |
| "rewards/alfworld_rollout_reward_func/std": 0.0, |
| "sampling/importance_sampling_ratio/max": 1.0216329097747803, |
| "sampling/importance_sampling_ratio/mean": 1.010430932044983, |
| "sampling/importance_sampling_ratio/min": 0.9992288947105408, |
| "sampling/sampling_logp_difference/max": 0.04146456718444824, |
| "sampling/sampling_logp_difference/mean": 0.0011078877141699195, |
| "step": 36, |
| "step_time": 16.18710341299993 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 51.0, |
| "completions/max_terminated_length": 51.0, |
| "completions/mean_length": 49.0, |
| "completions/mean_terminated_length": 49.0, |
| "completions/min_length": 47.0, |
| "completions/min_terminated_length": 47.0, |
| "entropy": 0.375491201877594, |
| "epoch": 0.03894736842105263, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 27.375, |
| "kl": 0.008600625209510326, |
| "learning_rate": 7.2000000000000005e-06, |
| "loss": 0.3645, |
| "num_tokens": 283600.0, |
| "reward": -0.08500000089406967, |
| "reward_std": 0.02121320366859436, |
| "rewards/alfworld_rollout_reward_func/mean": -0.08500000089406967, |
| "rewards/alfworld_rollout_reward_func/std": 0.02121320366859436, |
| "sampling/importance_sampling_ratio/max": 1.302950382232666, |
| "sampling/importance_sampling_ratio/mean": 0.8185228705406189, |
| "sampling/importance_sampling_ratio/min": 0.3340953290462494, |
| "sampling/sampling_logp_difference/max": 0.3411126136779785, |
| "sampling/sampling_logp_difference/mean": 0.03013680875301361, |
| "step": 37, |
| "step_time": 20.539627713000073 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12.0, |
| "completions/max_terminated_length": 12.0, |
| "completions/mean_length": 8.0, |
| "completions/mean_terminated_length": 8.0, |
| "completions/min_length": 4.0, |
| "completions/min_terminated_length": 4.0, |
| "entropy": 0.09109717607498169, |
| "epoch": 0.04, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 170.0, |
| "kl": 0.019233860075473785, |
| "learning_rate": 7.4e-06, |
| "loss": 0.2797, |
| "num_tokens": 291611.0, |
| "reward": 0.49000000953674316, |
| "reward_std": 0.7071067690849304, |
| "rewards/alfworld_rollout_reward_func/mean": 0.49000000953674316, |
| "rewards/alfworld_rollout_reward_func/std": 0.7071067690849304, |
| "sampling/importance_sampling_ratio/max": 1.4181230068206787, |
| "sampling/importance_sampling_ratio/mean": 1.2089695930480957, |
| "sampling/importance_sampling_ratio/min": 0.9998162984848022, |
| "sampling/sampling_logp_difference/max": 0.3511829376220703, |
| "sampling/sampling_logp_difference/mean": 0.022082466632127762, |
| "step": 38, |
| "step_time": 13.460569201999988 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 47.0, |
| "completions/max_terminated_length": 47.0, |
| "completions/mean_length": 31.0, |
| "completions/mean_terminated_length": 31.0, |
| "completions/min_length": 15.0, |
| "completions/min_terminated_length": 15.0, |
| "entropy": 0.35754403471946716, |
| "epoch": 0.04105263157894737, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 8.4375, |
| "kl": 0.01082681491971016, |
| "learning_rate": 7.600000000000001e-06, |
| "loss": -0.0723, |
| "num_tokens": 299145.0, |
| "reward": 0.4350000023841858, |
| "reward_std": 0.7990306615829468, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4350000023841858, |
| "rewards/alfworld_rollout_reward_func/std": 0.7990306615829468, |
| "sampling/importance_sampling_ratio/max": 1.0000016689300537, |
| "sampling/importance_sampling_ratio/mean": 0.5920178294181824, |
| "sampling/importance_sampling_ratio/min": 0.18403403460979462, |
| "sampling/sampling_logp_difference/max": 0.6585979461669922, |
| "sampling/sampling_logp_difference/mean": 0.041399676352739334, |
| "step": 39, |
| "step_time": 15.72952194599975 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 37.0, |
| "completions/max_terminated_length": 37.0, |
| "completions/mean_length": 22.5, |
| "completions/mean_terminated_length": 22.5, |
| "completions/min_length": 8.0, |
| "completions/min_terminated_length": 8.0, |
| "entropy": 0.35843947529792786, |
| "epoch": 0.042105263157894736, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 28.625, |
| "kl": 0.0046605560928583145, |
| "learning_rate": 7.800000000000002e-06, |
| "loss": -0.3742, |
| "num_tokens": 307197.0, |
| "reward": -0.009999999776482582, |
| "reward_std": 0.01414213515818119, |
| "rewards/alfworld_rollout_reward_func/mean": -0.009999999776482582, |
| "rewards/alfworld_rollout_reward_func/std": 0.01414213515818119, |
| "sampling/importance_sampling_ratio/max": 1.0000009536743164, |
| "sampling/importance_sampling_ratio/mean": 0.9322052001953125, |
| "sampling/importance_sampling_ratio/min": 0.8644094467163086, |
| "sampling/sampling_logp_difference/max": 0.34821510314941406, |
| "sampling/sampling_logp_difference/mean": 0.02805318869650364, |
| "step": 40, |
| "step_time": 13.67377691799993 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 12.0, |
| "completions/max_terminated_length": 12.0, |
| "completions/mean_length": 12.0, |
| "completions/mean_terminated_length": 12.0, |
| "completions/min_length": 12.0, |
| "completions/min_terminated_length": 12.0, |
| "entropy": 0.028120441362261772, |
| "epoch": 0.0431578947368421, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 92.5, |
| "kl": 0.0015685707330703735, |
| "learning_rate": 8.000000000000001e-06, |
| "loss": 0.0001, |
| "num_tokens": 313777.0, |
| "reward": 0.9399999976158142, |
| "reward_std": 0.05656857416033745, |
| "rewards/alfworld_rollout_reward_func/mean": 0.9399999976158142, |
| "rewards/alfworld_rollout_reward_func/std": 0.05656857416033745, |
| "sampling/importance_sampling_ratio/max": 1.0000027418136597, |
| "sampling/importance_sampling_ratio/mean": 0.9999538064002991, |
| "sampling/importance_sampling_ratio/min": 0.9999048709869385, |
| "sampling/sampling_logp_difference/max": 9.777725790627301e-05, |
| "sampling/sampling_logp_difference/mean": 4.304089543438749e-06, |
| "step": 41, |
| "step_time": 11.189458066000043 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 40.0, |
| "completions/max_terminated_length": 40.0, |
| "completions/mean_length": 36.0, |
| "completions/mean_terminated_length": 36.0, |
| "completions/min_length": 32.0, |
| "completions/min_terminated_length": 32.0, |
| "entropy": 0.24259884655475616, |
| "epoch": 0.04421052631578947, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 16.75, |
| "kl": 0.0016654051141813397, |
| "learning_rate": 8.2e-06, |
| "loss": 0.1025, |
| "num_tokens": 321642.0, |
| "reward": -0.014999999664723873, |
| "reward_std": 0.02121320366859436, |
| "rewards/alfworld_rollout_reward_func/mean": -0.014999999664723873, |
| "rewards/alfworld_rollout_reward_func/std": 0.02121320366859436, |
| "sampling/importance_sampling_ratio/max": 1.060155987739563, |
| "sampling/importance_sampling_ratio/mean": 1.0288386344909668, |
| "sampling/importance_sampling_ratio/min": 0.9975212216377258, |
| "sampling/sampling_logp_difference/max": 0.16695499420166016, |
| "sampling/sampling_logp_difference/mean": 0.009143915958702564, |
| "step": 42, |
| "step_time": 22.64027631700037 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 38.0, |
| "completions/max_terminated_length": 38.0, |
| "completions/mean_length": 23.5, |
| "completions/mean_terminated_length": 23.5, |
| "completions/min_length": 9.0, |
| "completions/min_terminated_length": 9.0, |
| "entropy": 0.20157602429389954, |
| "epoch": 0.045263157894736845, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 63.25, |
| "kl": 0.022462664172053337, |
| "learning_rate": 8.400000000000001e-06, |
| "loss": 0.2766, |
| "num_tokens": 325516.0, |
| "reward": 0.9850000143051147, |
| "reward_std": 0.007071061059832573, |
| "rewards/alfworld_rollout_reward_func/mean": 0.9850000143051147, |
| "rewards/alfworld_rollout_reward_func/std": 0.007071061059832573, |
| "sampling/importance_sampling_ratio/max": 0.9990339875221252, |
| "sampling/importance_sampling_ratio/mean": 0.8629282712936401, |
| "sampling/importance_sampling_ratio/min": 0.7268226146697998, |
| "sampling/sampling_logp_difference/max": 0.17243313789367676, |
| "sampling/sampling_logp_difference/mean": 0.015407886356115341, |
| "step": 43, |
| "step_time": 5.622832681000091 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 35.0, |
| "completions/max_terminated_length": 35.0, |
| "completions/mean_length": 33.5, |
| "completions/mean_terminated_length": 33.5, |
| "completions/min_length": 32.0, |
| "completions/min_terminated_length": 32.0, |
| "entropy": 0.2535250782966614, |
| "epoch": 0.04631578947368421, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 19.25, |
| "kl": 0.004778198432177305, |
| "learning_rate": 8.6e-06, |
| "loss": 0.0797, |
| "num_tokens": 333519.0, |
| "reward": 0.4699999988079071, |
| "reward_std": 0.7495331764221191, |
| "rewards/alfworld_rollout_reward_func/mean": 0.4699999988079071, |
| "rewards/alfworld_rollout_reward_func/std": 0.7495331764221191, |
| "sampling/importance_sampling_ratio/max": 0.7066987752914429, |
| "sampling/importance_sampling_ratio/mean": 0.6219298839569092, |
| "sampling/importance_sampling_ratio/min": 0.5371610522270203, |
| "sampling/sampling_logp_difference/max": 0.34627819061279297, |
| "sampling/sampling_logp_difference/mean": 0.017883947119116783, |
| "step": 44, |
| "step_time": 15.114469847999771 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 4750, |
| "num_input_tokens_seen": 333519, |
| "num_train_epochs": 5, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
