| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.5161290322580645, |
| "eval_steps": 500, |
| "global_step": 32, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.53125, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 487.0, |
| "completions/mean_length": 370.3359375, |
| "completions/mean_terminated_length": 209.78334045410156, |
| "completions/min_length": 66.0, |
| "completions/min_terminated_length": 66.0, |
| "entropy": 0.40961652249097824, |
| "epoch": 0.016129032258064516, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1428968608379364, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": -0.0891, |
| "num_tokens": 110863.0, |
| "reward": 0.09531250596046448, |
| "reward_std": 2.4042975902557373, |
| "rewards/alfworld_rollout_reward_func/mean": 0.09531247615814209, |
| "rewards/alfworld_rollout_reward_func/std": 2.9919276237487793, |
| "sampling/importance_sampling_ratio/max": 2.9990427494049072, |
| "sampling/importance_sampling_ratio/mean": 0.6262807846069336, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 4.450258255004883, |
| "sampling/sampling_logp_difference/mean": 0.024438712745904922, |
| "step": 1, |
| "step_time": 309.93943907799985 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.5703125, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 509.0, |
| "completions/mean_length": 380.1015625, |
| "completions/mean_terminated_length": 205.03636169433594, |
| "completions/min_length": 79.0, |
| "completions/min_terminated_length": 79.0, |
| "entropy": 0.40012454986572266, |
| "epoch": 0.03225806451612903, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.12177681177854538, |
| "kl": 0.0, |
| "learning_rate": 2.5e-05, |
| "loss": -0.085, |
| "num_tokens": 222814.0, |
| "reward": 0.5320311784744263, |
| "reward_std": 2.642284393310547, |
| "rewards/alfworld_rollout_reward_func/mean": 0.5320311784744263, |
| "rewards/alfworld_rollout_reward_func/std": 3.4178764820098877, |
| "sampling/importance_sampling_ratio/max": 2.909560441970825, |
| "sampling/importance_sampling_ratio/mean": 0.5676747560501099, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 4.949212551116943, |
| "sampling/sampling_logp_difference/mean": 0.023133086040616035, |
| "step": 2, |
| "step_time": 315.77915475400005 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.6015625, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 497.0, |
| "completions/mean_length": 378.34375, |
| "completions/mean_terminated_length": 176.5490264892578, |
| "completions/min_length": 76.0, |
| "completions/min_terminated_length": 76.0, |
| "entropy": 0.4211720749735832, |
| "epoch": 0.04838709677419355, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.13358080387115479, |
| "kl": 0.00272539074649103, |
| "learning_rate": 2.493586654239869e-05, |
| "loss": -0.0053, |
| "num_tokens": 332775.0, |
| "reward": 0.925000011920929, |
| "reward_std": 3.4297051429748535, |
| "rewards/alfworld_rollout_reward_func/mean": 0.925000011920929, |
| "rewards/alfworld_rollout_reward_func/std": 3.9396629333496094, |
| "sampling/importance_sampling_ratio/max": 2.511305809020996, |
| "sampling/importance_sampling_ratio/mean": 0.5851866602897644, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.5749268531799316, |
| "sampling/sampling_logp_difference/mean": 0.023533061146736145, |
| "step": 3, |
| "step_time": 296.23798338899906 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.53125, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 511.0, |
| "completions/mean_length": 357.84375, |
| "completions/mean_terminated_length": 183.1333465576172, |
| "completions/min_length": 66.0, |
| "completions/min_terminated_length": 66.0, |
| "entropy": 0.3783623240888119, |
| "epoch": 0.06451612903225806, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.15261100232601166, |
| "kl": 0.002557590647484176, |
| "learning_rate": 2.474412426565618e-05, |
| "loss": -0.0004, |
| "num_tokens": 441773.0, |
| "reward": 0.45625001192092896, |
| "reward_std": 2.879180908203125, |
| "rewards/alfworld_rollout_reward_func/mean": 0.45625001192092896, |
| "rewards/alfworld_rollout_reward_func/std": 3.427947759628296, |
| "sampling/importance_sampling_ratio/max": 2.91353440284729, |
| "sampling/importance_sampling_ratio/mean": 0.6235978603363037, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 4.593559741973877, |
| "sampling/sampling_logp_difference/mean": 0.023225955665111542, |
| "step": 4, |
| "step_time": 322.2789937999987 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.3984375, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 504.0, |
| "completions/mean_length": 298.53125, |
| "completions/mean_terminated_length": 157.14285278320312, |
| "completions/min_length": 64.0, |
| "completions/min_terminated_length": 64.0, |
| "entropy": 0.4225323870778084, |
| "epoch": 0.08064516129032258, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11338824778795242, |
| "kl": 0.003911561769200489, |
| "learning_rate": 2.442674070500061e-05, |
| "loss": 0.0261, |
| "num_tokens": 540709.0, |
| "reward": 0.48515623807907104, |
| "reward_std": 2.928551197052002, |
| "rewards/alfworld_rollout_reward_func/mean": 0.48515623807907104, |
| "rewards/alfworld_rollout_reward_func/std": 3.3146796226501465, |
| "sampling/importance_sampling_ratio/max": 2.970187187194824, |
| "sampling/importance_sampling_ratio/mean": 0.5831338763237, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 6.362844944000244, |
| "sampling/sampling_logp_difference/mean": 0.024303022772073746, |
| "step": 5, |
| "step_time": 281.38914586900137 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.34375, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 476.0, |
| "completions/mean_length": 280.25, |
| "completions/mean_terminated_length": 158.85714721679688, |
| "completions/min_length": 79.0, |
| "completions/min_terminated_length": 79.0, |
| "entropy": 0.4307485744357109, |
| "epoch": 0.0967741935483871, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.840685248374939, |
| "kl": 0.035365400719456375, |
| "learning_rate": 2.3986972645252883e-05, |
| "loss": 0.0634, |
| "num_tokens": 641237.0, |
| "reward": 1.1078124046325684, |
| "reward_std": 3.4873318672180176, |
| "rewards/alfworld_rollout_reward_func/mean": 1.107812523841858, |
| "rewards/alfworld_rollout_reward_func/std": 3.958895206451416, |
| "sampling/importance_sampling_ratio/max": 2.7673635482788086, |
| "sampling/importance_sampling_ratio/mean": 0.561335563659668, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 11.066271781921387, |
| "sampling/sampling_logp_difference/mean": 0.02636803314089775, |
| "step": 6, |
| "step_time": 306.572960538002 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.28125, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 474.0, |
| "completions/mean_length": 248.328125, |
| "completions/mean_terminated_length": 145.1521759033203, |
| "completions/min_length": 61.0, |
| "completions/min_terminated_length": 61.0, |
| "entropy": 0.4486994035542011, |
| "epoch": 0.11290322580645161, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.13363927602767944, |
| "kl": 0.010801714262925088, |
| "learning_rate": 2.342933270180728e-05, |
| "loss": 0.1147, |
| "num_tokens": 734359.0, |
| "reward": 0.7398437261581421, |
| "reward_std": 3.073120594024658, |
| "rewards/alfworld_rollout_reward_func/mean": 0.7398437261581421, |
| "rewards/alfworld_rollout_reward_func/std": 3.5458602905273438, |
| "sampling/importance_sampling_ratio/max": 2.4107882976531982, |
| "sampling/importance_sampling_ratio/mean": 0.5770797729492188, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 7.637388229370117, |
| "sampling/sampling_logp_difference/mean": 0.027358299121260643, |
| "step": 7, |
| "step_time": 291.4309037350031 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.2109375, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 487.0, |
| "completions/mean_length": 224.53125, |
| "completions/mean_terminated_length": 147.68316650390625, |
| "completions/min_length": 71.0, |
| "completions/min_terminated_length": 71.0, |
| "entropy": 0.4458822049200535, |
| "epoch": 0.12903225806451613, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1380084902048111, |
| "kl": 0.018724333960562944, |
| "learning_rate": 2.2759543015090955e-05, |
| "loss": 0.0322, |
| "num_tokens": 827827.0, |
| "reward": 0.8648437261581421, |
| "reward_std": 3.5357110500335693, |
| "rewards/alfworld_rollout_reward_func/mean": 0.8648437857627869, |
| "rewards/alfworld_rollout_reward_func/std": 3.7438578605651855, |
| "sampling/importance_sampling_ratio/max": 2.939615249633789, |
| "sampling/importance_sampling_ratio/mean": 0.5691805481910706, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 5.754482269287109, |
| "sampling/sampling_logp_difference/mean": 0.028573205694556236, |
| "step": 8, |
| "step_time": 305.87279431799834 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1875, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 504.0, |
| "completions/mean_length": 218.171875, |
| "completions/mean_terminated_length": 150.36538696289062, |
| "completions/min_length": 68.0, |
| "completions/min_terminated_length": 68.0, |
| "entropy": 0.4581918604671955, |
| "epoch": 0.14516129032258066, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.11444258689880371, |
| "kl": 0.02050452691037208, |
| "learning_rate": 2.1984476533659888e-05, |
| "loss": 0.0214, |
| "num_tokens": 918938.0, |
| "reward": 1.025781273841858, |
| "reward_std": 3.5701520442962646, |
| "rewards/alfworld_rollout_reward_func/mean": 1.0257811546325684, |
| "rewards/alfworld_rollout_reward_func/std": 3.894923686981201, |
| "sampling/importance_sampling_ratio/max": 2.700753688812256, |
| "sampling/importance_sampling_ratio/mean": 0.6189267635345459, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.1386852264404297, |
| "sampling/sampling_logp_difference/mean": 0.0270779300481081, |
| "step": 9, |
| "step_time": 308.85608878800485 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0546875, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 432.0, |
| "completions/mean_length": 150.1875, |
| "completions/mean_terminated_length": 129.25619506835938, |
| "completions/min_length": 62.0, |
| "completions/min_terminated_length": 62.0, |
| "entropy": 0.5325295478105545, |
| "epoch": 0.16129032258064516, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.2502661645412445, |
| "kl": 0.03952266392298043, |
| "learning_rate": 2.1112086488446085e-05, |
| "loss": 0.029, |
| "num_tokens": 1000430.0, |
| "reward": 0.8335937261581421, |
| "reward_std": 3.1759183406829834, |
| "rewards/alfworld_rollout_reward_func/mean": 0.8335937261581421, |
| "rewards/alfworld_rollout_reward_func/std": 3.7348177433013916, |
| "sampling/importance_sampling_ratio/max": 2.995654344558716, |
| "sampling/importance_sampling_ratio/mean": 0.7903203964233398, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.775932788848877, |
| "sampling/sampling_logp_difference/mean": 0.03168691694736481, |
| "step": 10, |
| "step_time": 285.58172405699224 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0859375, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 325.0, |
| "completions/mean_length": 156.7734375, |
| "completions/mean_terminated_length": 123.3760757446289, |
| "completions/min_length": 64.0, |
| "completions/min_terminated_length": 64.0, |
| "entropy": 0.5142566300928593, |
| "epoch": 0.1774193548387097, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.18933017551898956, |
| "kl": 0.04493421735242009, |
| "learning_rate": 2.0151324781845787e-05, |
| "loss": 0.0496, |
| "num_tokens": 1082621.0, |
| "reward": 0.9093749523162842, |
| "reward_std": 3.467135429382324, |
| "rewards/alfworld_rollout_reward_func/mean": 0.909375011920929, |
| "rewards/alfworld_rollout_reward_func/std": 3.8583264350891113, |
| "sampling/importance_sampling_ratio/max": 2.9975228309631348, |
| "sampling/importance_sampling_ratio/mean": 0.8283847570419312, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.3669824600219727, |
| "sampling/sampling_logp_difference/mean": 0.028965603560209274, |
| "step": 11, |
| "step_time": 300.83864827199795 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.046875, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 343.0, |
| "completions/mean_length": 140.28125, |
| "completions/mean_terminated_length": 121.99999237060547, |
| "completions/min_length": 51.0, |
| "completions/min_terminated_length": 51.0, |
| "entropy": 0.519548587501049, |
| "epoch": 0.1935483870967742, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.2622426152229309, |
| "kl": 0.05217514652758837, |
| "learning_rate": 1.911205012908703e-05, |
| "loss": -0.0594, |
| "num_tokens": 1162516.0, |
| "reward": 0.8148437738418579, |
| "reward_std": 3.1582674980163574, |
| "rewards/alfworld_rollout_reward_func/mean": 0.8148437142372131, |
| "rewards/alfworld_rollout_reward_func/std": 3.7535512447357178, |
| "sampling/importance_sampling_ratio/max": 2.8224730491638184, |
| "sampling/importance_sampling_ratio/mean": 0.7709423899650574, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.520348072052002, |
| "sampling/sampling_logp_difference/mean": 0.031547725200653076, |
| "step": 12, |
| "step_time": 287.3358050610004 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0703125, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 256.0, |
| "completions/mean_length": 151.046875, |
| "completions/mean_terminated_length": 123.7479019165039, |
| "completions/min_length": 60.0, |
| "completions/min_terminated_length": 60.0, |
| "entropy": 0.5261144414544106, |
| "epoch": 0.20967741935483872, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1863553375005722, |
| "kl": 0.06452470645308495, |
| "learning_rate": 1.800492689447043e-05, |
| "loss": -0.0566, |
| "num_tokens": 1243615.0, |
| "reward": 0.8781249523162842, |
| "reward_std": 3.3634252548217773, |
| "rewards/alfworld_rollout_reward_func/mean": 0.8781249523162842, |
| "rewards/alfworld_rollout_reward_func/std": 3.842200994491577, |
| "sampling/importance_sampling_ratio/max": 2.4663472175598145, |
| "sampling/importance_sampling_ratio/mean": 0.7910538911819458, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.4453465938568115, |
| "sampling/sampling_logp_difference/mean": 0.02982635796070099, |
| "step": 13, |
| "step_time": 293.8646844759969 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0078125, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 380.0, |
| "completions/mean_length": 129.8671875, |
| "completions/mean_terminated_length": 126.85826873779297, |
| "completions/min_length": 56.0, |
| "completions/min_terminated_length": 56.0, |
| "entropy": 0.5344252400100231, |
| "epoch": 0.22580645161290322, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.24170275032520294, |
| "kl": 0.06391164055094123, |
| "learning_rate": 1.6841315660560252e-05, |
| "loss": -0.1138, |
| "num_tokens": 1321112.0, |
| "reward": 0.9546874761581421, |
| "reward_std": 3.586996078491211, |
| "rewards/alfworld_rollout_reward_func/mean": 0.9546874761581421, |
| "rewards/alfworld_rollout_reward_func/std": 3.9051523208618164, |
| "sampling/importance_sampling_ratio/max": 2.8537843227386475, |
| "sampling/importance_sampling_ratio/mean": 0.7751458883285522, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.7649002075195312, |
| "sampling/sampling_logp_difference/mean": 0.03092074766755104, |
| "step": 14, |
| "step_time": 283.45238025300205 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0546875, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 457.0, |
| "completions/mean_length": 149.8515625, |
| "completions/mean_terminated_length": 128.90081787109375, |
| "completions/min_length": 70.0, |
| "completions/min_terminated_length": 70.0, |
| "entropy": 0.5125062428414822, |
| "epoch": 0.24193548387096775, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.16989152133464813, |
| "kl": 0.053708435501903296, |
| "learning_rate": 1.563315665323401e-05, |
| "loss": 0.0051, |
| "num_tokens": 1403786.0, |
| "reward": 0.34921878576278687, |
| "reward_std": 2.4881625175476074, |
| "rewards/alfworld_rollout_reward_func/mean": 0.3492187559604645, |
| "rewards/alfworld_rollout_reward_func/std": 3.1670050621032715, |
| "sampling/importance_sampling_ratio/max": 2.4405319690704346, |
| "sampling/importance_sampling_ratio/mean": 0.7845873832702637, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 1.7177300453186035, |
| "sampling/sampling_logp_difference/mean": 0.029662001878023148, |
| "step": 15, |
| "step_time": 308.28231060100006 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0703125, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 422.0, |
| "completions/mean_length": 156.3359375, |
| "completions/mean_terminated_length": 129.43698120117188, |
| "completions/min_length": 71.0, |
| "completions/min_terminated_length": 71.0, |
| "entropy": 0.5052059814333916, |
| "epoch": 0.25806451612903225, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.18680106103420258, |
| "kl": 0.052992088720202446, |
| "learning_rate": 1.439284721880721e-05, |
| "loss": -0.0061, |
| "num_tokens": 1485910.0, |
| "reward": 1.005468726158142, |
| "reward_std": 2.9901723861694336, |
| "rewards/alfworld_rollout_reward_func/mean": 1.005468726158142, |
| "rewards/alfworld_rollout_reward_func/std": 3.869877815246582, |
| "sampling/importance_sampling_ratio/max": 2.6479995250701904, |
| "sampling/importance_sampling_ratio/mean": 0.8335222005844116, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.2812881469726562, |
| "sampling/sampling_logp_difference/mean": 0.02822798863053322, |
| "step": 16, |
| "step_time": 288.45321577499635 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0625, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 436.0, |
| "completions/mean_length": 155.78125, |
| "completions/mean_terminated_length": 132.03334045410156, |
| "completions/min_length": 62.0, |
| "completions/min_terminated_length": 62.0, |
| "entropy": 0.5273009687662125, |
| "epoch": 0.27419354838709675, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.17423537373542786, |
| "kl": 0.07150922995060682, |
| "learning_rate": 1.3133114610483909e-05, |
| "loss": -0.0034, |
| "num_tokens": 1567778.0, |
| "reward": 0.25390625, |
| "reward_std": 2.4427285194396973, |
| "rewards/alfworld_rollout_reward_func/mean": 0.2539062201976776, |
| "rewards/alfworld_rollout_reward_func/std": 3.2506790161132812, |
| "sampling/importance_sampling_ratio/max": 2.4126062393188477, |
| "sampling/importance_sampling_ratio/mean": 0.8037413954734802, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.5358574390411377, |
| "sampling/sampling_logp_difference/mean": 0.029498092830181122, |
| "step": 17, |
| "step_time": 281.3368725319997 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1484375, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 477.0, |
| "completions/mean_length": 186.5, |
| "completions/mean_terminated_length": 129.76145935058594, |
| "completions/min_length": 73.0, |
| "completions/min_terminated_length": 73.0, |
| "entropy": 0.47341491281986237, |
| "epoch": 0.2903225806451613, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1518625020980835, |
| "kl": 0.049451613100245595, |
| "learning_rate": 1.1866885389516092e-05, |
| "loss": 0.0897, |
| "num_tokens": 1656705.0, |
| "reward": 0.80859375, |
| "reward_std": 3.4441990852355957, |
| "rewards/alfworld_rollout_reward_func/mean": 0.80859375, |
| "rewards/alfworld_rollout_reward_func/std": 3.8008499145507812, |
| "sampling/importance_sampling_ratio/max": 2.9426238536834717, |
| "sampling/importance_sampling_ratio/mean": 0.7826383113861084, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.3363547325134277, |
| "sampling/sampling_logp_difference/mean": 0.027312669903039932, |
| "step": 18, |
| "step_time": 334.05939280299935 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1015625, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 499.0, |
| "completions/mean_length": 166.0859375, |
| "completions/mean_terminated_length": 126.98260498046875, |
| "completions/min_length": 68.0, |
| "completions/min_terminated_length": 68.0, |
| "entropy": 0.47634539753198624, |
| "epoch": 0.3064516129032258, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.2188945859670639, |
| "kl": 0.0531598380766809, |
| "learning_rate": 1.0607152781192796e-05, |
| "loss": 0.0078, |
| "num_tokens": 1741466.0, |
| "reward": 0.6265625357627869, |
| "reward_std": 2.6468114852905273, |
| "rewards/alfworld_rollout_reward_func/mean": 0.6265624761581421, |
| "rewards/alfworld_rollout_reward_func/std": 3.4735782146453857, |
| "sampling/importance_sampling_ratio/max": 2.799043655395508, |
| "sampling/importance_sampling_ratio/mean": 0.8041483163833618, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.4288792610168457, |
| "sampling/sampling_logp_difference/mean": 0.02712005004286766, |
| "step": 19, |
| "step_time": 303.70944481100014 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0859375, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 403.0, |
| "completions/mean_length": 160.9375, |
| "completions/mean_terminated_length": 127.93163299560547, |
| "completions/min_length": 76.0, |
| "completions/min_terminated_length": 76.0, |
| "entropy": 0.46492085233330727, |
| "epoch": 0.3225806451612903, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.7110471725463867, |
| "kl": 0.09847234468907118, |
| "learning_rate": 9.366843346765992e-06, |
| "loss": 0.0238, |
| "num_tokens": 1823852.0, |
| "reward": 0.819531261920929, |
| "reward_std": 3.2860288619995117, |
| "rewards/alfworld_rollout_reward_func/mean": 0.8195312023162842, |
| "rewards/alfworld_rollout_reward_func/std": 3.780327320098877, |
| "sampling/importance_sampling_ratio/max": 2.7687222957611084, |
| "sampling/importance_sampling_ratio/mean": 0.7973555326461792, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 20.83700942993164, |
| "sampling/sampling_logp_difference/mean": 0.027603479102253914, |
| "step": 20, |
| "step_time": 296.20082658499814 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.078125, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 356.0, |
| "completions/mean_length": 154.0625, |
| "completions/mean_terminated_length": 123.72881317138672, |
| "completions/min_length": 58.0, |
| "completions/min_terminated_length": 58.0, |
| "entropy": 0.4891073890030384, |
| "epoch": 0.3387096774193548, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.3204660415649414, |
| "kl": 0.06161752389743924, |
| "learning_rate": 8.158684339439748e-06, |
| "loss": 0.1854, |
| "num_tokens": 1905839.0, |
| "reward": 0.66796875, |
| "reward_std": 2.7234749794006348, |
| "rewards/alfworld_rollout_reward_func/mean": 0.66796875, |
| "rewards/alfworld_rollout_reward_func/std": 3.580017566680908, |
| "sampling/importance_sampling_ratio/max": 2.9959678649902344, |
| "sampling/importance_sampling_ratio/mean": 0.9344255328178406, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.3660998344421387, |
| "sampling/sampling_logp_difference/mean": 0.029398081824183464, |
| "step": 21, |
| "step_time": 279.7738521129977 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.171875, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 446.0, |
| "completions/mean_length": 197.1015625, |
| "completions/mean_terminated_length": 131.7452850341797, |
| "completions/min_length": 65.0, |
| "completions/min_terminated_length": 65.0, |
| "entropy": 0.4758484289050102, |
| "epoch": 0.3548387096774194, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.16225755214691162, |
| "kl": 0.054194105323404074, |
| "learning_rate": 6.99507310552957e-06, |
| "loss": 0.0319, |
| "num_tokens": 1991294.0, |
| "reward": 0.69921875, |
| "reward_std": 3.503169298171997, |
| "rewards/alfworld_rollout_reward_func/mean": 0.69921875, |
| "rewards/alfworld_rollout_reward_func/std": 3.726823329925537, |
| "sampling/importance_sampling_ratio/max": 2.5170819759368896, |
| "sampling/importance_sampling_ratio/mean": 0.777429461479187, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.413573741912842, |
| "sampling/sampling_logp_difference/mean": 0.027101196348667145, |
| "step": 22, |
| "step_time": 285.0048562720003 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1171875, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 471.0, |
| "completions/mean_length": 169.8359375, |
| "completions/mean_terminated_length": 124.41593170166016, |
| "completions/min_length": 57.0, |
| "completions/min_terminated_length": 57.0, |
| "entropy": 0.5033188574016094, |
| "epoch": 0.3709677419354839, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.18970800936222076, |
| "kl": 0.04351181210950017, |
| "learning_rate": 5.8879498709129735e-06, |
| "loss": -0.0057, |
| "num_tokens": 2075803.0, |
| "reward": 0.8039063215255737, |
| "reward_std": 3.560253620147705, |
| "rewards/alfworld_rollout_reward_func/mean": 0.8039062023162842, |
| "rewards/alfworld_rollout_reward_func/std": 3.759028911590576, |
| "sampling/importance_sampling_ratio/max": 2.6937098503112793, |
| "sampling/importance_sampling_ratio/mean": 0.7721619606018066, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.6162972450256348, |
| "sampling/sampling_logp_difference/mean": 0.02900775894522667, |
| "step": 23, |
| "step_time": 298.5376572299974 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1484375, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 451.0, |
| "completions/mean_length": 186.1640625, |
| "completions/mean_terminated_length": 129.36697387695312, |
| "completions/min_length": 60.0, |
| "completions/min_terminated_length": 60.0, |
| "entropy": 0.4917047880589962, |
| "epoch": 0.3870967741935484, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.18956582248210907, |
| "kl": 0.04409886850044131, |
| "learning_rate": 4.848675218154214e-06, |
| "loss": -0.0152, |
| "num_tokens": 2160777.0, |
| "reward": 0.675000011920929, |
| "reward_std": 2.953065872192383, |
| "rewards/alfworld_rollout_reward_func/mean": 0.675000011920929, |
| "rewards/alfworld_rollout_reward_func/std": 3.6408443450927734, |
| "sampling/importance_sampling_ratio/max": 2.5711066722869873, |
| "sampling/importance_sampling_ratio/mean": 0.6986918449401855, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.6658549308776855, |
| "sampling/sampling_logp_difference/mean": 0.02768370695412159, |
| "step": 24, |
| "step_time": 286.79983711899695 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.140625, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 436.0, |
| "completions/mean_length": 176.5703125, |
| "completions/mean_terminated_length": 121.68181610107422, |
| "completions/min_length": 54.0, |
| "completions/min_terminated_length": 54.0, |
| "entropy": 0.4625844620168209, |
| "epoch": 0.4032258064516129, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1729527860879898, |
| "kl": 0.046662998385727406, |
| "learning_rate": 3.887913511553917e-06, |
| "loss": 0.0258, |
| "num_tokens": 2247234.0, |
| "reward": 0.8726562261581421, |
| "reward_std": 3.0254807472229004, |
| "rewards/alfworld_rollout_reward_func/mean": 0.8726562261581421, |
| "rewards/alfworld_rollout_reward_func/std": 3.7554731369018555, |
| "sampling/importance_sampling_ratio/max": 2.884495496749878, |
| "sampling/importance_sampling_ratio/mean": 0.7411025762557983, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.431777000427246, |
| "sampling/sampling_logp_difference/mean": 0.02642218954861164, |
| "step": 25, |
| "step_time": 331.329842252002 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1953125, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 437.0, |
| "completions/mean_length": 203.78125, |
| "completions/mean_terminated_length": 128.9708709716797, |
| "completions/min_length": 68.0, |
| "completions/min_terminated_length": 68.0, |
| "entropy": 0.4392157681286335, |
| "epoch": 0.41935483870967744, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.18243148922920227, |
| "kl": 0.03993735695257783, |
| "learning_rate": 3.0155234663401146e-06, |
| "loss": 0.1456, |
| "num_tokens": 2332769.0, |
| "reward": 2.0328125953674316, |
| "reward_std": 4.510900020599365, |
| "rewards/alfworld_rollout_reward_func/mean": 2.0328123569488525, |
| "rewards/alfworld_rollout_reward_func/std": 4.624057769775391, |
| "sampling/importance_sampling_ratio/max": 2.9809186458587646, |
| "sampling/importance_sampling_ratio/mean": 0.8058295845985413, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.739076852798462, |
| "sampling/sampling_logp_difference/mean": 0.025428924709558487, |
| "step": 26, |
| "step_time": 262.9066606970009 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.125, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 387.0, |
| "completions/mean_length": 173.7890625, |
| "completions/mean_terminated_length": 125.47322082519531, |
| "completions/min_length": 58.0, |
| "completions/min_terminated_length": 58.0, |
| "entropy": 0.46685169637203217, |
| "epoch": 0.43548387096774194, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.17903129756450653, |
| "kl": 0.046850728802382946, |
| "learning_rate": 2.240456984909049e-06, |
| "loss": 0.0513, |
| "num_tokens": 2415919.0, |
| "reward": 1.5, |
| "reward_std": 4.049795627593994, |
| "rewards/alfworld_rollout_reward_func/mean": 1.5, |
| "rewards/alfworld_rollout_reward_func/std": 4.301400661468506, |
| "sampling/importance_sampling_ratio/max": 2.8721108436584473, |
| "sampling/importance_sampling_ratio/mean": 0.813679575920105, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.3308191299438477, |
| "sampling/sampling_logp_difference/mean": 0.027277380228042603, |
| "step": 27, |
| "step_time": 287.4597008519995 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1640625, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 253.0, |
| "completions/mean_length": 182.4140625, |
| "completions/mean_terminated_length": 117.72896575927734, |
| "completions/min_length": 67.0, |
| "completions/min_terminated_length": 67.0, |
| "entropy": 0.4677879251539707, |
| "epoch": 0.45161290322580644, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.13941031694412231, |
| "kl": 0.042057349579408765, |
| "learning_rate": 1.570667298192724e-06, |
| "loss": 0.0048, |
| "num_tokens": 2500100.0, |
| "reward": 0.5835937261581421, |
| "reward_std": 2.6255595684051514, |
| "rewards/alfworld_rollout_reward_func/mean": 0.5835937261581421, |
| "rewards/alfworld_rollout_reward_func/std": 3.450979232788086, |
| "sampling/importance_sampling_ratio/max": 2.854583501815796, |
| "sampling/importance_sampling_ratio/mean": 0.7393490076065063, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 5.095232963562012, |
| "sampling/sampling_logp_difference/mean": 0.027033191174268723, |
| "step": 28, |
| "step_time": 285.79294812399894 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.2109375, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 479.0, |
| "completions/mean_length": 212.203125, |
| "completions/mean_terminated_length": 132.0594024658203, |
| "completions/min_length": 58.0, |
| "completions/min_terminated_length": 58.0, |
| "entropy": 0.4555159918963909, |
| "epoch": 0.46774193548387094, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.16522692143917084, |
| "kl": 0.04728960315696895, |
| "learning_rate": 1.0130273547471176e-06, |
| "loss": 0.0531, |
| "num_tokens": 2589663.0, |
| "reward": 0.828906238079071, |
| "reward_std": 3.300809860229492, |
| "rewards/alfworld_rollout_reward_func/mean": 0.8289061784744263, |
| "rewards/alfworld_rollout_reward_func/std": 3.700098991394043, |
| "sampling/importance_sampling_ratio/max": 2.9519269466400146, |
| "sampling/importance_sampling_ratio/mean": 0.7473565340042114, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 2.6239898204803467, |
| "sampling/sampling_logp_difference/mean": 0.024686329066753387, |
| "step": 29, |
| "step_time": 316.5334539249998 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.140625, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 404.0, |
| "completions/mean_length": 182.484375, |
| "completions/mean_terminated_length": 128.56362915039062, |
| "completions/min_length": 69.0, |
| "completions/min_terminated_length": 69.0, |
| "entropy": 0.4658549278974533, |
| "epoch": 0.4838709677419355, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1560395509004593, |
| "kl": 0.05294563015922904, |
| "learning_rate": 5.732592949993898e-07, |
| "loss": 0.0688, |
| "num_tokens": 2675234.0, |
| "reward": 0.84765625, |
| "reward_std": 3.315725803375244, |
| "rewards/alfworld_rollout_reward_func/mean": 0.84765625, |
| "rewards/alfworld_rollout_reward_func/std": 3.8803963661193848, |
| "sampling/importance_sampling_ratio/max": 2.622981548309326, |
| "sampling/importance_sampling_ratio/mean": 0.7878429293632507, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.5301332473754883, |
| "sampling/sampling_logp_difference/mean": 0.026188407093286514, |
| "step": 30, |
| "step_time": 298.4171370410004 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.1328125, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 446.0, |
| "completions/mean_length": 179.0078125, |
| "completions/mean_terminated_length": 128.00901794433594, |
| "completions/min_length": 63.0, |
| "completions/min_terminated_length": 63.0, |
| "entropy": 0.5069509670138359, |
| "epoch": 0.5, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.20257869362831116, |
| "kl": 0.055831174831837416, |
| "learning_rate": 2.5587573434381895e-07, |
| "loss": 0.0626, |
| "num_tokens": 2762673.0, |
| "reward": 0.6875, |
| "reward_std": 3.218989372253418, |
| "rewards/alfworld_rollout_reward_func/mean": 0.6875, |
| "rewards/alfworld_rollout_reward_func/std": 3.5843491554260254, |
| "sampling/importance_sampling_ratio/max": 2.8179516792297363, |
| "sampling/importance_sampling_ratio/mean": 0.7038029432296753, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 4.102773666381836, |
| "sampling/sampling_logp_difference/mean": 0.029051225632429123, |
| "step": 31, |
| "step_time": 305.9211267239989 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.140625, |
| "completions/max_length": 512.0, |
| "completions/max_terminated_length": 486.0, |
| "completions/mean_length": 193.7734375, |
| "completions/mean_terminated_length": 141.6999969482422, |
| "completions/min_length": 64.0, |
| "completions/min_terminated_length": 64.0, |
| "entropy": 0.50175466760993, |
| "epoch": 0.5161290322580645, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.14924179017543793, |
| "kl": 0.03636577841825783, |
| "learning_rate": 6.413345760131057e-08, |
| "loss": 0.1286, |
| "num_tokens": 2849119.0, |
| "reward": 0.9367187023162842, |
| "reward_std": 3.7098560333251953, |
| "rewards/alfworld_rollout_reward_func/mean": 0.9367187023162842, |
| "rewards/alfworld_rollout_reward_func/std": 3.8140616416931152, |
| "sampling/importance_sampling_ratio/max": 2.708343744277954, |
| "sampling/importance_sampling_ratio/mean": 0.8295129537582397, |
| "sampling/importance_sampling_ratio/min": 0.0, |
| "sampling/sampling_logp_difference/max": 3.4670004844665527, |
| "sampling/sampling_logp_difference/mean": 0.027453571557998657, |
| "step": 32, |
| "step_time": 294.9108767129983 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 32, |
| "num_input_tokens_seen": 2849119, |
| "num_train_epochs": 1, |
| "save_steps": 20, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
