{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.07401924500370097, "eval_steps": 500, "global_step": 500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.9, "completions/max_length": 256.0, "completions/max_terminated_length": 184.6, "completions/mean_length": 242.35, "completions/mean_terminated_length": 129.875, "completions/min_length": 64.4, "completions/min_terminated_length": 64.4, "epoch": 0.0007401924500370096, "frac_reward_zero_std": 0.0, "grad_norm": 11.3125, "kl": 0.05908113121986389, "learning_rate": 1.0958857142857143e-05, "loss": 0.0355, "num_tokens": 54058.0, "reward": -6979.34052734375, "reward_std": 1348.7608764648437, "rewards/wrapper/mean": -6979.34033203125, "rewards/wrapper/std": 1786.742578125, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.765, "completions/max_length": 256.0, "completions/max_terminated_length": 179.2, "completions/mean_length": 214.8850067138672, "completions/mean_terminated_length": 75.66153869628906, "completions/min_length": 12.6, "completions/min_terminated_length": 12.6, "epoch": 0.0014803849000740192, "frac_reward_zero_std": 0.0, "grad_norm": 4.4375, "kl": 0.12709049880504608, "learning_rate": 2.465742857142857e-05, "loss": 0.1053, "num_tokens": 102903.0, "reward": -5451.2669921875, "reward_std": 1737.884814453125, "rewards/wrapper/mean": -5451.2669921875, "rewards/wrapper/std": 2241.2872314453125, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.24500000000000002, "completions/max_length": 256.0, "completions/max_terminated_length": 241.6, "completions/mean_length": 108.87500228881837, "completions/mean_terminated_length": 64.4888931274414, "completions/min_length": 4.4, "completions/min_terminated_length": 4.4, "epoch": 0.0022205773501110288, "frac_reward_zero_std": 0.010000000149011612, "grad_norm": 8.6875, "kl": 0.2064658522605896, "learning_rate": 3.8356000000000003e-05, "loss": 0.3077, "num_tokens": 130842.0, "reward": -2267.2222900390625, "reward_std": 1258.246533203125, "rewards/wrapper/mean": -2267.2222900390625, "rewards/wrapper/std": 1737.7914306640625, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 13.6, "completions/max_terminated_length": 13.6, "completions/mean_length": 4.105000066757202, "completions/mean_terminated_length": 4.105000066757202, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.0029607698001480384, "frac_reward_zero_std": 0.5600000001490116, "grad_norm": 59.0, "kl": 4.3973316073417665, "learning_rate": 5.2054571428571426e-05, "loss": 0.3243, "num_tokens": 138247.0, "reward": -80.12234377861023, "reward_std": 67.20390920639038, "rewards/wrapper/mean": -80.12234377861023, "rewards/wrapper/std": 88.9855131149292, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 4.8, "completions/max_terminated_length": 4.8, "completions/mean_length": 1.2850000143051148, "completions/mean_terminated_length": 1.2850000143051148, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.003700962250185048, "frac_reward_zero_std": 0.9100000143051148, "grad_norm": 33.0, "kl": 6.291157531738281, "learning_rate": 6.575314285714286e-05, "loss": 0.292, "num_tokens": 144562.0, "reward": -4.0388649761676785, "reward_std": 2.521432876586914, "rewards/wrapper/mean": -4.038864785432816, "rewards/wrapper/std": 12.802475428581237, "step": 25 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.2, "completions/max_terminated_length": 1.2, "completions/mean_length": 1.0049999952316284, "completions/mean_terminated_length": 1.0049999952316284, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.0044411547002220575, "frac_reward_zero_std": 0.9899999976158143, "grad_norm": 4.249159246683121e-09, "kl": 7.156519985198974, "learning_rate": 7.94517142857143e-05, "loss": 0.2897, "num_tokens": 150521.0, "reward": -0.036386170983314516, "reward_std": 0.05145781636238098, "rewards/wrapper/mean": -0.036386170983314516, "rewards/wrapper/std": 0.23012635707855225, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.0051813471502590676, "frac_reward_zero_std": 1.0, "grad_norm": 5.525180313270539e-11, "kl": 7.132610511779785, "learning_rate": 9.315028571428571e-05, "loss": 0.2853, "num_tokens": 156963.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 35 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.005921539600296077, "frac_reward_zero_std": 1.0, "grad_norm": 2.9558577807620168e-12, "kl": 7.412882518768311, "learning_rate": 9.588998436360117e-05, "loss": 0.2965, "num_tokens": 163181.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.006661732050333087, "frac_reward_zero_std": 1.0, "grad_norm": 9.450218385609332e-13, "kl": 7.265022277832031, "learning_rate": 9.588992084075418e-05, "loss": 0.2906, "num_tokens": 169545.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 45 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.007401924500370096, "frac_reward_zero_std": 1.0, "grad_norm": 8.277822871605167e-13, "kl": 7.381829929351807, "learning_rate": 9.588980845427037e-05, "loss": 0.2953, "num_tokens": 175813.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.008142116950407105, "frac_reward_zero_std": 1.0, "grad_norm": 8.313350008393172e-13, "kl": 7.221054649353027, "learning_rate": 9.588964720430247e-05, "loss": 0.2888, "num_tokens": 182541.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 55 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.008882309400444115, "frac_reward_zero_std": 1.0, "grad_norm": 7.247535904753022e-13, "kl": 7.365166473388672, "learning_rate": 9.58894370910696e-05, "loss": 0.2946, "num_tokens": 189103.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.009622501850481125, "frac_reward_zero_std": 1.0, "grad_norm": 6.359357485052897e-13, "kl": 7.176050281524658, "learning_rate": 9.588917811485727e-05, "loss": 0.287, "num_tokens": 195479.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 65 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.010362694300518135, "frac_reward_zero_std": 1.0, "grad_norm": 7.673861546209082e-13, "kl": 7.271660041809082, "learning_rate": 9.588887027601739e-05, "loss": 0.2909, "num_tokens": 201989.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.011102886750555145, "frac_reward_zero_std": 1.0, "grad_norm": 7.958078640513122e-13, "kl": 7.367933368682861, "learning_rate": 9.588851357496828e-05, "loss": 0.2947, "num_tokens": 208223.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 75 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.011843079200592153, "frac_reward_zero_std": 1.0, "grad_norm": 6.750155989720952e-13, "kl": 7.358632755279541, "learning_rate": 9.588810801219468e-05, "loss": 0.2943, "num_tokens": 214185.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.012583271650629163, "frac_reward_zero_std": 1.0, "grad_norm": 6.750155989720952e-13, "kl": 7.351805114746094, "learning_rate": 9.588765358824768e-05, "loss": 0.2941, "num_tokens": 220693.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 85 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.013323464100666173, "frac_reward_zero_std": 1.0, "grad_norm": 7.212008767965017e-13, "kl": 7.173146152496338, "learning_rate": 9.588715030374479e-05, "loss": 0.2869, "num_tokens": 226653.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.014063656550703183, "frac_reward_zero_std": 1.0, "grad_norm": 6.785683126508957e-13, "kl": 7.250439167022705, "learning_rate": 9.58865981593699e-05, "loss": 0.29, "num_tokens": 232801.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 95 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.014803849000740192, "frac_reward_zero_std": 1.0, "grad_norm": 7.105427357601002e-13, "kl": 7.214993858337403, "learning_rate": 9.588599715587337e-05, "loss": 0.2886, "num_tokens": 239087.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.015544041450777202, "frac_reward_zero_std": 1.0, "grad_norm": 6.785683126508957e-13, "kl": 7.39037446975708, "learning_rate": 9.588534729407183e-05, "loss": 0.2956, "num_tokens": 244745.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.01628423390081421, "frac_reward_zero_std": 1.0, "grad_norm": 8.064660050877137e-13, "kl": 7.252386665344238, "learning_rate": 9.588464857484839e-05, "loss": 0.2901, "num_tokens": 251373.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.017024426350851222, "frac_reward_zero_std": 1.0, "grad_norm": 7.851497230149107e-13, "kl": 7.362007808685303, "learning_rate": 9.588390099915251e-05, "loss": 0.2945, "num_tokens": 257771.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.01776461880088823, "frac_reward_zero_std": 1.0, "grad_norm": 7.531752999057062e-13, "kl": 7.429892349243164, "learning_rate": 9.58831045680001e-05, "loss": 0.2972, "num_tokens": 263709.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.018504811250925242, "frac_reward_zero_std": 1.0, "grad_norm": 6.750155989720952e-13, "kl": 7.3208294868469235, "learning_rate": 9.588225928247338e-05, "loss": 0.2928, "num_tokens": 270455.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.01924500370096225, "frac_reward_zero_std": 1.0, "grad_norm": 7.638334409421077e-13, "kl": 7.414208221435547, "learning_rate": 9.5881365143721e-05, "loss": 0.2966, "num_tokens": 276747.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.01998519615099926, "frac_reward_zero_std": 1.0, "grad_norm": 7.567280135845067e-13, "kl": 7.104016399383545, "learning_rate": 9.5880422152958e-05, "loss": 0.2842, "num_tokens": 283469.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.02072538860103627, "frac_reward_zero_std": 1.0, "grad_norm": 7.176481631177012e-13, "kl": 7.234098434448242, "learning_rate": 9.58794303114658e-05, "loss": 0.2894, "num_tokens": 290295.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.02146558105107328, "frac_reward_zero_std": 1.0, "grad_norm": 7.709388682997087e-13, "kl": 7.09038200378418, "learning_rate": 9.587838962059217e-05, "loss": 0.2836, "num_tokens": 296289.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.02220577350111029, "frac_reward_zero_std": 1.0, "grad_norm": 8.242295734817162e-13, "kl": 7.263150310516357, "learning_rate": 9.587730008175132e-05, "loss": 0.2905, "num_tokens": 302929.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.0229459659511473, "frac_reward_zero_std": 1.0, "grad_norm": 6.927791673660977e-13, "kl": 7.336368465423584, "learning_rate": 9.587616169642378e-05, "loss": 0.2935, "num_tokens": 309091.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.023686158401184307, "frac_reward_zero_std": 1.0, "grad_norm": 6.998845947236987e-13, "kl": 6.995298194885254, "learning_rate": 9.58749744661565e-05, "loss": 0.2798, "num_tokens": 315121.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.02442635085122132, "frac_reward_zero_std": 1.0, "grad_norm": 6.714628852932947e-13, "kl": 7.178630065917969, "learning_rate": 9.587373839256279e-05, "loss": 0.2871, "num_tokens": 321249.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 165 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.025166543301258327, "frac_reward_zero_std": 1.0, "grad_norm": 8.206768598029157e-13, "kl": 7.316208839416504, "learning_rate": 9.58724534773223e-05, "loss": 0.2926, "num_tokens": 327963.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.025906735751295335, "frac_reward_zero_std": 1.0, "grad_norm": 6.785683126508957e-13, "kl": 7.494047927856445, "learning_rate": 9.587111972218111e-05, "loss": 0.2998, "num_tokens": 333947.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.026646928201332347, "frac_reward_zero_std": 1.0, "grad_norm": 7.531752999057062e-13, "kl": 7.236764144897461, "learning_rate": 9.586973712895163e-05, "loss": 0.2895, "num_tokens": 340189.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.027387120651369355, "frac_reward_zero_std": 1.0, "grad_norm": 7.425171588693047e-13, "kl": 7.204125022888183, "learning_rate": 9.586830569951264e-05, "loss": 0.2882, "num_tokens": 346513.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.028127313101406367, "frac_reward_zero_std": 1.0, "grad_norm": 6.963318810448982e-13, "kl": 7.235298156738281, "learning_rate": 9.586682543580928e-05, "loss": 0.2894, "num_tokens": 352817.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.028867505551443375, "frac_reward_zero_std": 1.0, "grad_norm": 7.069900220812997e-13, "kl": 7.307924270629883, "learning_rate": 9.586529633985307e-05, "loss": 0.2923, "num_tokens": 359241.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.029607698001480384, "frac_reward_zero_std": 1.0, "grad_norm": 7.673861546209082e-13, "kl": 7.321866607666015, "learning_rate": 9.586371841372186e-05, "loss": 0.2929, "num_tokens": 366023.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.030347890451517395, "frac_reward_zero_std": 1.0, "grad_norm": 7.744915819785092e-13, "kl": 7.239410018920898, "learning_rate": 9.586209165955988e-05, "loss": 0.2896, "num_tokens": 372187.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.031088082901554404, "frac_reward_zero_std": 1.0, "grad_norm": 7.069900220812997e-13, "kl": 7.310018539428711, "learning_rate": 9.586041607957769e-05, "loss": 0.2924, "num_tokens": 378361.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.03182827535159141, "frac_reward_zero_std": 1.0, "grad_norm": 7.247535904753022e-13, "kl": 7.397636413574219, "learning_rate": 9.585869167605221e-05, "loss": 0.2959, "num_tokens": 384715.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.03256846780162842, "frac_reward_zero_std": 1.0, "grad_norm": 7.567280135845067e-13, "kl": 7.372064876556396, "learning_rate": 9.585691845132668e-05, "loss": 0.2949, "num_tokens": 391405.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.033308660251665435, "frac_reward_zero_std": 1.0, "grad_norm": 7.212008767965017e-13, "kl": 7.226515293121338, "learning_rate": 9.585509640781075e-05, "loss": 0.2891, "num_tokens": 397575.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.034048852701702444, "frac_reward_zero_std": 1.0, "grad_norm": 8.313350008393172e-13, "kl": 7.266669654846192, "learning_rate": 9.585322554798035e-05, "loss": 0.2907, "num_tokens": 403857.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.03478904515173945, "frac_reward_zero_std": 1.0, "grad_norm": 7.780442956573097e-13, "kl": 7.295409297943115, "learning_rate": 9.585130587437773e-05, "loss": 0.2918, "num_tokens": 410151.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.03552923760177646, "frac_reward_zero_std": 1.0, "grad_norm": 6.146194664324867e-13, "kl": 7.396674633026123, "learning_rate": 9.584933738961154e-05, "loss": 0.2959, "num_tokens": 416273.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.03626943005181347, "frac_reward_zero_std": 1.0, "grad_norm": 6.536993168992922e-13, "kl": 7.26646785736084, "learning_rate": 9.58473200963567e-05, "loss": 0.2907, "num_tokens": 422301.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.037009622501850484, "frac_reward_zero_std": 1.0, "grad_norm": 7.034373084024992e-13, "kl": 7.283194446563721, "learning_rate": 9.584525399735448e-05, "loss": 0.2913, "num_tokens": 428601.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.03774981495188749, "frac_reward_zero_std": 1.0, "grad_norm": 7.176481631177012e-13, "kl": 7.263091945648194, "learning_rate": 9.584313909541247e-05, "loss": 0.2905, "num_tokens": 435383.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.0384900074019245, "frac_reward_zero_std": 1.0, "grad_norm": 7.389644451905042e-13, "kl": 7.358919715881347, "learning_rate": 9.584097539340459e-05, "loss": 0.2944, "num_tokens": 441713.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.03923019985196151, "frac_reward_zero_std": 1.0, "grad_norm": 7.567280135845067e-13, "kl": 7.283941078186035, "learning_rate": 9.5838762894271e-05, "loss": 0.2914, "num_tokens": 447967.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.03997039230199852, "frac_reward_zero_std": 1.0, "grad_norm": 6.465938895416912e-13, "kl": 7.424037170410156, "learning_rate": 9.58365016010183e-05, "loss": 0.297, "num_tokens": 454351.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.04071058475203553, "frac_reward_zero_std": 1.0, "grad_norm": 6.643574579356937e-13, "kl": 7.295189476013183, "learning_rate": 9.58341915167193e-05, "loss": 0.2918, "num_tokens": 460923.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.04145077720207254, "frac_reward_zero_std": 1.0, "grad_norm": 6.110667527536862e-13, "kl": 7.257896041870117, "learning_rate": 9.583183264451312e-05, "loss": 0.2903, "num_tokens": 467889.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.04219096965210955, "frac_reward_zero_std": 1.0, "grad_norm": 7.105427357601002e-13, "kl": 7.195250797271728, "learning_rate": 9.582942498760521e-05, "loss": 0.2878, "num_tokens": 474219.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.04293116210214656, "frac_reward_zero_std": 1.0, "grad_norm": 7.567280135845067e-13, "kl": 7.254886817932129, "learning_rate": 9.582696854926726e-05, "loss": 0.2902, "num_tokens": 480459.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.043671354552183565, "frac_reward_zero_std": 1.0, "grad_norm": 6.501466032204917e-13, "kl": 7.392390441894531, "learning_rate": 9.582446333283734e-05, "loss": 0.2957, "num_tokens": 486497.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.04441154700222058, "frac_reward_zero_std": 1.0, "grad_norm": 6.750155989720952e-13, "kl": 7.286004543304443, "learning_rate": 9.58219093417197e-05, "loss": 0.2914, "num_tokens": 492353.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.04515173945225759, "frac_reward_zero_std": 1.0, "grad_norm": 8.419931418757187e-13, "kl": 7.3324357032775875, "learning_rate": 9.581930657938494e-05, "loss": 0.2933, "num_tokens": 498641.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.0458919319022946, "frac_reward_zero_std": 1.0, "grad_norm": 7.744915819785092e-13, "kl": 7.113735198974609, "learning_rate": 9.58166550493699e-05, "loss": 0.2845, "num_tokens": 505269.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.046632124352331605, "frac_reward_zero_std": 1.0, "grad_norm": 7.176481631177012e-13, "kl": 7.322626686096191, "learning_rate": 9.58139547552777e-05, "loss": 0.2929, "num_tokens": 511419.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.047372316802368614, "frac_reward_zero_std": 1.0, "grad_norm": 8.135714324453147e-13, "kl": 7.057553958892822, "learning_rate": 9.581120570077772e-05, "loss": 0.2823, "num_tokens": 517853.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.04811250925240563, "frac_reward_zero_std": 1.0, "grad_norm": 8.100187187665142e-13, "kl": 7.399619674682617, "learning_rate": 9.580840788960561e-05, "loss": 0.296, "num_tokens": 524173.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.04885270170244264, "frac_reward_zero_std": 1.0, "grad_norm": 6.856737400084967e-13, "kl": 7.366583061218262, "learning_rate": 9.580556132556327e-05, "loss": 0.2947, "num_tokens": 529933.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.049592894152479645, "frac_reward_zero_std": 1.0, "grad_norm": 7.851497230149107e-13, "kl": 7.400002193450928, "learning_rate": 9.580266601251886e-05, "loss": 0.296, "num_tokens": 536727.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.050333086602516654, "frac_reward_zero_std": 1.0, "grad_norm": 7.247535904753022e-13, "kl": 7.381511974334717, "learning_rate": 9.579972195440675e-05, "loss": 0.2953, "num_tokens": 543063.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.05107327905255366, "frac_reward_zero_std": 1.0, "grad_norm": 7.460698725481052e-13, "kl": 7.14085807800293, "learning_rate": 9.579672915522759e-05, "loss": 0.2856, "num_tokens": 549373.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.05181347150259067, "frac_reward_zero_std": 1.0, "grad_norm": 8.384404281969182e-13, "kl": 7.318427848815918, "learning_rate": 9.579368761904823e-05, "loss": 0.2927, "num_tokens": 555495.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.052553663952627686, "frac_reward_zero_std": 1.0, "grad_norm": 7.389644451905042e-13, "kl": 7.159829998016358, "learning_rate": 9.579059735000177e-05, "loss": 0.2864, "num_tokens": 562127.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.053293856402664694, "frac_reward_zero_std": 1.0, "grad_norm": 8.064660050877137e-13, "kl": 7.242475605010986, "learning_rate": 9.578745835228752e-05, "loss": 0.2897, "num_tokens": 568551.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.0540340488527017, "frac_reward_zero_std": 1.0, "grad_norm": 7.354117315117037e-13, "kl": 7.275189208984375, "learning_rate": 9.578427063017101e-05, "loss": 0.291, "num_tokens": 575167.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.05477424130273871, "frac_reward_zero_std": 1.0, "grad_norm": 6.998845947236987e-13, "kl": 7.2319825172424315, "learning_rate": 9.5781034187984e-05, "loss": 0.2893, "num_tokens": 581399.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.05551443375277572, "frac_reward_zero_std": 1.0, "grad_norm": 7.496225862269057e-13, "kl": 7.255225563049317, "learning_rate": 9.577774903012441e-05, "loss": 0.2902, "num_tokens": 587697.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.056254626202812734, "frac_reward_zero_std": 1.0, "grad_norm": 7.780442956573097e-13, "kl": 7.262093353271484, "learning_rate": 9.577441516105642e-05, "loss": 0.2905, "num_tokens": 594255.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.05699481865284974, "frac_reward_zero_std": 1.0, "grad_norm": 6.750155989720952e-13, "kl": 7.432931327819825, "learning_rate": 9.577103258531032e-05, "loss": 0.2973, "num_tokens": 600797.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.05773501110288675, "frac_reward_zero_std": 1.0, "grad_norm": 6.714628852932947e-13, "kl": 7.2062122344970705, "learning_rate": 9.57676013074827e-05, "loss": 0.2882, "num_tokens": 607227.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.05847520355292376, "frac_reward_zero_std": 1.0, "grad_norm": 7.460698725481052e-13, "kl": 7.419683647155762, "learning_rate": 9.576412133223621e-05, "loss": 0.2968, "num_tokens": 613191.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 395 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.05921539600296077, "frac_reward_zero_std": 1.0, "grad_norm": 7.176481631177012e-13, "kl": 7.279653263092041, "learning_rate": 9.576059266429976e-05, "loss": 0.2912, "num_tokens": 619273.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.05995558845299778, "frac_reward_zero_std": 1.0, "grad_norm": 8.029132914089132e-13, "kl": 7.128233432769775, "learning_rate": 9.575701530846839e-05, "loss": 0.2851, "num_tokens": 625577.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 405 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.06069578090303479, "frac_reward_zero_std": 1.0, "grad_norm": 7.105427357601002e-13, "kl": 7.247030258178711, "learning_rate": 9.575338926960332e-05, "loss": 0.2899, "num_tokens": 631895.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.0614359733530718, "frac_reward_zero_std": 1.0, "grad_norm": 7.567280135845067e-13, "kl": 7.174131107330322, "learning_rate": 9.574971455263189e-05, "loss": 0.287, "num_tokens": 638175.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.06217616580310881, "frac_reward_zero_std": 1.0, "grad_norm": 7.069900220812997e-13, "kl": 7.249692440032959, "learning_rate": 9.574599116254761e-05, "loss": 0.29, "num_tokens": 644621.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.06291635825314582, "frac_reward_zero_std": 1.0, "grad_norm": 8.419931418757187e-13, "kl": 7.101159191131591, "learning_rate": 9.574221910441016e-05, "loss": 0.284, "num_tokens": 650559.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.06365655070318282, "frac_reward_zero_std": 1.0, "grad_norm": 6.643574579356937e-13, "kl": 7.182221412658691, "learning_rate": 9.573839838334531e-05, "loss": 0.2873, "num_tokens": 656631.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.06439674315321983, "frac_reward_zero_std": 1.0, "grad_norm": 7.993605777301127e-13, "kl": 7.46747932434082, "learning_rate": 9.573452900454498e-05, "loss": 0.2987, "num_tokens": 662673.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 435 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.06513693560325684, "frac_reward_zero_std": 1.0, "grad_norm": 8.242295734817162e-13, "kl": 7.33021354675293, "learning_rate": 9.573061097326721e-05, "loss": 0.2932, "num_tokens": 669077.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.06587712805329386, "frac_reward_zero_std": 1.0, "grad_norm": 7.744915819785092e-13, "kl": 7.222026348114014, "learning_rate": 9.572664429483614e-05, "loss": 0.2889, "num_tokens": 675701.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 445 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.06661732050333087, "frac_reward_zero_std": 1.0, "grad_norm": 6.714628852932947e-13, "kl": 7.401155662536621, "learning_rate": 9.572262897464201e-05, "loss": 0.296, "num_tokens": 681589.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.06735751295336788, "frac_reward_zero_std": 1.0, "grad_norm": 6.821210263296962e-13, "kl": 7.416054916381836, "learning_rate": 9.571856501814117e-05, "loss": 0.2966, "num_tokens": 688421.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 455 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.06809770540340489, "frac_reward_zero_std": 1.0, "grad_norm": 8.135714324453147e-13, "kl": 7.351655387878418, "learning_rate": 9.57144524308561e-05, "loss": 0.2941, "num_tokens": 694873.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.0688378978534419, "frac_reward_zero_std": 1.0, "grad_norm": 8.242295734817162e-13, "kl": 7.229579639434815, "learning_rate": 9.571029121837529e-05, "loss": 0.2892, "num_tokens": 701737.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.0695780903034789, "frac_reward_zero_std": 1.0, "grad_norm": 6.323830348264892e-13, "kl": 7.358371925354004, "learning_rate": 9.570608138635335e-05, "loss": 0.2943, "num_tokens": 707865.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.07031828275351591, "frac_reward_zero_std": 1.0, "grad_norm": 8.562039965909207e-13, "kl": 7.399039363861084, "learning_rate": 9.570182294051097e-05, "loss": 0.296, "num_tokens": 714529.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.07105847520355292, "frac_reward_zero_std": 1.0, "grad_norm": 6.465938895416912e-13, "kl": 7.261460590362549, "learning_rate": 9.569751588663486e-05, "loss": 0.2905, "num_tokens": 720797.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.07179866765358993, "frac_reward_zero_std": 1.0, "grad_norm": 6.963318810448982e-13, "kl": 7.2240135192871096, "learning_rate": 9.56931602305778e-05, "loss": 0.289, "num_tokens": 727563.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.07253886010362694, "frac_reward_zero_std": 1.0, "grad_norm": 6.856737400084967e-13, "kl": 7.210353088378906, "learning_rate": 9.568875597825863e-05, "loss": 0.2884, "num_tokens": 734131.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.07327905255366396, "frac_reward_zero_std": 1.0, "grad_norm": 6.927791673660977e-13, "kl": 7.441960048675537, "learning_rate": 9.568430313566222e-05, "loss": 0.2977, "num_tokens": 741029.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "epoch": 0.07401924500370097, "frac_reward_zero_std": 1.0, "grad_norm": 6.714628852932947e-13, "kl": 7.174128246307373, "learning_rate": 9.567980170883944e-05, "loss": 0.287, "num_tokens": 747423.0, "reward": 0.0, "reward_std": 0.0, "rewards/wrapper/mean": 0.0, "rewards/wrapper/std": 0.0, "step": 500 }, { "epoch": 0.07401924500370097, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0, "eval_completions/max_length": 1.0, "eval_completions/max_terminated_length": 1.0, "eval_completions/mean_length": 1.0, "eval_completions/mean_terminated_length": 1.0, "eval_completions/min_length": 1.0, "eval_completions/min_terminated_length": 1.0, "eval_frac_reward_zero_std": 1.0, "eval_kl": 7.285646305084229, "eval_loss": 0.2914258539676666, "eval_num_tokens": 747423.0, "eval_reward": 0.0, "eval_reward_std": 0.0, "eval_rewards/wrapper/mean": 0.0, "eval_rewards/wrapper/std": 0.0, "eval_runtime": 5.2566, "eval_samples_per_second": 38.048, "eval_steps_per_second": 9.512, "step": 500 } ], "logging_steps": 5, "max_steps": 13510, "num_input_tokens_seen": 747423, "num_train_epochs": 2, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 40, "trial_name": null, "trial_params": null }