| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.0005, |
| "eval_steps": 500, |
| "global_step": 50, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8966.0, |
| "completions/max_terminated_length": 8966.0, |
| "completions/mean_length": 7392.1875, |
| "completions/mean_terminated_length": 7392.1875, |
| "completions/min_length": 892.0, |
| "completions/min_terminated_length": 892.0, |
| "entropy": 0.11410779180005193, |
| "epoch": 1e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.795484185218811, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": -0.2193, |
| "num_tokens": 263588.0, |
| "reward": -0.3529798984527588, |
| "reward_std": 0.3798896074295044, |
| "rewards/rollout_reward_func/mean": -0.3529798984527588, |
| "rewards/rollout_reward_func/std": 0.4275760054588318, |
| "sampling/importance_sampling_ratio/max": 1.9381288290023804, |
| "sampling/importance_sampling_ratio/mean": 0.9994939565658569, |
| "sampling/importance_sampling_ratio/min": 0.48449021577835083, |
| "sampling/sampling_logp_difference/max": 0.7246580123901367, |
| "sampling/sampling_logp_difference/mean": 0.013468129560351372, |
| "step": 1, |
| "step_time": 93.399304126 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.11410779180005193, |
| "epoch": 2e-05, |
| "grad_norm": 1.7967665195465088, |
| "kl": 0.0, |
| "learning_rate": 2.8571428571428573e-06, |
| "loss": -0.2193, |
| "step": 2, |
| "step_time": 27.58208410200001 |
| }, |
| { |
| "clip_ratio/high_max": 0.002206696954090148, |
| "clip_ratio/high_mean": 0.001103348477045074, |
| "clip_ratio/low_mean": 0.0021773561311420053, |
| "clip_ratio/low_min": 0.0007183908019214869, |
| "clip_ratio/region_mean": 0.0032807046081870794, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8789.0, |
| "completions/max_terminated_length": 8789.0, |
| "completions/mean_length": 7273.3125, |
| "completions/mean_terminated_length": 7273.3125, |
| "completions/min_length": 1689.0, |
| "completions/min_terminated_length": 1689.0, |
| "entropy": 0.08583545382134616, |
| "epoch": 3e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.1726843118667603, |
| "kl": 0.0017027981803039438, |
| "learning_rate": 5.7142857142857145e-06, |
| "loss": -0.0032, |
| "num_tokens": 523711.0, |
| "reward": -0.3553134799003601, |
| "reward_std": 0.48743048310279846, |
| "rewards/rollout_reward_func/mean": -0.3553134799003601, |
| "rewards/rollout_reward_func/std": 0.5032535195350647, |
| "sampling/importance_sampling_ratio/max": 2.636791944503784, |
| "sampling/importance_sampling_ratio/mean": 1.0005749464035034, |
| "sampling/importance_sampling_ratio/min": 0.6534329652786255, |
| "sampling/sampling_logp_difference/max": 0.9695630073547363, |
| "sampling/sampling_logp_difference/mean": 0.008272429928183556, |
| "step": 3, |
| "step_time": 91.38279614700019 |
| }, |
| { |
| "clip_ratio/high_max": 0.00520685565425083, |
| "clip_ratio/high_mean": 0.002603427827125415, |
| "clip_ratio/low_mean": 0.0017682212928775698, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.004371649120002985, |
| "entropy": 0.08481810381636024, |
| "epoch": 4e-05, |
| "grad_norm": 2.2690632343292236, |
| "kl": 0.0009417296496394556, |
| "learning_rate": 8.571428571428573e-06, |
| "loss": -0.0051, |
| "step": 4, |
| "step_time": 26.98271870500014 |
| }, |
| { |
| "clip_ratio/high_max": 0.004254478495568037, |
| "clip_ratio/high_mean": 0.0021272392477840185, |
| "clip_ratio/low_mean": 0.002160204341635108, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0042874435894191265, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8986.0, |
| "completions/max_terminated_length": 8986.0, |
| "completions/mean_length": 7767.59375, |
| "completions/mean_terminated_length": 7767.59375, |
| "completions/min_length": 300.0, |
| "completions/min_terminated_length": 300.0, |
| "entropy": 0.10082893958315253, |
| "epoch": 5e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3848886489868164, |
| "kl": 0.0014980694850237342, |
| "learning_rate": 1.1428571428571429e-05, |
| "loss": -0.1089, |
| "num_tokens": 799123.0, |
| "reward": -0.4331575036048889, |
| "reward_std": 0.3397783935070038, |
| "rewards/rollout_reward_func/mean": -0.4331575036048889, |
| "rewards/rollout_reward_func/std": 0.37834790349006653, |
| "sampling/importance_sampling_ratio/max": 1.6952898502349854, |
| "sampling/importance_sampling_ratio/mean": 0.9987826347351074, |
| "sampling/importance_sampling_ratio/min": 0.2250514030456543, |
| "sampling/sampling_logp_difference/max": 1.4914264678955078, |
| "sampling/sampling_logp_difference/mean": 0.010691944509744644, |
| "step": 5, |
| "step_time": 94.93945038100003 |
| }, |
| { |
| "clip_ratio/high_max": 0.006353594711981714, |
| "clip_ratio/high_mean": 0.003176797355990857, |
| "clip_ratio/low_mean": 0.0018021696887444705, |
| "clip_ratio/low_min": 0.0007183908019214869, |
| "clip_ratio/region_mean": 0.004978967044735327, |
| "entropy": 0.10040339734405279, |
| "epoch": 6e-05, |
| "grad_norm": 1.3835108280181885, |
| "kl": 0.00129023771660286, |
| "learning_rate": 1.4285714285714285e-05, |
| "loss": -0.1114, |
| "step": 6, |
| "step_time": 27.51428913800021 |
| }, |
| { |
| "clip_ratio/high_max": 0.007119529938790947, |
| "clip_ratio/high_mean": 0.0035597649693954736, |
| "clip_ratio/low_mean": 0.0018344622512813658, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0053942272788845, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9295.0, |
| "completions/max_terminated_length": 9295.0, |
| "completions/mean_length": 8119.90625, |
| "completions/mean_terminated_length": 8119.90625, |
| "completions/min_length": 425.0, |
| "completions/min_terminated_length": 425.0, |
| "entropy": 0.08391103229951113, |
| "epoch": 7e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.6203776597976685, |
| "kl": 0.0014097213361310423, |
| "learning_rate": 1.7142857142857145e-05, |
| "loss": -0.1002, |
| "num_tokens": 1086692.0, |
| "reward": -0.5209137201309204, |
| "reward_std": 0.2494075447320938, |
| "rewards/rollout_reward_func/mean": -0.5209137201309204, |
| "rewards/rollout_reward_func/std": 0.2681976854801178, |
| "sampling/importance_sampling_ratio/max": 1.7586421966552734, |
| "sampling/importance_sampling_ratio/mean": 1.0001193284988403, |
| "sampling/importance_sampling_ratio/min": 0.5591319799423218, |
| "sampling/sampling_logp_difference/max": 0.5813696980476379, |
| "sampling/sampling_logp_difference/mean": 0.010238477028906345, |
| "step": 7, |
| "step_time": 99.45176190400002 |
| }, |
| { |
| "clip_ratio/high_max": 0.0050131616299040616, |
| "clip_ratio/high_mean": 0.0025065808149520308, |
| "clip_ratio/low_mean": 0.0048430965398438275, |
| "clip_ratio/low_min": 0.0014367816038429737, |
| "clip_ratio/region_mean": 0.00734967747121118, |
| "entropy": 0.08137266687117517, |
| "epoch": 8e-05, |
| "grad_norm": 1.417209267616272, |
| "kl": 0.0033996296788245672, |
| "learning_rate": 2e-05, |
| "loss": -0.1028, |
| "step": 8, |
| "step_time": 29.014337608000233 |
| }, |
| { |
| "clip_ratio/high_max": 0.0042058578110300004, |
| "clip_ratio/high_mean": 0.0024501511361449957, |
| "clip_ratio/low_mean": 0.0024779988452792168, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0049281499814242125, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9264.0, |
| "completions/max_terminated_length": 9264.0, |
| "completions/mean_length": 7546.03125, |
| "completions/mean_terminated_length": 7546.03125, |
| "completions/min_length": 860.0, |
| "completions/min_terminated_length": 860.0, |
| "entropy": 0.08567450987175107, |
| "epoch": 9e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.5940141677856445, |
| "kl": 0.010301298352715094, |
| "learning_rate": 2.2857142857142858e-05, |
| "loss": -0.3628, |
| "num_tokens": 1355432.0, |
| "reward": -0.40637093782424927, |
| "reward_std": 0.3687950074672699, |
| "rewards/rollout_reward_func/mean": -0.40637093782424927, |
| "rewards/rollout_reward_func/std": 0.3848126232624054, |
| "sampling/importance_sampling_ratio/max": 1.819852352142334, |
| "sampling/importance_sampling_ratio/mean": 1.0000147819519043, |
| "sampling/importance_sampling_ratio/min": 0.4127380847930908, |
| "sampling/sampling_logp_difference/max": 0.8849420547485352, |
| "sampling/sampling_logp_difference/mean": 0.01255122572183609, |
| "step": 9, |
| "step_time": 92.58362530700037 |
| }, |
| { |
| "clip_ratio/high_max": 0.010624476184602827, |
| "clip_ratio/high_mean": 0.006026733753969893, |
| "clip_ratio/low_mean": 0.01009578033699654, |
| "clip_ratio/low_min": 0.0042697106837295, |
| "clip_ratio/region_mean": 0.01612251391634345, |
| "entropy": 0.08229666040278971, |
| "epoch": 0.0001, |
| "grad_norm": 1.487949252128601, |
| "kl": 0.037161100313824136, |
| "learning_rate": 2.5714285714285714e-05, |
| "loss": -0.3665, |
| "step": 10, |
| "step_time": 28.38116288400022 |
| }, |
| { |
| "clip_ratio/high_max": 0.0049977844464592636, |
| "clip_ratio/high_mean": 0.0024988922232296318, |
| "clip_ratio/low_mean": 0.0011143402953166515, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0036132325185462832, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9244.0, |
| "completions/max_terminated_length": 9244.0, |
| "completions/mean_length": 7200.8125, |
| "completions/mean_terminated_length": 7200.8125, |
| "completions/min_length": 297.0, |
| "completions/min_terminated_length": 297.0, |
| "entropy": 0.06809334014542401, |
| "epoch": 0.00011, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.8031041622161865, |
| "kl": 0.015580801184114534, |
| "learning_rate": 2.857142857142857e-05, |
| "loss": -0.3986, |
| "num_tokens": 1613108.0, |
| "reward": -0.4156220555305481, |
| "reward_std": 0.5291995406150818, |
| "rewards/rollout_reward_func/mean": -0.4156220555305481, |
| "rewards/rollout_reward_func/std": 0.5197964906692505, |
| "sampling/importance_sampling_ratio/max": 2.4048261642456055, |
| "sampling/importance_sampling_ratio/mean": 0.9991490244865417, |
| "sampling/importance_sampling_ratio/min": 0.2525537610054016, |
| "sampling/sampling_logp_difference/max": 1.3761311769485474, |
| "sampling/sampling_logp_difference/mean": 0.009278691373765469, |
| "step": 11, |
| "step_time": 91.42414895300033 |
| }, |
| { |
| "clip_ratio/high_max": 0.00853408733382821, |
| "clip_ratio/high_mean": 0.004985434468835592, |
| "clip_ratio/low_mean": 0.006528367113787681, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.011513801524415612, |
| "entropy": 0.06515359785407782, |
| "epoch": 0.00012, |
| "grad_norm": 0.9123954176902771, |
| "kl": 0.05925005234166747, |
| "learning_rate": 3.142857142857143e-05, |
| "loss": -0.4041, |
| "step": 12, |
| "step_time": 27.261401052000338 |
| }, |
| { |
| "clip_ratio/high_max": 0.004881663771811873, |
| "clip_ratio/high_mean": 0.0027880541456397623, |
| "clip_ratio/low_mean": 0.0037891828105784953, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0065772370144259185, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9011.0, |
| "completions/max_terminated_length": 9011.0, |
| "completions/mean_length": 7481.90625, |
| "completions/mean_terminated_length": 7481.90625, |
| "completions/min_length": 2417.0, |
| "completions/min_terminated_length": 2417.0, |
| "entropy": 0.06863200129009783, |
| "epoch": 0.00013, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.003671407699585, |
| "kl": 0.039789786795154214, |
| "learning_rate": 3.428571428571429e-05, |
| "loss": -0.203, |
| "num_tokens": 1880242.0, |
| "reward": -0.5097812414169312, |
| "reward_std": 0.4184800386428833, |
| "rewards/rollout_reward_func/mean": -0.5097812414169312, |
| "rewards/rollout_reward_func/std": 0.4387035071849823, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 1.0012085437774658, |
| "sampling/importance_sampling_ratio/min": 0.21886371076107025, |
| "sampling/sampling_logp_difference/max": 2.316737651824951, |
| "sampling/sampling_logp_difference/mean": 0.01692187413573265, |
| "step": 13, |
| "step_time": 93.44268519900038 |
| }, |
| { |
| "clip_ratio/high_max": 0.012604236777406186, |
| "clip_ratio/high_mean": 0.006302118388703093, |
| "clip_ratio/low_mean": 0.01540282418136485, |
| "clip_ratio/low_min": 0.003511236049234867, |
| "clip_ratio/region_mean": 0.021704942628275603, |
| "entropy": 0.06422114372253418, |
| "epoch": 0.00014, |
| "grad_norm": 1.1214606761932373, |
| "kl": 0.07548845067503862, |
| "learning_rate": 3.7142857142857143e-05, |
| "loss": -0.2136, |
| "step": 14, |
| "step_time": 26.921327204999443 |
| }, |
| { |
| "clip_ratio/high_max": 0.003742929140571505, |
| "clip_ratio/high_mean": 0.0022186867718119174, |
| "clip_ratio/low_mean": 0.0041773931588977575, |
| "clip_ratio/low_min": 0.0014451349852606654, |
| "clip_ratio/region_mean": 0.0063960799016058445, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9200.0, |
| "completions/max_terminated_length": 9200.0, |
| "completions/mean_length": 7406.59375, |
| "completions/mean_terminated_length": 7406.59375, |
| "completions/min_length": 203.0, |
| "completions/min_terminated_length": 203.0, |
| "entropy": 0.046332313562743366, |
| "epoch": 0.00015, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3658106327056885, |
| "kl": 0.08860734064364806, |
| "learning_rate": 4e-05, |
| "loss": 0.0669, |
| "num_tokens": 2144669.0, |
| "reward": -0.3528895974159241, |
| "reward_std": 0.443051815032959, |
| "rewards/rollout_reward_func/mean": -0.3528895974159241, |
| "rewards/rollout_reward_func/std": 0.48428139090538025, |
| "sampling/importance_sampling_ratio/max": 1.8482800722122192, |
| "sampling/importance_sampling_ratio/mean": 0.9966132640838623, |
| "sampling/importance_sampling_ratio/min": 0.118045873939991, |
| "sampling/sampling_logp_difference/max": 2.1366820335388184, |
| "sampling/sampling_logp_difference/mean": 0.016083233058452606, |
| "step": 15, |
| "step_time": 91.83712665100029 |
| }, |
| { |
| "clip_ratio/high_max": 0.004916455829516053, |
| "clip_ratio/high_mean": 0.0024582279147580266, |
| "clip_ratio/low_mean": 0.008916221850086004, |
| "clip_ratio/low_min": 0.0021388453897088766, |
| "clip_ratio/region_mean": 0.011374449823051691, |
| "entropy": 0.04555429570609704, |
| "epoch": 0.00016, |
| "grad_norm": 1.3114889860153198, |
| "kl": 0.1314735203923192, |
| "learning_rate": 4.2857142857142856e-05, |
| "loss": 0.0618, |
| "step": 16, |
| "step_time": 27.011356363000004 |
| }, |
| { |
| "clip_ratio/high_max": 0.008057098719291389, |
| "clip_ratio/high_mean": 0.00437577159027569, |
| "clip_ratio/low_mean": 0.004278601991245523, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.008654373581521213, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9228.0, |
| "completions/max_terminated_length": 9228.0, |
| "completions/mean_length": 7819.875, |
| "completions/mean_terminated_length": 7819.875, |
| "completions/min_length": 3760.0, |
| "completions/min_terminated_length": 3760.0, |
| "entropy": 0.07362106395885348, |
| "epoch": 0.00017, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.2947819232940674, |
| "kl": 0.13101027125958353, |
| "learning_rate": 4.5714285714285716e-05, |
| "loss": -0.0547, |
| "num_tokens": 2422510.0, |
| "reward": -0.35005444288253784, |
| "reward_std": 0.49921366572380066, |
| "rewards/rollout_reward_func/mean": -0.35005444288253784, |
| "rewards/rollout_reward_func/std": 0.522201657295227, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 1.0008654594421387, |
| "sampling/importance_sampling_ratio/min": 0.04268655180931091, |
| "sampling/sampling_logp_difference/max": 3.1538712978363037, |
| "sampling/sampling_logp_difference/mean": 0.027018871158361435, |
| "step": 17, |
| "step_time": 96.36866129600003 |
| }, |
| { |
| "clip_ratio/high_max": 0.011791770870331675, |
| "clip_ratio/high_mean": 0.0066107546736020595, |
| "clip_ratio/low_mean": 0.01208667631726712, |
| "clip_ratio/low_min": 0.0028746770112775266, |
| "clip_ratio/region_mean": 0.01869743096176535, |
| "entropy": 0.0709565794095397, |
| "epoch": 0.00018, |
| "grad_norm": 3.9747724533081055, |
| "kl": 0.45611770218238235, |
| "learning_rate": 4.8571428571428576e-05, |
| "loss": -0.0527, |
| "step": 18, |
| "step_time": 27.170892336000406 |
| }, |
| { |
| "clip_ratio/high_max": 0.006368288770318031, |
| "clip_ratio/high_mean": 0.0031841443851590157, |
| "clip_ratio/low_mean": 0.0007183908019214869, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0039025351288728416, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9443.0, |
| "completions/max_terminated_length": 9443.0, |
| "completions/mean_length": 7551.3125, |
| "completions/mean_terminated_length": 7551.3125, |
| "completions/min_length": 2300.0, |
| "completions/min_terminated_length": 2300.0, |
| "entropy": 0.05112030985765159, |
| "epoch": 0.00019, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3253093957901, |
| "kl": 0.06368285935604945, |
| "learning_rate": 5.142857142857143e-05, |
| "loss": -0.1504, |
| "num_tokens": 2691652.0, |
| "reward": -0.4411429762840271, |
| "reward_std": 0.4299367666244507, |
| "rewards/rollout_reward_func/mean": -0.4411429762840271, |
| "rewards/rollout_reward_func/std": 0.4561052918434143, |
| "sampling/importance_sampling_ratio/max": 1.6581709384918213, |
| "sampling/importance_sampling_ratio/mean": 0.997830867767334, |
| "sampling/importance_sampling_ratio/min": 0.07465386390686035, |
| "sampling/sampling_logp_difference/max": 2.594892978668213, |
| "sampling/sampling_logp_difference/mean": 0.013442842289805412, |
| "step": 19, |
| "step_time": 93.42405358899964 |
| }, |
| { |
| "clip_ratio/high_max": 0.01606024947250262, |
| "clip_ratio/high_mean": 0.008740443765418604, |
| "clip_ratio/low_mean": 0.006419806071789935, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.015160249720793217, |
| "entropy": 0.053090699831955135, |
| "epoch": 0.0002, |
| "grad_norm": 1.2332507371902466, |
| "kl": 0.1118333391204942, |
| "learning_rate": 5.428571428571428e-05, |
| "loss": -0.156, |
| "step": 20, |
| "step_time": 27.701954217999173 |
| }, |
| { |
| "clip_ratio/high_max": 0.005651562649291009, |
| "clip_ratio/high_mean": 0.0028257813246455044, |
| "clip_ratio/low_mean": 0.0027939277642872185, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.005619709118036553, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9189.0, |
| "completions/max_terminated_length": 9189.0, |
| "completions/mean_length": 7761.09375, |
| "completions/mean_terminated_length": 7761.09375, |
| "completions/min_length": 1417.0, |
| "completions/min_terminated_length": 1417.0, |
| "entropy": 0.06741820118622854, |
| "epoch": 0.00021, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 3.6862173080444336, |
| "kl": 0.4830274658452254, |
| "learning_rate": 5.714285714285714e-05, |
| "loss": -0.2263, |
| "num_tokens": 2967426.0, |
| "reward": -0.28399354219436646, |
| "reward_std": 0.4300430417060852, |
| "rewards/rollout_reward_func/mean": -0.28399354219436646, |
| "rewards/rollout_reward_func/std": 0.5347884297370911, |
| "sampling/importance_sampling_ratio/max": 2.1812915802001953, |
| "sampling/importance_sampling_ratio/mean": 0.997977614402771, |
| "sampling/importance_sampling_ratio/min": 0.4058607518672943, |
| "sampling/sampling_logp_difference/max": 0.9017452001571655, |
| "sampling/sampling_logp_difference/mean": 0.014318476431071758, |
| "step": 21, |
| "step_time": 94.09382517800077 |
| }, |
| { |
| "clip_ratio/high_max": 0.009377924376167357, |
| "clip_ratio/high_mean": 0.006085653760237619, |
| "clip_ratio/low_mean": 0.012686202622717246, |
| "clip_ratio/low_min": 0.002905860950704664, |
| "clip_ratio/region_mean": 0.018771856353851035, |
| "entropy": 0.07064696005545557, |
| "epoch": 0.00022, |
| "grad_norm": 6.065097332000732, |
| "kl": 0.08845078520243987, |
| "learning_rate": 6e-05, |
| "loss": -0.2227, |
| "step": 22, |
| "step_time": 27.40954218999991 |
| }, |
| { |
| "clip_ratio/high_max": 0.0028251263429410756, |
| "clip_ratio/high_mean": 0.0017597853729967028, |
| "clip_ratio/low_mean": 0.0020223904284648597, |
| "clip_ratio/low_min": 0.0005252101109363139, |
| "clip_ratio/region_mean": 0.0037821758014615625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8721.0, |
| "completions/max_terminated_length": 8721.0, |
| "completions/mean_length": 7136.59375, |
| "completions/mean_terminated_length": 7136.59375, |
| "completions/min_length": 959.0, |
| "completions/min_terminated_length": 959.0, |
| "entropy": 0.06426455150358379, |
| "epoch": 0.00023, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.0993857383728027, |
| "kl": 0.1524171840865165, |
| "learning_rate": 6.285714285714286e-05, |
| "loss": -0.0482, |
| "num_tokens": 3222856.0, |
| "reward": -0.365644633769989, |
| "reward_std": 0.5254338979721069, |
| "rewards/rollout_reward_func/mean": -0.365644633769989, |
| "rewards/rollout_reward_func/std": 0.5246545076370239, |
| "sampling/importance_sampling_ratio/max": 2.320268392562866, |
| "sampling/importance_sampling_ratio/mean": 0.9996501803398132, |
| "sampling/importance_sampling_ratio/min": 0.09908817708492279, |
| "sampling/sampling_logp_difference/max": 2.3117451667785645, |
| "sampling/sampling_logp_difference/mean": 0.011567876674234867, |
| "step": 23, |
| "step_time": 89.85609540400014 |
| }, |
| { |
| "clip_ratio/high_max": 0.009038183547090739, |
| "clip_ratio/high_mean": 0.005584432714385912, |
| "clip_ratio/low_mean": 0.005201028659939766, |
| "clip_ratio/low_min": 0.0015668768319301307, |
| "clip_ratio/region_mean": 0.010785461287014186, |
| "entropy": 0.06729507446289062, |
| "epoch": 0.00024, |
| "grad_norm": 1.5627487897872925, |
| "kl": 0.1648054391844198, |
| "learning_rate": 6.571428571428571e-05, |
| "loss": -0.0548, |
| "step": 24, |
| "step_time": 25.88749120899979 |
| }, |
| { |
| "clip_ratio/high_max": 0.005618693074211478, |
| "clip_ratio/high_mean": 0.002809346537105739, |
| "clip_ratio/low_mean": 0.001929406455019489, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.004738752992125228, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8856.0, |
| "completions/max_terminated_length": 8856.0, |
| "completions/mean_length": 7467.65625, |
| "completions/mean_terminated_length": 7467.65625, |
| "completions/min_length": 402.0, |
| "completions/min_terminated_length": 402.0, |
| "entropy": 0.07340177637524903, |
| "epoch": 0.00025, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.2772353887557983, |
| "kl": 0.0859682586742565, |
| "learning_rate": 6.857142857142858e-05, |
| "loss": -0.2314, |
| "num_tokens": 3488411.0, |
| "reward": -0.3830329477787018, |
| "reward_std": 0.5107574462890625, |
| "rewards/rollout_reward_func/mean": -0.3830329477787018, |
| "rewards/rollout_reward_func/std": 0.5173285007476807, |
| "sampling/importance_sampling_ratio/max": 2.1774182319641113, |
| "sampling/importance_sampling_ratio/mean": 0.999387264251709, |
| "sampling/importance_sampling_ratio/min": 0.29978206753730774, |
| "sampling/sampling_logp_difference/max": 1.2046995162963867, |
| "sampling/sampling_logp_difference/mean": 0.011983634904026985, |
| "step": 25, |
| "step_time": 92.61141428400015 |
| }, |
| { |
| "clip_ratio/high_max": 0.014493243885226548, |
| "clip_ratio/high_mean": 0.008288288547191769, |
| "clip_ratio/low_mean": 0.013486813055351377, |
| "clip_ratio/low_min": 0.0028249489841982722, |
| "clip_ratio/region_mean": 0.021775101457023993, |
| "entropy": 0.0781157671008259, |
| "epoch": 0.00026, |
| "grad_norm": 0.8266052603721619, |
| "kl": 0.08862257207510993, |
| "learning_rate": 7.142857142857143e-05, |
| "loss": -0.2407, |
| "step": 26, |
| "step_time": 26.541104049000296 |
| }, |
| { |
| "clip_ratio/high_max": 0.0022112573496997356, |
| "clip_ratio/high_mean": 0.001456752244848758, |
| "clip_ratio/low_mean": 0.0013966916594654322, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.00285344390431419, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9222.0, |
| "completions/max_terminated_length": 9222.0, |
| "completions/mean_length": 7990.125, |
| "completions/mean_terminated_length": 7990.125, |
| "completions/min_length": 2192.0, |
| "completions/min_terminated_length": 2192.0, |
| "entropy": 0.08974491525441408, |
| "epoch": 0.00027, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.0755014419555664, |
| "kl": 0.08955412585055456, |
| "learning_rate": 7.428571428571429e-05, |
| "loss": -0.0612, |
| "num_tokens": 3770720.0, |
| "reward": -0.1771392822265625, |
| "reward_std": 0.4869571924209595, |
| "rewards/rollout_reward_func/mean": -0.1771392822265625, |
| "rewards/rollout_reward_func/std": 0.46993595361709595, |
| "sampling/importance_sampling_ratio/max": 1.7862199544906616, |
| "sampling/importance_sampling_ratio/mean": 1.0007758140563965, |
| "sampling/importance_sampling_ratio/min": 0.294488787651062, |
| "sampling/sampling_logp_difference/max": 1.2225143909454346, |
| "sampling/sampling_logp_difference/mean": 0.011052686721086502, |
| "step": 27, |
| "step_time": 95.3786270669998 |
| }, |
| { |
| "clip_ratio/high_max": 0.012634392420295626, |
| "clip_ratio/high_mean": 0.00806891280808486, |
| "clip_ratio/low_mean": 0.005300049553625286, |
| "clip_ratio/low_min": 0.0014046717551536858, |
| "clip_ratio/region_mean": 0.013368962419917807, |
| "entropy": 0.09402668941766024, |
| "epoch": 0.00028, |
| "grad_norm": 0.9924318790435791, |
| "kl": 0.0753595522546675, |
| "learning_rate": 7.714285714285715e-05, |
| "loss": -0.0693, |
| "step": 28, |
| "step_time": 27.647992973000328 |
| }, |
| { |
| "clip_ratio/high_max": 0.0035870341234840453, |
| "clip_ratio/high_mean": 0.002140739292372018, |
| "clip_ratio/low_mean": 0.0017440025694668293, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.003884741832735017, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9067.0, |
| "completions/max_terminated_length": 9067.0, |
| "completions/mean_length": 7836.9375, |
| "completions/mean_terminated_length": 7836.9375, |
| "completions/min_length": 1928.0, |
| "completions/min_terminated_length": 1928.0, |
| "entropy": 0.09281386900693178, |
| "epoch": 0.00029, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.068334937095642, |
| "kl": 0.0688294384162873, |
| "learning_rate": 8e-05, |
| "loss": 0.0442, |
| "num_tokens": 4047979.0, |
| "reward": -0.25341448187828064, |
| "reward_std": 0.49919599294662476, |
| "rewards/rollout_reward_func/mean": -0.25341448187828064, |
| "rewards/rollout_reward_func/std": 0.500785231590271, |
| "sampling/importance_sampling_ratio/max": 1.868089199066162, |
| "sampling/importance_sampling_ratio/mean": 1.0002467632293701, |
| "sampling/importance_sampling_ratio/min": 0.5016597509384155, |
| "sampling/sampling_logp_difference/max": 0.6898331642150879, |
| "sampling/sampling_logp_difference/mean": 0.009339751675724983, |
| "step": 29, |
| "step_time": 94.94639882799993 |
| }, |
| { |
| "clip_ratio/high_max": 0.015361635130830109, |
| "clip_ratio/high_mean": 0.0076808175654150546, |
| "clip_ratio/low_mean": 0.007387197489151731, |
| "clip_ratio/low_min": 0.0006944444612599909, |
| "clip_ratio/region_mean": 0.015068015054566786, |
| "entropy": 0.09539854992181063, |
| "epoch": 0.0003, |
| "grad_norm": 0.7568414807319641, |
| "kl": 0.06834399094805121, |
| "learning_rate": 8.285714285714287e-05, |
| "loss": 0.0376, |
| "step": 30, |
| "step_time": 26.633041288999266 |
| }, |
| { |
| "clip_ratio/high_max": 0.004851312725804746, |
| "clip_ratio/high_mean": 0.002425656362902373, |
| "clip_ratio/low_mean": 0.003376906446646899, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.00580256269313395, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9536.0, |
| "completions/max_terminated_length": 9536.0, |
| "completions/mean_length": 8069.3125, |
| "completions/mean_terminated_length": 8069.3125, |
| "completions/min_length": 2100.0, |
| "completions/min_terminated_length": 2100.0, |
| "entropy": 0.1009219279512763, |
| "epoch": 0.00031, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.0075691938400269, |
| "kl": 0.06613797519821674, |
| "learning_rate": 8.571428571428571e-05, |
| "loss": 0.0875, |
| "num_tokens": 4332991.0, |
| "reward": -0.2427319884300232, |
| "reward_std": 0.5179601907730103, |
| "rewards/rollout_reward_func/mean": -0.2427319884300232, |
| "rewards/rollout_reward_func/std": 0.5765778422355652, |
| "sampling/importance_sampling_ratio/max": 1.7835460901260376, |
| "sampling/importance_sampling_ratio/mean": 0.9999290704727173, |
| "sampling/importance_sampling_ratio/min": 0.6331738829612732, |
| "sampling/sampling_logp_difference/max": 0.5786035060882568, |
| "sampling/sampling_logp_difference/mean": 0.009435810148715973, |
| "step": 31, |
| "step_time": 94.81605570200009 |
| }, |
| { |
| "clip_ratio/high_max": 0.017235137231182307, |
| "clip_ratio/high_mean": 0.008985215710708871, |
| "clip_ratio/low_mean": 0.012812130036763847, |
| "clip_ratio/low_min": 0.0027777778450399637, |
| "clip_ratio/region_mean": 0.021797345718368888, |
| "entropy": 0.10036895563825965, |
| "epoch": 0.00032, |
| "grad_norm": 0.7584834098815918, |
| "kl": 0.11454511061310768, |
| "learning_rate": 8.857142857142857e-05, |
| "loss": 0.0788, |
| "step": 32, |
| "step_time": 28.761855419000312 |
| }, |
| { |
| "clip_ratio/high_max": 0.0031494522700086236, |
| "clip_ratio/high_mean": 0.0015747261350043118, |
| "clip_ratio/low_mean": 0.0024462497676722705, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.004020975931780413, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9354.0, |
| "completions/max_terminated_length": 9354.0, |
| "completions/mean_length": 7928.5, |
| "completions/mean_terminated_length": 7928.5, |
| "completions/min_length": 3150.0, |
| "completions/min_terminated_length": 3150.0, |
| "entropy": 0.1094990405254066, |
| "epoch": 0.00033, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.8852567672729492, |
| "kl": 0.0793593015987426, |
| "learning_rate": 9.142857142857143e-05, |
| "loss": 0.1924, |
| "num_tokens": 4613150.0, |
| "reward": -0.1065119206905365, |
| "reward_std": 0.6632397174835205, |
| "rewards/rollout_reward_func/mean": -0.1065119206905365, |
| "rewards/rollout_reward_func/std": 0.6485887765884399, |
| "sampling/importance_sampling_ratio/max": 1.593437671661377, |
| "sampling/importance_sampling_ratio/mean": 0.9991821050643921, |
| "sampling/importance_sampling_ratio/min": 0.5760508179664612, |
| "sampling/sampling_logp_difference/max": 0.551559329032898, |
| "sampling/sampling_logp_difference/mean": 0.0090299928560853, |
| "step": 33, |
| "step_time": 94.56970139799978 |
| }, |
| { |
| "clip_ratio/high_max": 0.010831523861270398, |
| "clip_ratio/high_mean": 0.006442332087317482, |
| "clip_ratio/low_mean": 0.014884777745464817, |
| "clip_ratio/low_min": 0.004198966489639133, |
| "clip_ratio/region_mean": 0.02132710992009379, |
| "entropy": 0.1035804757848382, |
| "epoch": 0.00034, |
| "grad_norm": 6.932408332824707, |
| "kl": 0.36925772204995155, |
| "learning_rate": 9.428571428571429e-05, |
| "loss": 0.19, |
| "step": 34, |
| "step_time": 28.189770212999974 |
| }, |
| { |
| "clip_ratio/high_max": 0.004503837670199573, |
| "clip_ratio/high_mean": 0.0022519188350997865, |
| "clip_ratio/low_mean": 0.001957070722710341, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.004208989586913958, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9253.0, |
| "completions/max_terminated_length": 9253.0, |
| "completions/mean_length": 8121.40625, |
| "completions/mean_terminated_length": 8121.40625, |
| "completions/min_length": 3211.0, |
| "completions/min_terminated_length": 3211.0, |
| "entropy": 0.08986494969576597, |
| "epoch": 0.00035, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.0189476013183594, |
| "kl": 0.11070865276269615, |
| "learning_rate": 9.714285714285715e-05, |
| "loss": -0.1358, |
| "num_tokens": 4899944.0, |
| "reward": -0.3683607578277588, |
| "reward_std": 0.39094337821006775, |
| "rewards/rollout_reward_func/mean": -0.3683607578277588, |
| "rewards/rollout_reward_func/std": 0.3886735141277313, |
| "sampling/importance_sampling_ratio/max": 1.678568720817566, |
| "sampling/importance_sampling_ratio/mean": 0.9991096258163452, |
| "sampling/importance_sampling_ratio/min": 0.2759600579738617, |
| "sampling/sampling_logp_difference/max": 1.287499189376831, |
| "sampling/sampling_logp_difference/mean": 0.009727813303470612, |
| "step": 35, |
| "step_time": 95.49759846299912 |
| }, |
| { |
| "clip_ratio/high_max": 0.016388497198931873, |
| "clip_ratio/high_mean": 0.008888693118933588, |
| "clip_ratio/low_mean": 0.012551680905744433, |
| "clip_ratio/low_min": 0.002083333383779973, |
| "clip_ratio/region_mean": 0.021440374111989513, |
| "entropy": 0.08508480247110128, |
| "epoch": 0.00036, |
| "grad_norm": 0.7093645334243774, |
| "kl": 0.13321141549386084, |
| "learning_rate": 0.0001, |
| "loss": -0.142, |
| "step": 36, |
| "step_time": 28.9255163910002 |
| }, |
| { |
| "clip_ratio/high_max": 0.0037438530707731843, |
| "clip_ratio/high_mean": 0.0018719265353865921, |
| "clip_ratio/low_mean": 0.0017557953542564064, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0036277218896429986, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8904.0, |
| "completions/max_terminated_length": 8904.0, |
| "completions/mean_length": 7886.75, |
| "completions/mean_terminated_length": 7886.75, |
| "completions/min_length": 2730.0, |
| "completions/min_terminated_length": 2730.0, |
| "entropy": 0.0856572980992496, |
| "epoch": 0.00037, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.8132036328315735, |
| "kl": 0.10761717008426785, |
| "learning_rate": 9.918053502751771e-05, |
| "loss": -0.0071, |
| "num_tokens": 5179039.0, |
| "reward": -0.2326236367225647, |
| "reward_std": 0.5471592545509338, |
| "rewards/rollout_reward_func/mean": -0.2326236367225647, |
| "rewards/rollout_reward_func/std": 0.5504636168479919, |
| "sampling/importance_sampling_ratio/max": 1.7522045373916626, |
| "sampling/importance_sampling_ratio/mean": 0.9999704360961914, |
| "sampling/importance_sampling_ratio/min": 0.6301739811897278, |
| "sampling/sampling_logp_difference/max": 0.5608747005462646, |
| "sampling/sampling_logp_difference/mean": 0.00934191420674324, |
| "step": 37, |
| "step_time": 91.66157414699956 |
| }, |
| { |
| "clip_ratio/high_max": 0.009254805103410035, |
| "clip_ratio/high_mean": 0.004974624811438844, |
| "clip_ratio/low_mean": 0.0146892913908232, |
| "clip_ratio/low_min": 0.002091136120725423, |
| "clip_ratio/region_mean": 0.019663916405988857, |
| "entropy": 0.08511333051137626, |
| "epoch": 0.00038, |
| "grad_norm": 1.016090750694275, |
| "kl": 0.13888527592644095, |
| "learning_rate": 9.675795466159755e-05, |
| "loss": -0.0123, |
| "step": 38, |
| "step_time": 27.996872589999384 |
| }, |
| { |
| "clip_ratio/high_max": 0.004166666767559946, |
| "clip_ratio/high_mean": 0.002083333383779973, |
| "clip_ratio/low_mean": 0.003956988512072712, |
| "clip_ratio/low_min": 0.0013966916594654322, |
| "clip_ratio/region_mean": 0.006040321895852685, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9174.0, |
| "completions/max_terminated_length": 9174.0, |
| "completions/mean_length": 7773.78125, |
| "completions/mean_terminated_length": 7773.78125, |
| "completions/min_length": 644.0, |
| "completions/min_terminated_length": 644.0, |
| "entropy": 0.08465631771832705, |
| "epoch": 0.00039, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.2169103622436523, |
| "kl": 0.08935846015810966, |
| "learning_rate": 9.283813728906054e-05, |
| "loss": -0.1228, |
| "num_tokens": 5455087.0, |
| "reward": -0.23303870856761932, |
| "reward_std": 0.587963879108429, |
| "rewards/rollout_reward_func/mean": -0.23303870856761932, |
| "rewards/rollout_reward_func/std": 0.6264134645462036, |
| "sampling/importance_sampling_ratio/max": 2.5134615898132324, |
| "sampling/importance_sampling_ratio/mean": 1.000549077987671, |
| "sampling/importance_sampling_ratio/min": 0.555368959903717, |
| "sampling/sampling_logp_difference/max": 0.9216609001159668, |
| "sampling/sampling_logp_difference/mean": 0.009383916854858398, |
| "step": 39, |
| "step_time": 87.98878976900005 |
| }, |
| { |
| "clip_ratio/high_max": 0.007900029362645, |
| "clip_ratio/high_mean": 0.004326520691392943, |
| "clip_ratio/low_mean": 0.011018106888514012, |
| "clip_ratio/low_min": 0.0018596546142362058, |
| "clip_ratio/region_mean": 0.015344627550803125, |
| "entropy": 0.08598045213147998, |
| "epoch": 0.0004, |
| "grad_norm": 1.855177640914917, |
| "kl": 0.08451172313652933, |
| "learning_rate": 8.759239773845718e-05, |
| "loss": -0.1252, |
| "step": 40, |
| "step_time": 28.765726263999568 |
| }, |
| { |
| "clip_ratio/high_max": 0.005807758076116443, |
| "clip_ratio/high_mean": 0.003251101268688217, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.003251101268688217, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8897.0, |
| "completions/max_terminated_length": 8897.0, |
| "completions/mean_length": 7154.53125, |
| "completions/mean_terminated_length": 7154.53125, |
| "completions/min_length": 2028.0, |
| "completions/min_terminated_length": 2028.0, |
| "entropy": 0.07370180264115334, |
| "epoch": 0.00041, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.6197995543479919, |
| "kl": 0.08875341480597854, |
| "learning_rate": 8.125000000000001e-05, |
| "loss": -0.0075, |
| "num_tokens": 5710693.0, |
| "reward": -0.11052435636520386, |
| "reward_std": 0.6305065155029297, |
| "rewards/rollout_reward_func/mean": -0.11052435636520386, |
| "rewards/rollout_reward_func/std": 0.6378242373466492, |
| "sampling/importance_sampling_ratio/max": 1.3583569526672363, |
| "sampling/importance_sampling_ratio/mean": 0.9987045526504517, |
| "sampling/importance_sampling_ratio/min": 0.3902639150619507, |
| "sampling/sampling_logp_difference/max": 0.940932035446167, |
| "sampling/sampling_logp_difference/mean": 0.007857292890548706, |
| "step": 41, |
| "step_time": 89.4707287680003 |
| }, |
| { |
| "clip_ratio/high_max": 0.011104670935310423, |
| "clip_ratio/high_mean": 0.005552335467655212, |
| "clip_ratio/low_mean": 0.0056608943559695035, |
| "clip_ratio/low_min": 0.0024853801587596536, |
| "clip_ratio/region_mean": 0.011213229881832376, |
| "entropy": 0.07147298031486571, |
| "epoch": 0.00042, |
| "grad_norm": 0.532979428768158, |
| "kl": 0.1001066651660949, |
| "learning_rate": 7.408813728906053e-05, |
| "loss": -0.0097, |
| "step": 42, |
| "step_time": 27.083737006999854 |
| }, |
| { |
| "clip_ratio/high_max": 0.002190170984249562, |
| "clip_ratio/high_mean": 0.001095085492124781, |
| "clip_ratio/low_mean": 0.0017723122437018901, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.002867397735826671, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9047.0, |
| "completions/max_terminated_length": 9047.0, |
| "completions/mean_length": 7559.5625, |
| "completions/mean_terminated_length": 7559.5625, |
| "completions/min_length": 714.0, |
| "completions/min_terminated_length": 714.0, |
| "entropy": 0.07850536378100514, |
| "epoch": 0.00043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.6568584442138672, |
| "kl": 0.13246144936420023, |
| "learning_rate": 6.6419817372537e-05, |
| "loss": -0.164, |
| "num_tokens": 5979343.0, |
| "reward": -0.18616318702697754, |
| "reward_std": 0.5167393088340759, |
| "rewards/rollout_reward_func/mean": -0.18616318702697754, |
| "rewards/rollout_reward_func/std": 0.5579463243484497, |
| "sampling/importance_sampling_ratio/max": 1.8209000825881958, |
| "sampling/importance_sampling_ratio/mean": 1.0000110864639282, |
| "sampling/importance_sampling_ratio/min": 0.5961325764656067, |
| "sampling/sampling_logp_difference/max": 0.5993309020996094, |
| "sampling/sampling_logp_difference/mean": 0.007760843262076378, |
| "step": 43, |
| "step_time": 89.20096494900008 |
| }, |
| { |
| "clip_ratio/high_max": 0.011211407894734293, |
| "clip_ratio/high_mean": 0.0063001484086271375, |
| "clip_ratio/low_mean": 0.00652579654706642, |
| "clip_ratio/low_min": 0.0014211886445991695, |
| "clip_ratio/region_mean": 0.012825944955693558, |
| "entropy": 0.07914698123931885, |
| "epoch": 0.00044, |
| "grad_norm": 0.4922528564929962, |
| "kl": 0.1358487531542778, |
| "learning_rate": 5.858018262746301e-05, |
| "loss": -0.1705, |
| "step": 44, |
| "step_time": 28.31045281400202 |
| }, |
| { |
| "clip_ratio/high_max": 0.0007102272938936949, |
| "clip_ratio/high_mean": 0.00035511364694684744, |
| "clip_ratio/low_mean": 0.0017025089473463595, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.002057622594293207, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8960.0, |
| "completions/max_terminated_length": 8960.0, |
| "completions/mean_length": 7259.9375, |
| "completions/mean_terminated_length": 7259.9375, |
| "completions/min_length": 1793.0, |
| "completions/min_terminated_length": 1793.0, |
| "entropy": 0.0675961917731911, |
| "epoch": 0.00045, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.4965215027332306, |
| "kl": 0.12192867766134441, |
| "learning_rate": 5.0911862710939485e-05, |
| "loss": 0.0928, |
| "num_tokens": 6238685.0, |
| "reward": -0.19794124364852905, |
| "reward_std": 0.6908308267593384, |
| "rewards/rollout_reward_func/mean": -0.19794124364852905, |
| "rewards/rollout_reward_func/std": 0.6924669742584229, |
| "sampling/importance_sampling_ratio/max": 1.7397383451461792, |
| "sampling/importance_sampling_ratio/mean": 1.0005090236663818, |
| "sampling/importance_sampling_ratio/min": 0.6195511817932129, |
| "sampling/sampling_logp_difference/max": 0.5537347793579102, |
| "sampling/sampling_logp_difference/mean": 0.006288852542638779, |
| "step": 45, |
| "step_time": 88.08988413000043 |
| }, |
| { |
| "clip_ratio/high_max": 0.0034880051389336586, |
| "clip_ratio/high_mean": 0.0017440025694668293, |
| "clip_ratio/low_mean": 0.0030215850565582514, |
| "clip_ratio/low_min": 0.0006944444612599909, |
| "clip_ratio/region_mean": 0.004765587626025081, |
| "entropy": 0.06737200380302966, |
| "epoch": 0.00046, |
| "grad_norm": 0.537150502204895, |
| "kl": 0.12145327543839812, |
| "learning_rate": 4.375000000000001e-05, |
| "loss": 0.092, |
| "step": 46, |
| "step_time": 27.73081736800077 |
| }, |
| { |
| "clip_ratio/high_max": 0.001753766497131437, |
| "clip_ratio/high_mean": 0.0012241055082995445, |
| "clip_ratio/low_mean": 0.001392790290992707, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0026168957992922515, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9253.0, |
| "completions/max_terminated_length": 9253.0, |
| "completions/mean_length": 7692.15625, |
| "completions/mean_terminated_length": 7692.15625, |
| "completions/min_length": 293.0, |
| "completions/min_terminated_length": 293.0, |
| "entropy": 0.0828928763512522, |
| "epoch": 0.00047, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.5421693921089172, |
| "kl": 0.13829960371367633, |
| "learning_rate": 3.740760226154283e-05, |
| "loss": 0.1124, |
| "num_tokens": 6512018.0, |
| "reward": -0.13337820768356323, |
| "reward_std": 0.6952483057975769, |
| "rewards/rollout_reward_func/mean": -0.13337820768356323, |
| "rewards/rollout_reward_func/std": 0.6867809295654297, |
| "sampling/importance_sampling_ratio/max": 1.9118043184280396, |
| "sampling/importance_sampling_ratio/mean": 1.000080943107605, |
| "sampling/importance_sampling_ratio/min": 0.6484359502792358, |
| "sampling/sampling_logp_difference/max": 0.6480474472045898, |
| "sampling/sampling_logp_difference/mean": 0.007844729349017143, |
| "step": 47, |
| "step_time": 89.50516965200086 |
| }, |
| { |
| "clip_ratio/high_max": 0.0034878277801908553, |
| "clip_ratio/high_mean": 0.002519218367524445, |
| "clip_ratio/low_mean": 0.00764669178170152, |
| "clip_ratio/low_min": 0.0013888889225199819, |
| "clip_ratio/region_mean": 0.010165910149225965, |
| "entropy": 0.08108305186033249, |
| "epoch": 0.00048, |
| "grad_norm": 0.5435065031051636, |
| "kl": 0.15144605189561844, |
| "learning_rate": 3.2161862710939476e-05, |
| "loss": 0.1097, |
| "step": 48, |
| "step_time": 28.342568254000525 |
| }, |
| { |
| "clip_ratio/high_max": 0.004411048837937415, |
| "clip_ratio/high_mean": 0.0022055244189687073, |
| "clip_ratio/low_mean": 0.0010455680603627115, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.003251092479331419, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9355.0, |
| "completions/max_terminated_length": 9355.0, |
| "completions/mean_length": 7641.09375, |
| "completions/mean_terminated_length": 7641.09375, |
| "completions/min_length": 395.0, |
| "completions/min_terminated_length": 395.0, |
| "entropy": 0.06896222359500825, |
| "epoch": 0.00049, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.7906138300895691, |
| "kl": 0.17947973730042577, |
| "learning_rate": 2.8242045338402468e-05, |
| "loss": 0.0025, |
| "num_tokens": 6783110.0, |
| "reward": -0.24048486351966858, |
| "reward_std": 0.5240083932876587, |
| "rewards/rollout_reward_func/mean": -0.24048486351966858, |
| "rewards/rollout_reward_func/std": 0.6190164685249329, |
| "sampling/importance_sampling_ratio/max": 1.339056372642517, |
| "sampling/importance_sampling_ratio/mean": 1.000028371810913, |
| "sampling/importance_sampling_ratio/min": 0.6448177099227905, |
| "sampling/sampling_logp_difference/max": 0.43878769874572754, |
| "sampling/sampling_logp_difference/mean": 0.0061599984765052795, |
| "step": 49, |
| "step_time": 91.6421234169984 |
| }, |
| { |
| "clip_ratio/high_max": 0.00502964504994452, |
| "clip_ratio/high_mean": 0.00251482252497226, |
| "clip_ratio/low_mean": 0.00382724724477157, |
| "clip_ratio/low_min": 0.0006944444612599909, |
| "clip_ratio/region_mean": 0.00634206976974383, |
| "entropy": 0.0660756464349106, |
| "epoch": 0.0005, |
| "grad_norm": 0.6092560887336731, |
| "kl": 0.1758871516212821, |
| "learning_rate": 2.5819464972482288e-05, |
| "loss": -0.0001, |
| "step": 50, |
| "step_time": 28.18612482699973 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 50, |
| "num_input_tokens_seen": 6783110, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
