| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.975609756097561, | |
| "eval_steps": 1, | |
| "global_step": 90, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 411.93750762939453, | |
| "epoch": 0.032520325203252036, | |
| "grad_norm": 0.041010234504938126, | |
| "kl": 0.0, | |
| "learning_rate": 3.333333333333333e-07, | |
| "loss": -0.0227, | |
| "reward": 0.7500000018626451, | |
| "reward_std": 0.18404608592391014, | |
| "rewards/accuracy_reward": 0.7500000018626451, | |
| "rewards/format_reward": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 507.3958435058594, | |
| "epoch": 0.06504065040650407, | |
| "grad_norm": 0.04812910035252571, | |
| "kl": 0.0, | |
| "learning_rate": 6.666666666666666e-07, | |
| "loss": 0.0693, | |
| "reward": 0.562500013038516, | |
| "reward_std": 0.21764283627271652, | |
| "rewards/accuracy_reward": 0.562500013038516, | |
| "rewards/format_reward": 0.0, | |
| "step": 2 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 512.7708473205566, | |
| "epoch": 0.0975609756097561, | |
| "grad_norm": 0.03587072715163231, | |
| "kl": -1.9647181034088135e-05, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0052, | |
| "reward": 0.6458333376795053, | |
| "reward_std": 0.3170611374080181, | |
| "rewards/accuracy_reward": 0.6458333376795053, | |
| "rewards/format_reward": 0.0, | |
| "step": 3 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 514.5833473205566, | |
| "epoch": 0.13008130081300814, | |
| "grad_norm": 0.04658569023013115, | |
| "kl": -3.5881996154785156e-05, | |
| "learning_rate": 1.3333333333333332e-06, | |
| "loss": -0.0059, | |
| "reward": 0.5000000149011612, | |
| "reward_std": 0.3332235962152481, | |
| "rewards/accuracy_reward": 0.5000000149011612, | |
| "rewards/format_reward": 0.0, | |
| "step": 4 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 534.9166831970215, | |
| "epoch": 0.16260162601626016, | |
| "grad_norm": 0.019696101546287537, | |
| "kl": -3.643333911895752e-05, | |
| "learning_rate": 1.6666666666666669e-06, | |
| "loss": 0.0292, | |
| "reward": 0.7291666697710752, | |
| "reward_std": 0.11558076366782188, | |
| "rewards/accuracy_reward": 0.7291666697710752, | |
| "rewards/format_reward": 0.0, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 482.1041793823242, | |
| "epoch": 0.1951219512195122, | |
| "grad_norm": 0.04941963776946068, | |
| "kl": -5.251169204711914e-05, | |
| "learning_rate": 2e-06, | |
| "loss": 0.022, | |
| "reward": 0.687500013038516, | |
| "reward_std": 0.35457348078489304, | |
| "rewards/accuracy_reward": 0.687500013038516, | |
| "rewards/format_reward": 0.0, | |
| "step": 6 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 521.3125076293945, | |
| "epoch": 0.22764227642276422, | |
| "grad_norm": 0.0479104146361351, | |
| "kl": -7.079541683197021e-05, | |
| "learning_rate": 2.3333333333333336e-06, | |
| "loss": 0.0171, | |
| "reward": 0.7291666753590107, | |
| "reward_std": 0.21764283627271652, | |
| "rewards/accuracy_reward": 0.7291666753590107, | |
| "rewards/format_reward": 0.0, | |
| "step": 7 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 447.35417556762695, | |
| "epoch": 0.2601626016260163, | |
| "grad_norm": 0.028429586440324783, | |
| "kl": -7.56382942199707e-05, | |
| "learning_rate": 2.6666666666666664e-06, | |
| "loss": 0.0024, | |
| "reward": 0.5833333432674408, | |
| "reward_std": 0.10206207260489464, | |
| "rewards/accuracy_reward": 0.5833333432674408, | |
| "rewards/format_reward": 0.0, | |
| "step": 8 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 629.3125190734863, | |
| "epoch": 0.2926829268292683, | |
| "grad_norm": 0.036269526928663254, | |
| "kl": -6.125122308731079e-05, | |
| "learning_rate": 3e-06, | |
| "loss": -0.0114, | |
| "reward": 0.45833334140479565, | |
| "reward_std": 0.16661180183291435, | |
| "rewards/accuracy_reward": 0.45833334140479565, | |
| "rewards/format_reward": 0.0, | |
| "step": 9 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 413.9166889190674, | |
| "epoch": 0.3252032520325203, | |
| "grad_norm": 0.047799333930015564, | |
| "kl": 4.0762126445770264e-05, | |
| "learning_rate": 2.9988719287563454e-06, | |
| "loss": -0.0099, | |
| "reward": 0.6250000074505806, | |
| "reward_std": 0.2861081585288048, | |
| "rewards/accuracy_reward": 0.6250000074505806, | |
| "rewards/format_reward": 0.0, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 525.1666870117188, | |
| "epoch": 0.35772357723577236, | |
| "grad_norm": 0.022231411188840866, | |
| "kl": 0.00024199485778808594, | |
| "learning_rate": 2.995489411751688e-06, | |
| "loss": 0.0023, | |
| "reward": 0.6875000018626451, | |
| "reward_std": 0.11558076366782188, | |
| "rewards/accuracy_reward": 0.6875000018626451, | |
| "rewards/format_reward": 0.0, | |
| "step": 11 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 550.8125114440918, | |
| "epoch": 0.3902439024390244, | |
| "grad_norm": 0.014551201835274696, | |
| "kl": 0.0006639957427978516, | |
| "learning_rate": 2.989857536612915e-06, | |
| "loss": 0.0, | |
| "reward": 0.5833333358168602, | |
| "reward_std": 0.06454972922801971, | |
| "rewards/accuracy_reward": 0.5833333358168602, | |
| "rewards/format_reward": 0.0, | |
| "step": 12 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 567.5416831970215, | |
| "epoch": 0.42276422764227645, | |
| "grad_norm": 0.04071850702166557, | |
| "kl": 0.0005123615264892578, | |
| "learning_rate": 2.9819847742152137e-06, | |
| "loss": 0.0243, | |
| "reward": 0.5833333395421505, | |
| "reward_std": 0.22155842930078506, | |
| "rewards/accuracy_reward": 0.5833333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 13 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 532.9375152587891, | |
| "epoch": 0.45528455284552843, | |
| "grad_norm": 0.04049593582749367, | |
| "kl": 0.0007052421569824219, | |
| "learning_rate": 2.971882965941077e-06, | |
| "loss": 0.0228, | |
| "reward": 0.7083333358168602, | |
| "reward_std": 0.20148037374019623, | |
| "rewards/accuracy_reward": 0.7083333358168602, | |
| "rewards/format_reward": 0.0, | |
| "step": 14 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 527.9375190734863, | |
| "epoch": 0.4878048780487805, | |
| "grad_norm": 0.046176813542842865, | |
| "kl": 0.001251220703125, | |
| "learning_rate": 2.959567305869736e-06, | |
| "loss": 0.0091, | |
| "reward": 0.5208333432674408, | |
| "reward_std": 0.1705273948609829, | |
| "rewards/accuracy_reward": 0.5208333432674408, | |
| "rewards/format_reward": 0.0, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 605.4166717529297, | |
| "epoch": 0.5203252032520326, | |
| "grad_norm": 0.028073379769921303, | |
| "kl": 0.0017507076263427734, | |
| "learning_rate": 2.9450563179238205e-06, | |
| "loss": -0.0022, | |
| "reward": 0.5000000055879354, | |
| "reward_std": 0.10206206887960434, | |
| "rewards/accuracy_reward": 0.5000000055879354, | |
| "rewards/format_reward": 0.0, | |
| "step": 16 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 441.91667556762695, | |
| "epoch": 0.5528455284552846, | |
| "grad_norm": 0.04000205546617508, | |
| "kl": 0.002796173095703125, | |
| "learning_rate": 2.928371828007623e-06, | |
| "loss": 0.0111, | |
| "reward": 0.7916666716337204, | |
| "reward_std": 0.11949635669589043, | |
| "rewards/accuracy_reward": 0.7916666716337204, | |
| "rewards/format_reward": 0.0, | |
| "step": 17 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 376.1250114440918, | |
| "epoch": 0.5853658536585366, | |
| "grad_norm": 0.051672812551259995, | |
| "kl": 0.0023317337036132812, | |
| "learning_rate": 2.9095389311788626e-06, | |
| "loss": 0.0413, | |
| "reward": 0.8541666716337204, | |
| "reward_std": 0.18796167895197868, | |
| "rewards/accuracy_reward": 0.8541666716337204, | |
| "rewards/format_reward": 0.0, | |
| "step": 18 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 385.3125171661377, | |
| "epoch": 0.6178861788617886, | |
| "grad_norm": 0.06200782582163811, | |
| "kl": 0.0025587081909179688, | |
| "learning_rate": 2.888585953903336e-06, | |
| "loss": 0.018, | |
| "reward": 0.6041666828095913, | |
| "reward_std": 0.37377968430519104, | |
| "rewards/accuracy_reward": 0.6041666828095913, | |
| "rewards/format_reward": 0.0, | |
| "step": 19 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 531.9375076293945, | |
| "epoch": 0.6504065040650406, | |
| "grad_norm": 0.03772967308759689, | |
| "kl": 0.0052127838134765625, | |
| "learning_rate": 2.86554441144922e-06, | |
| "loss": 0.055, | |
| "reward": 0.5625000055879354, | |
| "reward_std": 0.2446802221238613, | |
| "rewards/accuracy_reward": 0.5625000055879354, | |
| "rewards/format_reward": 0.0, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 503.27084732055664, | |
| "epoch": 0.6829268292682927, | |
| "grad_norm": 0.06043640151619911, | |
| "kl": 0.005889892578125, | |
| "learning_rate": 2.8404489604851183e-06, | |
| "loss": 0.0113, | |
| "reward": 0.6458333469927311, | |
| "reward_std": 0.33713918924331665, | |
| "rewards/accuracy_reward": 0.6458333469927311, | |
| "rewards/format_reward": 0.0, | |
| "step": 21 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 574.1458549499512, | |
| "epoch": 0.7154471544715447, | |
| "grad_norm": 0.06661098450422287, | |
| "kl": 0.0051898956298828125, | |
| "learning_rate": 2.8133373469531365e-06, | |
| "loss": 0.0463, | |
| "reward": 0.7500000111758709, | |
| "reward_std": 0.30354244261980057, | |
| "rewards/accuracy_reward": 0.7500000111758709, | |
| "rewards/format_reward": 0.0, | |
| "step": 22 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 492.6458435058594, | |
| "epoch": 0.7479674796747967, | |
| "grad_norm": 0.05463815852999687, | |
| "kl": 0.0066204071044921875, | |
| "learning_rate": 2.7842503492953997e-06, | |
| "loss": 0.0033, | |
| "reward": 0.7291666753590107, | |
| "reward_std": 0.3266642391681671, | |
| "rewards/accuracy_reward": 0.7291666753590107, | |
| "rewards/format_reward": 0.0, | |
| "step": 23 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 615.7708511352539, | |
| "epoch": 0.7804878048780488, | |
| "grad_norm": 0.06719698756933212, | |
| "kl": 0.012079238891601562, | |
| "learning_rate": 2.753231717119405e-06, | |
| "loss": 0.0303, | |
| "reward": 0.47916667349636555, | |
| "reward_std": 0.3816108703613281, | |
| "rewards/accuracy_reward": 0.47916667349636555, | |
| "rewards/format_reward": 0.0, | |
| "step": 24 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 507.9791831970215, | |
| "epoch": 0.8130081300813008, | |
| "grad_norm": 0.04937650263309479, | |
| "kl": 0.011255264282226562, | |
| "learning_rate": 2.720328105394451e-06, | |
| "loss": -0.0146, | |
| "reward": 0.45833334140479565, | |
| "reward_std": 0.16661179810762405, | |
| "rewards/accuracy_reward": 0.45833334140479565, | |
| "rewards/format_reward": 0.0, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 669.6875152587891, | |
| "epoch": 0.8455284552845529, | |
| "grad_norm": 0.05630700662732124, | |
| "kl": 0.00904083251953125, | |
| "learning_rate": 2.685589004278139e-06, | |
| "loss": 0.0147, | |
| "reward": 0.5208333395421505, | |
| "reward_std": 0.36417658254504204, | |
| "rewards/accuracy_reward": 0.5208333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 26 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 620.3541831970215, | |
| "epoch": 0.8780487804878049, | |
| "grad_norm": 0.062427032738924026, | |
| "kl": 0.014118194580078125, | |
| "learning_rate": 2.649066664678467e-06, | |
| "loss": 0.0143, | |
| "reward": 0.35416667349636555, | |
| "reward_std": 0.2446802221238613, | |
| "rewards/accuracy_reward": 0.35416667349636555, | |
| "rewards/format_reward": 0.0, | |
| "step": 27 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 406.3541774749756, | |
| "epoch": 0.9105691056910569, | |
| "grad_norm": 0.08615698665380478, | |
| "kl": 0.02004241943359375, | |
| "learning_rate": 2.610816019663507e-06, | |
| "loss": 0.0082, | |
| "reward": 0.604166679084301, | |
| "reward_std": 0.3720077648758888, | |
| "rewards/accuracy_reward": 0.604166679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 28 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 462.54167556762695, | |
| "epoch": 0.943089430894309, | |
| "grad_norm": 0.042346589267253876, | |
| "kl": 0.014068603515625, | |
| "learning_rate": 2.5708946018368487e-06, | |
| "loss": 0.0005, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.10206207633018494, | |
| "rewards/accuracy_reward": 0.8333333432674408, | |
| "rewards/format_reward": 0.0, | |
| "step": 29 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 434.97918128967285, | |
| "epoch": 0.975609756097561, | |
| "grad_norm": 0.05360697954893112, | |
| "kl": 0.0150909423828125, | |
| "learning_rate": 2.529362456803101e-06, | |
| "loss": -0.0145, | |
| "reward": 0.5000000037252903, | |
| "reward_std": 0.12909945845603943, | |
| "rewards/accuracy_reward": 0.5000000037252903, | |
| "rewards/format_reward": 0.0, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 653.1458473205566, | |
| "epoch": 1.032520325203252, | |
| "grad_norm": 0.06400468945503235, | |
| "kl": 0.012498855590820312, | |
| "learning_rate": 2.4862820528535957e-06, | |
| "loss": -0.0046, | |
| "reward": 0.6250000111758709, | |
| "reward_std": 0.30354244634509087, | |
| "rewards/accuracy_reward": 0.6250000111758709, | |
| "rewards/format_reward": 0.0, | |
| "step": 31 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 493.81250762939453, | |
| "epoch": 1.065040650406504, | |
| "grad_norm": 0.0848759189248085, | |
| "kl": 0.01270294189453125, | |
| "learning_rate": 2.441718187008148e-06, | |
| "loss": 0.0259, | |
| "reward": 0.6666666809469461, | |
| "reward_std": 0.3506578914821148, | |
| "rewards/accuracy_reward": 0.6666666809469461, | |
| "rewards/format_reward": 0.0, | |
| "step": 32 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 577.541675567627, | |
| "epoch": 1.0975609756097562, | |
| "grad_norm": 0.07703862339258194, | |
| "kl": 0.0131988525390625, | |
| "learning_rate": 2.3957378875541795e-06, | |
| "loss": 0.0724, | |
| "reward": 0.39583334513008595, | |
| "reward_std": 0.35457347705960274, | |
| "rewards/accuracy_reward": 0.39583334513008595, | |
| "rewards/format_reward": 0.0, | |
| "step": 33 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 507.0000114440918, | |
| "epoch": 1.1300813008130082, | |
| "grad_norm": 0.06679866462945938, | |
| "kl": 0.0151214599609375, | |
| "learning_rate": 2.348410313229808e-06, | |
| "loss": -0.004, | |
| "reward": 0.7500000149011612, | |
| "reward_std": 0.3680921792984009, | |
| "rewards/accuracy_reward": 0.7500000149011612, | |
| "rewards/format_reward": 0.0, | |
| "step": 34 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 480.5416831970215, | |
| "epoch": 1.1626016260162602, | |
| "grad_norm": 0.035936519503593445, | |
| "kl": 0.01000213623046875, | |
| "learning_rate": 2.2998066492025373e-06, | |
| "loss": 0.0041, | |
| "reward": 0.5000000018626451, | |
| "reward_std": 0.18404608592391014, | |
| "rewards/accuracy_reward": 0.5000000018626451, | |
| "rewards/format_reward": 0.0, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 593.8541831970215, | |
| "epoch": 1.1951219512195121, | |
| "grad_norm": 0.05383417382836342, | |
| "kl": 0.0099029541015625, | |
| "learning_rate": 2.25e-06, | |
| "loss": 0.0303, | |
| "reward": 0.5833333395421505, | |
| "reward_std": 0.33057983219623566, | |
| "rewards/accuracy_reward": 0.5833333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 36 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 516.3333511352539, | |
| "epoch": 1.2276422764227641, | |
| "grad_norm": 0.06559046357870102, | |
| "kl": 0.012409210205078125, | |
| "learning_rate": 2.1990652795538085e-06, | |
| "loss": 0.0007, | |
| "reward": 0.6875000074505806, | |
| "reward_std": 0.11558076366782188, | |
| "rewards/accuracy_reward": 0.6875000074505806, | |
| "rewards/format_reward": 0.0, | |
| "step": 37 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 485.75001335144043, | |
| "epoch": 1.2601626016260163, | |
| "grad_norm": 0.04288559779524803, | |
| "kl": 0.008958816528320312, | |
| "learning_rate": 2.1470790985218807e-06, | |
| "loss": 0.0117, | |
| "reward": 0.5833333414047956, | |
| "reward_std": 0.16661179810762405, | |
| "rewards/accuracy_reward": 0.5833333414047956, | |
| "rewards/format_reward": 0.0, | |
| "step": 38 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 450.83334159851074, | |
| "epoch": 1.2926829268292683, | |
| "grad_norm": 0.09712168574333191, | |
| "kl": 0.024381637573242188, | |
| "learning_rate": 2.0941196490587354e-06, | |
| "loss": 0.0067, | |
| "reward": 0.604166679084301, | |
| "reward_std": 0.33713919296860695, | |
| "rewards/accuracy_reward": 0.604166679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 39 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 576.8125152587891, | |
| "epoch": 1.3252032520325203, | |
| "grad_norm": 0.05104967951774597, | |
| "kl": 0.016981124877929688, | |
| "learning_rate": 2.040266587207066e-06, | |
| "loss": -0.024, | |
| "reward": 0.5208333414047956, | |
| "reward_std": 0.2350771203637123, | |
| "rewards/accuracy_reward": 0.5208333414047956, | |
| "rewards/format_reward": 0.0, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 446.14584732055664, | |
| "epoch": 1.3577235772357723, | |
| "grad_norm": 0.10665060579776764, | |
| "kl": 0.014129638671875, | |
| "learning_rate": 1.985600913087482e-06, | |
| "loss": 0.0179, | |
| "reward": 0.8541666753590107, | |
| "reward_std": 0.1801304928958416, | |
| "rewards/accuracy_reward": 0.8541666753590107, | |
| "rewards/format_reward": 0.0, | |
| "step": 41 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 448.16668701171875, | |
| "epoch": 1.3902439024390243, | |
| "grad_norm": 0.03846089169383049, | |
| "kl": 0.016796112060546875, | |
| "learning_rate": 1.9302048490666355e-06, | |
| "loss": -0.0025, | |
| "reward": 0.7500000055879354, | |
| "reward_std": 0.10206206887960434, | |
| "rewards/accuracy_reward": 0.7500000055879354, | |
| "rewards/format_reward": 0.0, | |
| "step": 42 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 533.6875190734863, | |
| "epoch": 1.4227642276422765, | |
| "grad_norm": 0.11329367011785507, | |
| "kl": 0.02149200439453125, | |
| "learning_rate": 1.874161716086972e-06, | |
| "loss": 0.0687, | |
| "reward": 0.7083333395421505, | |
| "reward_std": 0.22155842930078506, | |
| "rewards/accuracy_reward": 0.7083333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 43 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 578.666675567627, | |
| "epoch": 1.4552845528455285, | |
| "grad_norm": 0.09458328783512115, | |
| "kl": 0.026372909545898438, | |
| "learning_rate": 1.8175558083441164e-06, | |
| "loss": 0.0187, | |
| "reward": 0.6875000111758709, | |
| "reward_std": 0.2446802221238613, | |
| "rewards/accuracy_reward": 0.6875000111758709, | |
| "rewards/format_reward": 0.0, | |
| "step": 44 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 480.3958435058594, | |
| "epoch": 1.4878048780487805, | |
| "grad_norm": 0.06678877025842667, | |
| "kl": 0.043430328369140625, | |
| "learning_rate": 1.7604722665003958e-06, | |
| "loss": 0.0241, | |
| "reward": 0.708333333954215, | |
| "reward_std": 0.11949635669589043, | |
| "rewards/accuracy_reward": 0.708333333954215, | |
| "rewards/format_reward": 0.0, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 446.1041717529297, | |
| "epoch": 1.5203252032520327, | |
| "grad_norm": 0.09147606790065765, | |
| "kl": 0.043731689453125, | |
| "learning_rate": 1.702996949625197e-06, | |
| "loss": 0.0073, | |
| "reward": 0.7916666734963655, | |
| "reward_std": 0.16661180183291435, | |
| "rewards/accuracy_reward": 0.7916666734963655, | |
| "rewards/format_reward": 0.0, | |
| "step": 46 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 449.3333549499512, | |
| "epoch": 1.5528455284552845, | |
| "grad_norm": 0.05224497243762016, | |
| "kl": 0.0131683349609375, | |
| "learning_rate": 1.6452163060547687e-06, | |
| "loss": 0.0098, | |
| "reward": 0.5833333432674408, | |
| "reward_std": 0.10206206887960434, | |
| "rewards/accuracy_reward": 0.5833333432674408, | |
| "rewards/format_reward": 0.0, | |
| "step": 47 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 461.6041774749756, | |
| "epoch": 1.5853658536585367, | |
| "grad_norm": 0.06403756141662598, | |
| "kl": 0.047855377197265625, | |
| "learning_rate": 1.5872172433657137e-06, | |
| "loss": 0.0109, | |
| "reward": 0.7083333358168602, | |
| "reward_std": 0.06454972922801971, | |
| "rewards/accuracy_reward": 0.7083333358168602, | |
| "rewards/format_reward": 0.0, | |
| "step": 48 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 539.0416870117188, | |
| "epoch": 1.6178861788617886, | |
| "grad_norm": 0.11434386670589447, | |
| "kl": 0.07855987548828125, | |
| "learning_rate": 1.5290869976577366e-06, | |
| "loss": 0.0296, | |
| "reward": 0.4583333358168602, | |
| "reward_std": 0.20148037374019623, | |
| "rewards/accuracy_reward": 0.4583333358168602, | |
| "rewards/format_reward": 0.0, | |
| "step": 49 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 424.2291793823242, | |
| "epoch": 1.6504065040650406, | |
| "grad_norm": 0.18005186319351196, | |
| "kl": 0.11967849731445312, | |
| "learning_rate": 1.4709130023422637e-06, | |
| "loss": -0.0011, | |
| "reward": 0.7708333414047956, | |
| "reward_std": 0.2350771240890026, | |
| "rewards/accuracy_reward": 0.7708333414047956, | |
| "rewards/format_reward": 0.0, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 501.14584732055664, | |
| "epoch": 1.6829268292682928, | |
| "grad_norm": 0.11921941488981247, | |
| "kl": 0.054943084716796875, | |
| "learning_rate": 1.4127827566342864e-06, | |
| "loss": 0.031, | |
| "reward": 0.5416666679084301, | |
| "reward_std": 0.06454972922801971, | |
| "rewards/accuracy_reward": 0.5416666679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 51 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 503.5833435058594, | |
| "epoch": 1.7154471544715446, | |
| "grad_norm": 0.2314152866601944, | |
| "kl": 0.046413421630859375, | |
| "learning_rate": 1.3547836939452313e-06, | |
| "loss": 0.0801, | |
| "reward": 0.6666666716337204, | |
| "reward_std": 0.12909945845603943, | |
| "rewards/accuracy_reward": 0.6666666716337204, | |
| "rewards/format_reward": 0.0, | |
| "step": 52 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 585.8958358764648, | |
| "epoch": 1.7479674796747968, | |
| "grad_norm": 0.07401817291975021, | |
| "kl": 0.0781097412109375, | |
| "learning_rate": 1.297003050374804e-06, | |
| "loss": 0.0253, | |
| "reward": 0.6875000074505806, | |
| "reward_std": 0.11558076366782188, | |
| "rewards/accuracy_reward": 0.6875000074505806, | |
| "rewards/format_reward": 0.0, | |
| "step": 53 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 401.1041717529297, | |
| "epoch": 1.7804878048780488, | |
| "grad_norm": 0.20789878070354462, | |
| "kl": 0.037078857421875, | |
| "learning_rate": 1.2395277334996047e-06, | |
| "loss": 0.0499, | |
| "reward": 0.8125000074505806, | |
| "reward_std": 0.11558076739311218, | |
| "rewards/accuracy_reward": 0.8125000074505806, | |
| "rewards/format_reward": 0.0, | |
| "step": 54 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 478.5000114440918, | |
| "epoch": 1.8130081300813008, | |
| "grad_norm": 0.8033259510993958, | |
| "kl": 0.11248397827148438, | |
| "learning_rate": 1.1824441916558843e-06, | |
| "loss": 0.1429, | |
| "reward": 0.7500000149011612, | |
| "reward_std": 0.3776952736079693, | |
| "rewards/accuracy_reward": 0.7500000149011612, | |
| "rewards/format_reward": 0.0, | |
| "step": 55 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 461.6458396911621, | |
| "epoch": 1.845528455284553, | |
| "grad_norm": 0.7502633929252625, | |
| "kl": 0.13535308837890625, | |
| "learning_rate": 1.1258382839130282e-06, | |
| "loss": 0.0708, | |
| "reward": 0.708333358168602, | |
| "reward_std": 0.3236205019056797, | |
| "rewards/accuracy_reward": 0.708333358168602, | |
| "rewards/format_reward": 0.0, | |
| "step": 56 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 604.7708511352539, | |
| "epoch": 1.8780487804878048, | |
| "grad_norm": 0.47473546862602234, | |
| "kl": 0.28479576110839844, | |
| "learning_rate": 1.069795150933365e-06, | |
| "loss": 0.0366, | |
| "reward": 0.47916667349636555, | |
| "reward_std": 0.2350771240890026, | |
| "rewards/accuracy_reward": 0.47916667349636555, | |
| "rewards/format_reward": 0.0, | |
| "step": 57 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 555.5625114440918, | |
| "epoch": 1.910569105691057, | |
| "grad_norm": 0.11512857675552368, | |
| "kl": 0.1971893310546875, | |
| "learning_rate": 1.0143990869125186e-06, | |
| "loss": 0.009, | |
| "reward": 0.4375000074505806, | |
| "reward_std": 0.11558076366782188, | |
| "rewards/accuracy_reward": 0.4375000074505806, | |
| "rewards/format_reward": 0.0, | |
| "step": 58 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 566.0833511352539, | |
| "epoch": 1.943089430894309, | |
| "grad_norm": 0.25552570819854736, | |
| "kl": 0.4318656921386719, | |
| "learning_rate": 9.597334127929346e-07, | |
| "loss": 0.0466, | |
| "reward": 0.5625000018626451, | |
| "reward_std": 0.11558076739311218, | |
| "rewards/accuracy_reward": 0.5625000018626451, | |
| "rewards/format_reward": 0.0, | |
| "step": 59 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 417.3541831970215, | |
| "epoch": 1.975609756097561, | |
| "grad_norm": 0.4009849727153778, | |
| "kl": 0.46466064453125, | |
| "learning_rate": 9.058803509412648e-07, | |
| "loss": 0.0189, | |
| "reward": 0.7708333432674408, | |
| "reward_std": 0.1705273948609829, | |
| "rewards/accuracy_reward": 0.7708333432674408, | |
| "rewards/format_reward": 0.0, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 494.45834732055664, | |
| "epoch": 2.032520325203252, | |
| "grad_norm": 0.7852746844291687, | |
| "kl": 0.4476318359375, | |
| "learning_rate": 8.529209014781202e-07, | |
| "loss": 0.0736, | |
| "reward": 0.6875000055879354, | |
| "reward_std": 0.2446802221238613, | |
| "rewards/accuracy_reward": 0.6875000055879354, | |
| "rewards/format_reward": 0.0, | |
| "step": 61 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 445.9583435058594, | |
| "epoch": 2.065040650406504, | |
| "grad_norm": 0.45981186628341675, | |
| "kl": 0.4035301208496094, | |
| "learning_rate": 8.009347204461922e-07, | |
| "loss": 0.0114, | |
| "reward": 0.6458333432674408, | |
| "reward_std": 0.2446802258491516, | |
| "rewards/accuracy_reward": 0.6458333432674408, | |
| "rewards/format_reward": 0.0, | |
| "step": 62 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 588.666675567627, | |
| "epoch": 2.097560975609756, | |
| "grad_norm": 0.9820762276649475, | |
| "kl": 1.2833786010742188, | |
| "learning_rate": 7.500000000000003e-07, | |
| "loss": 0.0928, | |
| "reward": 0.5416666679084301, | |
| "reward_std": 0.10206207260489464, | |
| "rewards/accuracy_reward": 0.5416666679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 63 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 489.3541831970215, | |
| "epoch": 2.130081300813008, | |
| "grad_norm": 1.9332908391952515, | |
| "kl": 0.6612548828125, | |
| "learning_rate": 7.001933507974634e-07, | |
| "loss": 0.1461, | |
| "reward": 0.6458333525806665, | |
| "reward_std": 0.41129202768206596, | |
| "rewards/accuracy_reward": 0.6458333525806665, | |
| "rewards/format_reward": 0.0, | |
| "step": 64 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 560.1875228881836, | |
| "epoch": 2.16260162601626, | |
| "grad_norm": 1.457485318183899, | |
| "kl": 0.762725830078125, | |
| "learning_rate": 6.515896867701923e-07, | |
| "loss": 0.131, | |
| "reward": 0.479166679084301, | |
| "reward_std": 0.34674229472875595, | |
| "rewards/accuracy_reward": 0.479166679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 494.08334732055664, | |
| "epoch": 2.1951219512195124, | |
| "grad_norm": 0.7749570608139038, | |
| "kl": 0.5813465118408203, | |
| "learning_rate": 6.04262112445821e-07, | |
| "loss": 0.0776, | |
| "reward": 0.604166679084301, | |
| "reward_std": 0.2996268533170223, | |
| "rewards/accuracy_reward": 0.604166679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 66 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 536.0208415985107, | |
| "epoch": 2.227642276422764, | |
| "grad_norm": 0.4476146697998047, | |
| "kl": 1.7618732452392578, | |
| "learning_rate": 5.582818129918525e-07, | |
| "loss": 0.1542, | |
| "reward": 0.6666666753590107, | |
| "reward_std": 0.24859581887722015, | |
| "rewards/accuracy_reward": 0.6666666753590107, | |
| "rewards/format_reward": 0.0, | |
| "step": 67 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 556.4166717529297, | |
| "epoch": 2.2601626016260163, | |
| "grad_norm": 1.5170872211456299, | |
| "kl": 2.1071243286132812, | |
| "learning_rate": 5.137179471464047e-07, | |
| "loss": 0.131, | |
| "reward": 0.4791666679084301, | |
| "reward_std": 0.13301505148410797, | |
| "rewards/accuracy_reward": 0.4791666679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 68 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 596.5000076293945, | |
| "epoch": 2.292682926829268, | |
| "grad_norm": 0.9765549898147583, | |
| "kl": 1.0570831298828125, | |
| "learning_rate": 4.7063754319689976e-07, | |
| "loss": 0.0878, | |
| "reward": 0.6875000037252903, | |
| "reward_std": 0.19756478071212769, | |
| "rewards/accuracy_reward": 0.6875000037252903, | |
| "rewards/format_reward": 0.0, | |
| "step": 69 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 496.04168701171875, | |
| "epoch": 2.3252032520325203, | |
| "grad_norm": 1.364784598350525, | |
| "kl": 0.8105888366699219, | |
| "learning_rate": 4.291053981631517e-07, | |
| "loss": 0.1012, | |
| "reward": 0.625000013038516, | |
| "reward_std": 0.2861081585288048, | |
| "rewards/accuracy_reward": 0.625000013038516, | |
| "rewards/format_reward": 0.0, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 540.4583415985107, | |
| "epoch": 2.3577235772357725, | |
| "grad_norm": 0.753292977809906, | |
| "kl": 1.4221649169921875, | |
| "learning_rate": 3.891839803364934e-07, | |
| "loss": 0.1611, | |
| "reward": 0.6458333395421505, | |
| "reward_std": 0.2525114081799984, | |
| "rewards/accuracy_reward": 0.6458333395421505, | |
| "rewards/format_reward": 0.0, | |
| "step": 71 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 491.3750114440918, | |
| "epoch": 2.3902439024390243, | |
| "grad_norm": 0.9958664178848267, | |
| "kl": 0.7973136901855469, | |
| "learning_rate": 3.5093333532153313e-07, | |
| "loss": 0.073, | |
| "reward": 0.7083333358168602, | |
| "reward_std": 0.06454972922801971, | |
| "rewards/accuracy_reward": 0.7083333358168602, | |
| "rewards/format_reward": 0.0, | |
| "step": 72 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 567.5833473205566, | |
| "epoch": 2.4227642276422765, | |
| "grad_norm": 0.12560848891735077, | |
| "kl": 0.027166366577148438, | |
| "learning_rate": 3.144109957218612e-07, | |
| "loss": 0.0161, | |
| "reward": 0.6666666753590107, | |
| "reward_std": 0.2861081585288048, | |
| "rewards/accuracy_reward": 0.6666666753590107, | |
| "rewards/format_reward": 0.0, | |
| "step": 73 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 358.87500762939453, | |
| "epoch": 2.4552845528455283, | |
| "grad_norm": 0.09765256196260452, | |
| "kl": 0.0715789794921875, | |
| "learning_rate": 2.796718946055488e-07, | |
| "loss": 0.0099, | |
| "reward": 0.9166666716337204, | |
| "reward_std": 0.11949636042118073, | |
| "rewards/accuracy_reward": 0.9166666716337204, | |
| "rewards/format_reward": 0.0, | |
| "step": 74 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 523.7083435058594, | |
| "epoch": 2.4878048780487805, | |
| "grad_norm": 1.1270173788070679, | |
| "kl": 1.6110763549804688, | |
| "learning_rate": 2.467682828805956e-07, | |
| "loss": 0.1044, | |
| "reward": 0.8541666772216558, | |
| "reward_std": 0.1530931107699871, | |
| "rewards/accuracy_reward": 0.8541666772216558, | |
| "rewards/format_reward": 0.0, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 487.1458511352539, | |
| "epoch": 2.5203252032520327, | |
| "grad_norm": 0.848898708820343, | |
| "kl": 1.3041305541992188, | |
| "learning_rate": 2.1574965070460045e-07, | |
| "loss": 0.1177, | |
| "reward": 0.7291666809469461, | |
| "reward_std": 0.28219256922602654, | |
| "rewards/accuracy_reward": 0.7291666809469461, | |
| "rewards/format_reward": 0.0, | |
| "step": 76 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 422.89584732055664, | |
| "epoch": 2.5528455284552845, | |
| "grad_norm": 0.6274386644363403, | |
| "kl": 1.1170845031738281, | |
| "learning_rate": 1.8666265304686387e-07, | |
| "loss": 0.0599, | |
| "reward": 0.5833333358168602, | |
| "reward_std": 0.06454972922801971, | |
| "rewards/accuracy_reward": 0.5833333358168602, | |
| "rewards/format_reward": 0.0, | |
| "step": 77 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 455.81251525878906, | |
| "epoch": 2.5853658536585367, | |
| "grad_norm": 0.08515371382236481, | |
| "kl": 0.03688812255859375, | |
| "learning_rate": 1.5955103951488177e-07, | |
| "loss": 0.0254, | |
| "reward": 0.6250000111758709, | |
| "reward_std": 0.16661180183291435, | |
| "rewards/accuracy_reward": 0.6250000111758709, | |
| "rewards/format_reward": 0.0, | |
| "step": 78 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 608.4791831970215, | |
| "epoch": 2.617886178861789, | |
| "grad_norm": 1.5989761352539062, | |
| "kl": 2.8103561401367188, | |
| "learning_rate": 1.3445558855078017e-07, | |
| "loss": 0.1075, | |
| "reward": 0.35416667349636555, | |
| "reward_std": 0.2350771203637123, | |
| "rewards/accuracy_reward": 0.35416667349636555, | |
| "rewards/format_reward": 0.0, | |
| "step": 79 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 566.7291870117188, | |
| "epoch": 2.6504065040650406, | |
| "grad_norm": 1.5026016235351562, | |
| "kl": 1.9703483581542969, | |
| "learning_rate": 1.114140460966645e-07, | |
| "loss": 0.1054, | |
| "reward": 0.5000000037252903, | |
| "reward_std": 0.26603010296821594, | |
| "rewards/accuracy_reward": 0.5000000037252903, | |
| "rewards/format_reward": 0.0, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 467.8125114440918, | |
| "epoch": 2.682926829268293, | |
| "grad_norm": 0.3330284059047699, | |
| "kl": 0.9720802307128906, | |
| "learning_rate": 9.046106882113752e-08, | |
| "loss": 0.0702, | |
| "reward": 0.7916666734963655, | |
| "reward_std": 0.16661179810762405, | |
| "rewards/accuracy_reward": 0.7916666734963655, | |
| "rewards/format_reward": 0.0, | |
| "step": 81 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 541.625020980835, | |
| "epoch": 2.7154471544715446, | |
| "grad_norm": 1.0519593954086304, | |
| "kl": 1.0916862487792969, | |
| "learning_rate": 7.162817199237703e-08, | |
| "loss": 0.0588, | |
| "reward": 0.4791666716337204, | |
| "reward_std": 0.05103103443980217, | |
| "rewards/accuracy_reward": 0.4791666716337204, | |
| "rewards/format_reward": 0.0, | |
| "step": 82 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 456.2708549499512, | |
| "epoch": 2.747967479674797, | |
| "grad_norm": 0.5059961676597595, | |
| "kl": 0.6999807357788086, | |
| "learning_rate": 5.49436820761795e-08, | |
| "loss": 0.0346, | |
| "reward": 0.5625000055879354, | |
| "reward_std": 0.1705273911356926, | |
| "rewards/accuracy_reward": 0.5625000055879354, | |
| "rewards/format_reward": 0.0, | |
| "step": 83 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 487.5000114440918, | |
| "epoch": 2.7804878048780486, | |
| "grad_norm": 0.8734613060951233, | |
| "kl": 1.3879623413085938, | |
| "learning_rate": 4.0432694130264294e-08, | |
| "loss": 0.0628, | |
| "reward": 0.6458333414047956, | |
| "reward_std": 0.2446802258491516, | |
| "rewards/accuracy_reward": 0.6458333414047956, | |
| "rewards/format_reward": 0.0, | |
| "step": 84 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 511.2500114440918, | |
| "epoch": 2.813008130081301, | |
| "grad_norm": 0.7217493653297424, | |
| "kl": 1.629373550415039, | |
| "learning_rate": 2.8117034058922964e-08, | |
| "loss": 0.0686, | |
| "reward": 0.6250000074505806, | |
| "reward_std": 0.25819891691207886, | |
| "rewards/accuracy_reward": 0.6250000074505806, | |
| "rewards/format_reward": 0.0, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 428.6041774749756, | |
| "epoch": 2.845528455284553, | |
| "grad_norm": 0.4210287928581238, | |
| "kl": 0.6216049194335938, | |
| "learning_rate": 1.8015225784786483e-08, | |
| "loss": -0.0001, | |
| "reward": 0.6875000018626451, | |
| "reward_std": 0.11558076366782188, | |
| "rewards/accuracy_reward": 0.6875000018626451, | |
| "rewards/format_reward": 0.0, | |
| "step": 86 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 533.2083396911621, | |
| "epoch": 2.8780487804878048, | |
| "grad_norm": 0.43215566873550415, | |
| "kl": 1.061065673828125, | |
| "learning_rate": 1.0142463387085465e-08, | |
| "loss": 0.0424, | |
| "reward": 0.5416666679084301, | |
| "reward_std": 0.06454972922801971, | |
| "rewards/accuracy_reward": 0.5416666679084301, | |
| "rewards/format_reward": 0.0, | |
| "step": 87 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 586.6041793823242, | |
| "epoch": 2.910569105691057, | |
| "grad_norm": 0.8715842366218567, | |
| "kl": 1.849679946899414, | |
| "learning_rate": 4.510588248311964e-09, | |
| "loss": 0.1967, | |
| "reward": 0.5833333469927311, | |
| "reward_std": 0.268673874437809, | |
| "rewards/accuracy_reward": 0.5833333469927311, | |
| "rewards/format_reward": 0.0, | |
| "step": 88 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 399.47917556762695, | |
| "epoch": 2.943089430894309, | |
| "grad_norm": 0.48902833461761475, | |
| "kl": 0.33133697509765625, | |
| "learning_rate": 1.1280712436549378e-09, | |
| "loss": 0.0284, | |
| "reward": 0.8750000074505806, | |
| "reward_std": 0.18404608592391014, | |
| "rewards/accuracy_reward": 0.8750000074505806, | |
| "rewards/format_reward": 0.0, | |
| "step": 89 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 443.4791793823242, | |
| "epoch": 2.975609756097561, | |
| "grad_norm": 0.6357685327529907, | |
| "kl": 0.7278213500976562, | |
| "learning_rate": 0.0, | |
| "loss": 0.0558, | |
| "reward": 0.7083333432674408, | |
| "reward_std": 0.10206206887960434, | |
| "rewards/accuracy_reward": 0.7083333432674408, | |
| "rewards/format_reward": 0.0, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 2.975609756097561, | |
| "step": 90, | |
| "total_flos": 0.0, | |
| "train_loss": 0.03994659806432739, | |
| "train_runtime": 6206.0471, | |
| "train_samples_per_second": 0.119, | |
| "train_steps_per_second": 0.015 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 90, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |