{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.4, "eval_steps": 500, "global_step": 2000, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0002, "grad_norm": 3.150317668914795, "kl": 0.00016507542022736743, "learning_rate": 0.0, "loss": 0.0, "num_tokens": 8600.0, "reward": 0.6983642578125, "reward_std": 0.012111281976103783, "rewards//mean": 0.6983642578125, "rewards//std": 0.04253233224153519, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0004, "grad_norm": 3.1340620517730713, "kl": 9.68448439380154e-05, "learning_rate": 2e-08, "loss": 0.0, "num_tokens": 17200.0, "reward": 0.72998046875, "reward_std": 0.016355562955141068, "rewards//mean": 0.72998046875, "rewards//std": 0.05969127640128136, "step": 2 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0006, "grad_norm": 3.628222703933716, "kl": 0.0007720550274825655, "learning_rate": 4e-08, "loss": 0.0, "num_tokens": 25872.0, "reward": 0.737060546875, "reward_std": 0.01231062039732933, "rewards//mean": 0.737060546875, "rewards//std": 0.04868406429886818, "step": 3 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0008, "grad_norm": 3.346497058868408, "kl": 0.0007726833282504231, "learning_rate": 6e-08, "loss": 0.0, "num_tokens": 34600.0, "reward": 0.7144775390625, "reward_std": 0.013249381445348263, "rewards//mean": 0.7144775390625, "rewards//std": 0.057925332337617874, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.001, "grad_norm": 3.1139121055603027, "kl": 0.0006869801800348796, "learning_rate": 8e-08, "loss": 0.0, "num_tokens": 43304.0, "reward": 0.71942138671875, "reward_std": 0.014566982164978981, "rewards//mean": 0.71942138671875, "rewards//std": 0.05822908505797386, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0012, "grad_norm": 3.0228867530822754, "kl": 0.0007238477919599973, "learning_rate": 1e-07, "loss": 0.0, "num_tokens": 51992.0, "reward": 0.71722412109375, "reward_std": 0.012525280006229877, "rewards//mean": 0.71722412109375, "rewards//std": 0.057698674499988556, "step": 6 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0014, "grad_norm": 3.3383941650390625, "kl": 0.0007060512798489071, "learning_rate": 1.2e-07, "loss": 0.0, "num_tokens": 60696.0, "reward": 0.7244873046875, "reward_std": 0.013975013047456741, "rewards//mean": 0.7244873046875, "rewards//std": 0.03846875578165054, "step": 7 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0016, "grad_norm": 3.2544026374816895, "kl": 0.0007304435930564068, "learning_rate": 1.4e-07, "loss": 0.0, "num_tokens": 69336.0, "reward": 0.73199462890625, "reward_std": 0.014917248860001564, "rewards//mean": 0.73199462890625, "rewards//std": 0.054437216371297836, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0018, "grad_norm": 3.2664318084716797, "kl": 0.0006970661124796607, "learning_rate": 1.6e-07, "loss": 0.0, "num_tokens": 78008.0, "reward": 0.7119140625, "reward_std": 0.016640130430459976, "rewards//mean": 0.7119140625, "rewards//std": 0.051102180033922195, "step": 9 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.002, "grad_norm": 3.30499529838562, "kl": 0.0007583869155496359, "learning_rate": 1.8e-07, "loss": 0.0, "num_tokens": 86648.0, "reward": 0.67755126953125, "reward_std": 0.015648527070879936, "rewards//mean": 0.67755126953125, "rewards//std": 0.04872909188270569, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0022, "grad_norm": 3.266948699951172, "kl": 0.0007290949215530418, "learning_rate": 2e-07, "loss": 0.0, "num_tokens": 95456.0, "reward": 0.69329833984375, "reward_std": 0.014868234284222126, "rewards//mean": 0.69329833984375, "rewards//std": 0.04043947160243988, "step": 11 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0024, "grad_norm": 3.2822325229644775, "kl": 0.000768863974371925, "learning_rate": 2.1999999999999998e-07, "loss": 0.0, "num_tokens": 104112.0, "reward": 0.70721435546875, "reward_std": 0.015602790750563145, "rewards//mean": 0.70721435546875, "rewards//std": 0.04803575947880745, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0026, "grad_norm": 3.1537024974823, "kl": 0.000746092431654688, "learning_rate": 2.4e-07, "loss": 0.0, "num_tokens": 112728.0, "reward": 0.72723388671875, "reward_std": 0.013450011610984802, "rewards//mean": 0.72723388671875, "rewards//std": 0.04304227605462074, "step": 13 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0028, "grad_norm": 3.1742215156555176, "kl": 0.0006956102952244692, "learning_rate": 2.6e-07, "loss": 0.0, "num_tokens": 121352.0, "reward": 0.75360107421875, "reward_std": 0.018189093098044395, "rewards//mean": 0.75360107421875, "rewards//std": 0.04565153270959854, "step": 14 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.003, "grad_norm": 3.1125144958496094, "kl": 0.0006477628121501766, "learning_rate": 2.8e-07, "loss": 0.0, "num_tokens": 130072.0, "reward": 0.73187255859375, "reward_std": 0.0172879658639431, "rewards//mean": 0.73187255859375, "rewards//std": 0.06254403293132782, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0032, "grad_norm": 3.3312599658966064, "kl": 0.0007021335186436772, "learning_rate": 3e-07, "loss": 0.0, "num_tokens": 138680.0, "reward": 0.73480224609375, "reward_std": 0.011995160952210426, "rewards//mean": 0.73480224609375, "rewards//std": 0.04787761718034744, "step": 16 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0034, "grad_norm": 3.2479970455169678, "kl": 0.0006967349399928935, "learning_rate": 3.2e-07, "loss": 0.0, "num_tokens": 147320.0, "reward": 0.68505859375, "reward_std": 0.013261607848107815, "rewards//mean": 0.68505859375, "rewards//std": 0.054848916828632355, "step": 17 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0036, "grad_norm": 3.3135740756988525, "kl": 0.000760370421630796, "learning_rate": 3.4000000000000003e-07, "loss": 0.0, "num_tokens": 155984.0, "reward": 0.72137451171875, "reward_std": 0.012660497799515724, "rewards//mean": 0.72137451171875, "rewards//std": 0.04701458662748337, "step": 18 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0038, "grad_norm": 3.2698936462402344, "kl": 0.0007952585219754837, "learning_rate": 3.6e-07, "loss": 0.0, "num_tokens": 164608.0, "reward": 0.7022705078125, "reward_std": 0.023350292816758156, "rewards//mean": 0.7022705078125, "rewards//std": 0.06180427595973015, "step": 19 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.004, "grad_norm": 3.1324961185455322, "kl": 0.00072584531881148, "learning_rate": 3.7999999999999996e-07, "loss": 0.0, "num_tokens": 173168.0, "reward": 0.71453857421875, "reward_std": 0.01809067279100418, "rewards//mean": 0.71453857421875, "rewards//std": 0.056603968143463135, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0042, "grad_norm": 3.1413869857788086, "kl": 0.0007194258432718925, "learning_rate": 4e-07, "loss": 0.0, "num_tokens": 181792.0, "reward": 0.7171630859375, "reward_std": 0.013818057253956795, "rewards//mean": 0.7171630859375, "rewards//std": 0.051577337086200714, "step": 21 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0044, "grad_norm": 3.1343071460723877, "kl": 0.0007037941031740047, "learning_rate": 4.1999999999999995e-07, "loss": 0.0, "num_tokens": 190392.0, "reward": 0.731689453125, "reward_std": 0.01459466852247715, "rewards//mean": 0.731689453125, "rewards//std": 0.03735656663775444, "step": 22 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0046, "grad_norm": 3.26190185546875, "kl": 0.0007583637707284652, "learning_rate": 4.3999999999999997e-07, "loss": 0.0, "num_tokens": 199080.0, "reward": 0.698486328125, "reward_std": 0.015130313113331795, "rewards//mean": 0.698486328125, "rewards//std": 0.04446122422814369, "step": 23 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0048, "grad_norm": 3.3528366088867188, "kl": 0.0008030652170418762, "learning_rate": 4.6e-07, "loss": 0.0, "num_tokens": 207720.0, "reward": 0.72381591796875, "reward_std": 0.015433166176080704, "rewards//mean": 0.72381591796875, "rewards//std": 0.053040314465761185, "step": 24 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.005, "grad_norm": 3.0460476875305176, "kl": 0.0007812085095793009, "learning_rate": 4.8e-07, "loss": 0.0, "num_tokens": 216296.0, "reward": 0.70770263671875, "reward_std": 0.014261599630117416, "rewards//mean": 0.70770263671875, "rewards//std": 0.05814167112112045, "step": 25 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0052, "grad_norm": 3.184849500656128, "kl": 0.0007379321614280343, "learning_rate": 5e-07, "loss": 0.0, "num_tokens": 224968.0, "reward": 0.74749755859375, "reward_std": 0.01661759801208973, "rewards//mean": 0.74749755859375, "rewards//std": 0.05052240192890167, "step": 26 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0054, "grad_norm": 3.071376085281372, "kl": 0.0007392473853542469, "learning_rate": 5.2e-07, "loss": 0.0, "num_tokens": 233528.0, "reward": 0.67926025390625, "reward_std": 0.015911182388663292, "rewards//mean": 0.67926025390625, "rewards//std": 0.07468271255493164, "step": 27 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0056, "grad_norm": 3.1181528568267822, "kl": 0.0007620643737027422, "learning_rate": 5.4e-07, "loss": 0.0, "num_tokens": 242280.0, "reward": 0.68609619140625, "reward_std": 0.015002391301095486, "rewards//mean": 0.68609619140625, "rewards//std": 0.05366988107562065, "step": 28 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0058, "grad_norm": 3.293017864227295, "kl": 0.0007734682003501803, "learning_rate": 5.6e-07, "loss": 0.0, "num_tokens": 250976.0, "reward": 0.7156982421875, "reward_std": 0.011221669614315033, "rewards//mean": 0.7156982421875, "rewards//std": 0.050113748759031296, "step": 29 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.006, "grad_norm": 3.1560003757476807, "kl": 0.0007398290908895433, "learning_rate": 5.8e-07, "loss": 0.0, "num_tokens": 259632.0, "reward": 0.703857421875, "reward_std": 0.015436184592545033, "rewards//mean": 0.703857421875, "rewards//std": 0.04807832092046738, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0062, "grad_norm": 2.963697671890259, "kl": 0.0007791937241563573, "learning_rate": 6e-07, "loss": 0.0, "num_tokens": 268208.0, "reward": 0.74041748046875, "reward_std": 0.018390730023384094, "rewards//mean": 0.74041748046875, "rewards//std": 0.045399487018585205, "step": 31 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0064, "grad_norm": 3.0100553035736084, "kl": 0.0008315800077980384, "learning_rate": 6.2e-07, "loss": 0.0, "num_tokens": 276824.0, "reward": 0.71026611328125, "reward_std": 0.015266066417098045, "rewards//mean": 0.71026611328125, "rewards//std": 0.05298949033021927, "step": 32 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0066, "grad_norm": 3.014559268951416, "kl": 0.0007704183590249158, "learning_rate": 6.4e-07, "loss": 0.0, "num_tokens": 285520.0, "reward": 0.73516845703125, "reward_std": 0.01908334344625473, "rewards//mean": 0.73516845703125, "rewards//std": 0.050059158354997635, "step": 33 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0068, "grad_norm": 3.116197109222412, "kl": 0.00073820207035169, "learning_rate": 6.6e-07, "loss": 0.0, "num_tokens": 294144.0, "reward": 0.70306396484375, "reward_std": 0.015610162168741226, "rewards//mean": 0.70306396484375, "rewards//std": 0.050647489726543427, "step": 34 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.007, "grad_norm": 3.0357425212860107, "kl": 0.000833665588288568, "learning_rate": 6.800000000000001e-07, "loss": 0.0, "num_tokens": 302888.0, "reward": 0.73577880859375, "reward_std": 0.012224329635500908, "rewards//mean": 0.73577880859375, "rewards//std": 0.04008077085018158, "step": 35 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0072, "grad_norm": 3.2299728393554688, "kl": 0.0008243657503044233, "learning_rate": 7e-07, "loss": 0.0, "num_tokens": 311504.0, "reward": 0.711181640625, "reward_std": 0.014584844931960106, "rewards//mean": 0.711181640625, "rewards//std": 0.0634838342666626, "step": 36 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0074, "grad_norm": 3.1693902015686035, "kl": 0.0008555862732464448, "learning_rate": 7.2e-07, "loss": 0.0, "num_tokens": 320208.0, "reward": 0.7293701171875, "reward_std": 0.016974415630102158, "rewards//mean": 0.7293701171875, "rewards//std": 0.045011255890131, "step": 37 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0076, "grad_norm": 3.0704615116119385, "kl": 0.0007757811254123226, "learning_rate": 7.4e-07, "loss": 0.0, "num_tokens": 328904.0, "reward": 0.707763671875, "reward_std": 0.011619615368545055, "rewards//mean": 0.707763671875, "rewards//std": 0.0381837859749794, "step": 38 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0078, "grad_norm": 3.3403103351593018, "kl": 0.0008441023965133354, "learning_rate": 7.599999999999999e-07, "loss": 0.0, "num_tokens": 337632.0, "reward": 0.7027587890625, "reward_std": 0.014964539557695389, "rewards//mean": 0.7027587890625, "rewards//std": 0.06321561336517334, "step": 39 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.008, "grad_norm": 3.3247711658477783, "kl": 0.0009583122737240046, "learning_rate": 7.799999999999999e-07, "loss": 0.0, "num_tokens": 346264.0, "reward": 0.6885986328125, "reward_std": 0.017985574901103973, "rewards//mean": 0.6885986328125, "rewards//std": 0.048691995441913605, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0082, "grad_norm": 3.1681082248687744, "kl": 0.000914801872568205, "learning_rate": 8e-07, "loss": 0.0, "num_tokens": 354912.0, "reward": 0.72589111328125, "reward_std": 0.014821438118815422, "rewards//mean": 0.72589111328125, "rewards//std": 0.06690064072608948, "step": 41 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0084, "grad_norm": 3.135946750640869, "kl": 0.0009579255347489379, "learning_rate": 8.199999999999999e-07, "loss": 0.0, "num_tokens": 363544.0, "reward": 0.74017333984375, "reward_std": 0.013390760868787766, "rewards//mean": 0.74017333984375, "rewards//std": 0.04873934015631676, "step": 42 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0086, "grad_norm": 3.5682313442230225, "kl": 0.0010489100313861854, "learning_rate": 8.399999999999999e-07, "loss": 0.0, "num_tokens": 372144.0, "reward": 0.73565673828125, "reward_std": 0.020230459049344063, "rewards//mean": 0.73565673828125, "rewards//std": 0.05228316783905029, "step": 43 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0088, "grad_norm": 3.349270820617676, "kl": 0.0009648135091993026, "learning_rate": 8.599999999999999e-07, "loss": 0.0, "num_tokens": 380848.0, "reward": 0.73638916015625, "reward_std": 0.013494866900146008, "rewards//mean": 0.73638916015625, "rewards//std": 0.049878910183906555, "step": 44 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.009, "grad_norm": 3.222097396850586, "kl": 0.0010578580695437267, "learning_rate": 8.799999999999999e-07, "loss": 0.0, "num_tokens": 389504.0, "reward": 0.67010498046875, "reward_std": 0.013411211781203747, "rewards//mean": 0.67010498046875, "rewards//std": 0.052416760474443436, "step": 45 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0092, "grad_norm": 3.1397805213928223, "kl": 0.0009474716134718619, "learning_rate": 9e-07, "loss": 0.0, "num_tokens": 398200.0, "reward": 0.72802734375, "reward_std": 0.011726010590791702, "rewards//mean": 0.72802734375, "rewards//std": 0.06396511197090149, "step": 46 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0094, "grad_norm": 3.248054027557373, "kl": 0.0010289600759278983, "learning_rate": 9.2e-07, "loss": 0.0, "num_tokens": 406848.0, "reward": 0.69805908203125, "reward_std": 0.018465079367160797, "rewards//mean": 0.69805908203125, "rewards//std": 0.05887490510940552, "step": 47 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0096, "grad_norm": 3.210569143295288, "kl": 0.0010431300543132238, "learning_rate": 9.399999999999999e-07, "loss": 0.0, "num_tokens": 415520.0, "reward": 0.7083740234375, "reward_std": 0.014447823166847229, "rewards//mean": 0.7083740234375, "rewards//std": 0.06324434280395508, "step": 48 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0098, "grad_norm": 3.0604236125946045, "kl": 0.0009561540064169094, "learning_rate": 9.6e-07, "loss": 0.0, "num_tokens": 424088.0, "reward": 0.7181396484375, "reward_std": 0.016763266175985336, "rewards//mean": 0.7181396484375, "rewards//std": 0.06502308696508408, "step": 49 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.01, "grad_norm": 3.190124988555908, "kl": 0.0010131369563168846, "learning_rate": 9.8e-07, "loss": 0.0, "num_tokens": 432696.0, "reward": 0.703125, "reward_std": 0.014796003699302673, "rewards//mean": 0.703125, "rewards//std": 0.05059729143977165, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0102, "grad_norm": 2.9405922889709473, "kl": 0.0010682813081075437, "learning_rate": 1e-06, "loss": 0.0, "num_tokens": 441352.0, "reward": 0.7259521484375, "reward_std": 0.01541807409375906, "rewards//mean": 0.7259521484375, "rewards//std": 0.03761079162359238, "step": 51 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0104, "grad_norm": 3.0414485931396484, "kl": 0.0011730788828572258, "learning_rate": 9.999998993000298e-07, "loss": 0.0, "num_tokens": 449960.0, "reward": 0.736572265625, "reward_std": 0.01379593275487423, "rewards//mean": 0.736572265625, "rewards//std": 0.05054400861263275, "step": 52 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0106, "grad_norm": 3.2517385482788086, "kl": 0.001291139968088828, "learning_rate": 9.999995972001601e-07, "loss": 0.0001, "num_tokens": 458512.0, "reward": 0.720703125, "reward_std": 0.017797932028770447, "rewards//mean": 0.720703125, "rewards//std": 0.04921412467956543, "step": 53 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0108, "grad_norm": 3.3084728717803955, "kl": 0.0012594063809956424, "learning_rate": 9.999990937005123e-07, "loss": 0.0001, "num_tokens": 467048.0, "reward": 0.7257080078125, "reward_std": 0.013701392337679863, "rewards//mean": 0.7257080078125, "rewards//std": 0.04821210727095604, "step": 54 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.011, "grad_norm": 3.0424487590789795, "kl": 0.00110630490235053, "learning_rate": 9.999983888012896e-07, "loss": 0.0, "num_tokens": 475728.0, "reward": 0.7081298828125, "reward_std": 0.017631450667977333, "rewards//mean": 0.7081298828125, "rewards//std": 0.07252521812915802, "step": 55 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0112, "grad_norm": 3.2253611087799072, "kl": 0.001307873819314409, "learning_rate": 9.999974825027754e-07, "loss": 0.0001, "num_tokens": 484360.0, "reward": 0.72161865234375, "reward_std": 0.012384735979139805, "rewards//mean": 0.72161865234375, "rewards//std": 0.054741695523262024, "step": 56 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0114, "grad_norm": 3.382544994354248, "kl": 0.0015232005753205158, "learning_rate": 9.999963748053354e-07, "loss": 0.0001, "num_tokens": 493000.0, "reward": 0.73602294921875, "reward_std": 0.016367167234420776, "rewards//mean": 0.73602294921875, "rewards//std": 0.05018298700451851, "step": 57 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0116, "grad_norm": 3.4117281436920166, "kl": 0.0017642830134718679, "learning_rate": 9.99995065709415e-07, "loss": 0.0001, "num_tokens": 501632.0, "reward": 0.72265625, "reward_std": 0.014317058026790619, "rewards//mean": 0.72265625, "rewards//std": 0.04878905043005943, "step": 58 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0118, "grad_norm": 3.2561867237091064, "kl": 0.0014012602186994627, "learning_rate": 9.999935552155421e-07, "loss": 0.0001, "num_tokens": 510288.0, "reward": 0.726318359375, "reward_std": 0.014060637913644314, "rewards//mean": 0.726318359375, "rewards//std": 0.04617677628993988, "step": 59 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.012, "grad_norm": 3.177072286605835, "kl": 0.0016652587437420152, "learning_rate": 9.99991843324325e-07, "loss": 0.0001, "num_tokens": 518952.0, "reward": 0.71875, "reward_std": 0.018966306000947952, "rewards//mean": 0.71875, "rewards//std": 0.05035256966948509, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0122, "grad_norm": 3.005466938018799, "kl": 0.0016885494842426851, "learning_rate": 9.999899300364532e-07, "loss": 0.0001, "num_tokens": 527520.0, "reward": 0.71490478515625, "reward_std": 0.01147861871868372, "rewards//mean": 0.71490478515625, "rewards//std": 0.04259505867958069, "step": 61 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0124, "grad_norm": 3.3473289012908936, "kl": 0.0016567637867410667, "learning_rate": 9.999878153526972e-07, "loss": 0.0001, "num_tokens": 536112.0, "reward": 0.70928955078125, "reward_std": 0.018018949776887894, "rewards//mean": 0.70928955078125, "rewards//std": 0.05387454107403755, "step": 62 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0126, "grad_norm": 3.2961602210998535, "kl": 0.002063060695945751, "learning_rate": 9.999854992739093e-07, "loss": 0.0001, "num_tokens": 544736.0, "reward": 0.6990966796875, "reward_std": 0.012440960854291916, "rewards//mean": 0.6990966796875, "rewards//std": 0.04440177232027054, "step": 63 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0128, "grad_norm": 3.2076351642608643, "kl": 0.0022169264702824876, "learning_rate": 9.999829818010219e-07, "loss": 0.0001, "num_tokens": 553408.0, "reward": 0.71697998046875, "reward_std": 0.015242921188473701, "rewards//mean": 0.71697998046875, "rewards//std": 0.04653756693005562, "step": 64 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.013, "grad_norm": 3.063619375228882, "kl": 0.00204440436209552, "learning_rate": 9.999802629350491e-07, "loss": 0.0001, "num_tokens": 562064.0, "reward": 0.72161865234375, "reward_std": 0.0141282444819808, "rewards//mean": 0.72161865234375, "rewards//std": 0.046424560248851776, "step": 65 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0132, "grad_norm": 2.984975576400757, "kl": 0.002333598895347677, "learning_rate": 9.999773426770863e-07, "loss": 0.0001, "num_tokens": 570664.0, "reward": 0.71575927734375, "reward_std": 0.011713966727256775, "rewards//mean": 0.71575927734375, "rewards//std": 0.04769832640886307, "step": 66 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0134, "grad_norm": 3.0676863193511963, "kl": 0.002809927915222943, "learning_rate": 9.999742210283097e-07, "loss": 0.0001, "num_tokens": 579360.0, "reward": 0.71435546875, "reward_std": 0.011259706690907478, "rewards//mean": 0.71435546875, "rewards//std": 0.048482783138751984, "step": 67 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0136, "grad_norm": 2.854644298553467, "kl": 0.0022947356701479293, "learning_rate": 9.999708979899767e-07, "loss": 0.0001, "num_tokens": 587992.0, "reward": 0.7469482421875, "reward_std": 0.015341498889029026, "rewards//mean": 0.7469482421875, "rewards//std": 0.039588894695043564, "step": 68 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0138, "grad_norm": 2.998915195465088, "kl": 0.0029584673466160893, "learning_rate": 9.999673735634259e-07, "loss": 0.0001, "num_tokens": 596608.0, "reward": 0.69549560546875, "reward_std": 0.01312573067843914, "rewards//mean": 0.69549560546875, "rewards//std": 0.036401353776454926, "step": 69 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.014, "grad_norm": 3.1461427211761475, "kl": 0.003096007196290884, "learning_rate": 9.999636477500764e-07, "loss": 0.0001, "num_tokens": 605248.0, "reward": 0.69873046875, "reward_std": 0.02277829684317112, "rewards//mean": 0.69873046875, "rewards//std": 0.044763218611478806, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0142, "grad_norm": 3.069575071334839, "kl": 0.003110720237600617, "learning_rate": 9.999597205514296e-07, "loss": 0.0001, "num_tokens": 613824.0, "reward": 0.69732666015625, "reward_std": 0.018011249601840973, "rewards//mean": 0.69732666015625, "rewards//std": 0.04875921458005905, "step": 71 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0144, "grad_norm": 3.1159253120422363, "kl": 0.00321734477620339, "learning_rate": 9.999555919690672e-07, "loss": 0.0001, "num_tokens": 622328.0, "reward": 0.74285888671875, "reward_std": 0.016840215772390366, "rewards//mean": 0.74285888671875, "rewards//std": 0.06359697133302689, "step": 72 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0146, "grad_norm": 3.0868067741394043, "kl": 0.003257128431869205, "learning_rate": 9.99951262004652e-07, "loss": 0.0001, "num_tokens": 630976.0, "reward": 0.6861572265625, "reward_std": 0.017197417095303535, "rewards//mean": 0.6861572265625, "rewards//std": 0.04307975620031357, "step": 73 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0148, "grad_norm": 2.944159984588623, "kl": 0.0035032854502787814, "learning_rate": 9.999467306599285e-07, "loss": 0.0001, "num_tokens": 639624.0, "reward": 0.73211669921875, "reward_std": 0.012926152907311916, "rewards//mean": 0.73211669921875, "rewards//std": 0.03628389537334442, "step": 74 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.015, "grad_norm": 3.0424153804779053, "kl": 0.004879061205429025, "learning_rate": 9.999419979367214e-07, "loss": 0.0002, "num_tokens": 648320.0, "reward": 0.704345703125, "reward_std": 0.014891589060425758, "rewards//mean": 0.704345703125, "rewards//std": 0.057871606200933456, "step": 75 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0152, "grad_norm": 3.2213408946990967, "kl": 0.004950453294441104, "learning_rate": 9.999370638369376e-07, "loss": 0.0002, "num_tokens": 657080.0, "reward": 0.7288818359375, "reward_std": 0.014744549989700317, "rewards//mean": 0.7288818359375, "rewards//std": 0.04380178079009056, "step": 76 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0154, "grad_norm": 3.053184747695923, "kl": 0.0044919641222804785, "learning_rate": 9.99931928362564e-07, "loss": 0.0002, "num_tokens": 665720.0, "reward": 0.6983642578125, "reward_std": 0.0167398564517498, "rewards//mean": 0.6983642578125, "rewards//std": 0.06560251861810684, "step": 77 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0156, "grad_norm": 3.2201783657073975, "kl": 0.00456669855338987, "learning_rate": 9.999265915156696e-07, "loss": 0.0002, "num_tokens": 674336.0, "reward": 0.7213134765625, "reward_std": 0.014512969180941582, "rewards//mean": 0.7213134765625, "rewards//std": 0.05112217366695404, "step": 78 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0158, "grad_norm": 3.2414867877960205, "kl": 0.004756899201311171, "learning_rate": 9.999210532984038e-07, "loss": 0.0002, "num_tokens": 682968.0, "reward": 0.68731689453125, "reward_std": 0.01768694445490837, "rewards//mean": 0.68731689453125, "rewards//std": 0.06721082329750061, "step": 79 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.016, "grad_norm": 2.93839430809021, "kl": 0.005078462738310918, "learning_rate": 9.999153137129977e-07, "loss": 0.0002, "num_tokens": 691640.0, "reward": 0.740478515625, "reward_std": 0.011827691458165646, "rewards//mean": 0.740478515625, "rewards//std": 0.06183329224586487, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0162, "grad_norm": 3.0919079780578613, "kl": 0.00472830337821506, "learning_rate": 9.999093727617628e-07, "loss": 0.0002, "num_tokens": 700264.0, "reward": 0.7027587890625, "reward_std": 0.015644310042262077, "rewards//mean": 0.7027587890625, "rewards//std": 0.039406463503837585, "step": 81 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0164, "grad_norm": 3.026963710784912, "kl": 0.005380780916311778, "learning_rate": 9.999032304470924e-07, "loss": 0.0002, "num_tokens": 708984.0, "reward": 0.73486328125, "reward_std": 0.014729069545865059, "rewards//mean": 0.73486328125, "rewards//std": 0.043728627264499664, "step": 82 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0166, "grad_norm": 3.0130741596221924, "kl": 0.006531667575472966, "learning_rate": 9.998968867714608e-07, "loss": 0.0003, "num_tokens": 717568.0, "reward": 0.72998046875, "reward_std": 0.012801310047507286, "rewards//mean": 0.72998046875, "rewards//std": 0.04702718183398247, "step": 83 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0168, "grad_norm": 3.0457839965820312, "kl": 0.0055532880942337215, "learning_rate": 9.998903417374226e-07, "loss": 0.0002, "num_tokens": 726304.0, "reward": 0.710693359375, "reward_std": 0.01631096936762333, "rewards//mean": 0.710693359375, "rewards//std": 0.04329654574394226, "step": 84 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.017, "grad_norm": 3.1099348068237305, "kl": 0.005788850947283208, "learning_rate": 9.998835953476147e-07, "loss": 0.0002, "num_tokens": 735000.0, "reward": 0.7353515625, "reward_std": 0.016055503860116005, "rewards//mean": 0.7353515625, "rewards//std": 0.05147054046392441, "step": 85 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0172, "grad_norm": 3.253512144088745, "kl": 0.007270405476447195, "learning_rate": 9.998766476047545e-07, "loss": 0.0003, "num_tokens": 743648.0, "reward": 0.7015380859375, "reward_std": 0.01626693084836006, "rewards//mean": 0.7015380859375, "rewards//std": 0.06402271240949631, "step": 86 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0174, "grad_norm": 3.139979600906372, "kl": 0.008355276426300406, "learning_rate": 9.998694985116404e-07, "loss": 0.0003, "num_tokens": 752416.0, "reward": 0.7178955078125, "reward_std": 0.013003869913518429, "rewards//mean": 0.7178955078125, "rewards//std": 0.06749863922595978, "step": 87 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0176, "grad_norm": 3.1358847618103027, "kl": 0.007668178965104744, "learning_rate": 9.99862148071152e-07, "loss": 0.0003, "num_tokens": 761040.0, "reward": 0.73675537109375, "reward_std": 0.018933400511741638, "rewards//mean": 0.73675537109375, "rewards//std": 0.06281549483537674, "step": 88 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0178, "grad_norm": 3.1049060821533203, "kl": 0.007956226007081568, "learning_rate": 9.998545962862501e-07, "loss": 0.0003, "num_tokens": 769656.0, "reward": 0.743408203125, "reward_std": 0.016416165977716446, "rewards//mean": 0.743408203125, "rewards//std": 0.05655932053923607, "step": 89 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.018, "grad_norm": 2.9268035888671875, "kl": 0.008059439191129059, "learning_rate": 9.998468431599767e-07, "loss": 0.0003, "num_tokens": 778248.0, "reward": 0.7052001953125, "reward_std": 0.016768010333180428, "rewards//mean": 0.7052001953125, "rewards//std": 0.022466329857707024, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0182, "grad_norm": 3.3397057056427, "kl": 0.010496469039935619, "learning_rate": 9.998388886954545e-07, "loss": 0.0004, "num_tokens": 786856.0, "reward": 0.69293212890625, "reward_std": 0.014789804816246033, "rewards//mean": 0.69293212890625, "rewards//std": 0.069489985704422, "step": 91 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0184, "grad_norm": 3.0880322456359863, "kl": 0.009906304272590205, "learning_rate": 9.998307328958877e-07, "loss": 0.0004, "num_tokens": 795544.0, "reward": 0.6934814453125, "reward_std": 0.015139150433242321, "rewards//mean": 0.6934814453125, "rewards//std": 0.06462985277175903, "step": 92 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0186, "grad_norm": 3.5390796661376953, "kl": 0.010611087462166324, "learning_rate": 9.998223757645617e-07, "loss": 0.0004, "num_tokens": 804104.0, "reward": 0.7440185546875, "reward_std": 0.013973723165690899, "rewards//mean": 0.7440185546875, "rewards//std": 0.04960612207651138, "step": 93 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0188, "grad_norm": 3.8798980712890625, "kl": 0.011519388906890526, "learning_rate": 9.998138173048423e-07, "loss": 0.0005, "num_tokens": 812768.0, "reward": 0.7451171875, "reward_std": 0.015056892298161983, "rewards//mean": 0.7451171875, "rewards//std": 0.0555574931204319, "step": 94 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.019, "grad_norm": 3.1219942569732666, "kl": 0.009766744391527027, "learning_rate": 9.99805057520177e-07, "loss": 0.0004, "num_tokens": 821400.0, "reward": 0.74365234375, "reward_std": 0.013678168877959251, "rewards//mean": 0.74365234375, "rewards//std": 0.046017181128263474, "step": 95 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0192, "grad_norm": 2.997413396835327, "kl": 0.00959825367317535, "learning_rate": 9.997960964140945e-07, "loss": 0.0004, "num_tokens": 829952.0, "reward": 0.7000732421875, "reward_std": 0.016240671277046204, "rewards//mean": 0.7000732421875, "rewards//std": 0.03586537390947342, "step": 96 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0194, "grad_norm": 3.278810501098633, "kl": 0.011237991857342422, "learning_rate": 9.99786933990204e-07, "loss": 0.0004, "num_tokens": 838520.0, "reward": 0.71533203125, "reward_std": 0.011809727177023888, "rewards//mean": 0.71533203125, "rewards//std": 0.04853271320462227, "step": 97 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0196, "grad_norm": 2.8756959438323975, "kl": 0.010547184152528644, "learning_rate": 9.997775702521965e-07, "loss": 0.0004, "num_tokens": 847128.0, "reward": 0.707275390625, "reward_std": 0.017715217545628548, "rewards//mean": 0.707275390625, "rewards//std": 0.07538240402936935, "step": 98 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0198, "grad_norm": 3.121904134750366, "kl": 0.012521670694695786, "learning_rate": 9.997680052038434e-07, "loss": 0.0005, "num_tokens": 855824.0, "reward": 0.69482421875, "reward_std": 0.01390664093196392, "rewards//mean": 0.69482421875, "rewards//std": 0.05070488154888153, "step": 99 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.02, "grad_norm": 3.230713129043579, "kl": 0.012724154919851571, "learning_rate": 9.997582388489973e-07, "loss": 0.0005, "num_tokens": 864520.0, "reward": 0.71173095703125, "reward_std": 0.015987861901521683, "rewards//mean": 0.71173095703125, "rewards//std": 0.08442246913909912, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0202, "grad_norm": 3.1842575073242188, "kl": 0.014311595150502399, "learning_rate": 9.997482711915925e-07, "loss": 0.0006, "num_tokens": 873152.0, "reward": 0.70654296875, "reward_std": 0.017925892025232315, "rewards//mean": 0.70654296875, "rewards//std": 0.0551220178604126, "step": 101 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0204, "grad_norm": 2.9082493782043457, "kl": 0.010199115553405136, "learning_rate": 9.99738102235644e-07, "loss": 0.0004, "num_tokens": 881824.0, "reward": 0.7142333984375, "reward_std": 0.014553748071193695, "rewards//mean": 0.7142333984375, "rewards//std": 0.04478874057531357, "step": 102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0206, "grad_norm": 3.063404083251953, "kl": 0.012870212813140824, "learning_rate": 9.997277319852474e-07, "loss": 0.0005, "num_tokens": 890368.0, "reward": 0.748779296875, "reward_std": 0.011821390129625797, "rewards//mean": 0.748779296875, "rewards//std": 0.04491109028458595, "step": 103 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0208, "grad_norm": 3.2403464317321777, "kl": 0.015398608316900209, "learning_rate": 9.997171604445802e-07, "loss": 0.0006, "num_tokens": 899128.0, "reward": 0.762451171875, "reward_std": 0.019849859178066254, "rewards//mean": 0.762451171875, "rewards//std": 0.04154951870441437, "step": 104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.021, "grad_norm": 3.1215336322784424, "kl": 0.012001491704722866, "learning_rate": 9.997063876179007e-07, "loss": 0.0005, "num_tokens": 907808.0, "reward": 0.71649169921875, "reward_std": 0.015484604984521866, "rewards//mean": 0.71649169921875, "rewards//std": 0.05297234654426575, "step": 105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0212, "grad_norm": 2.9595353603363037, "kl": 0.015639638353604823, "learning_rate": 9.996954135095478e-07, "loss": 0.0006, "num_tokens": 916384.0, "reward": 0.74432373046875, "reward_std": 0.016129087656736374, "rewards//mean": 0.74432373046875, "rewards//std": 0.047977738082408905, "step": 106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0214, "grad_norm": 2.8981754779815674, "kl": 0.016912667255382985, "learning_rate": 9.996842381239422e-07, "loss": 0.0007, "num_tokens": 925000.0, "reward": 0.7364501953125, "reward_std": 0.013939950615167618, "rewards//mean": 0.7364501953125, "rewards//std": 0.031581275165081024, "step": 107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0216, "grad_norm": 2.97182559967041, "kl": 0.01626437672530301, "learning_rate": 9.996728614655853e-07, "loss": 0.0007, "num_tokens": 933680.0, "reward": 0.724609375, "reward_std": 0.015291033312678337, "rewards//mean": 0.724609375, "rewards//std": 0.03565902262926102, "step": 108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0218, "grad_norm": 2.94159197807312, "kl": 0.01212124724406749, "learning_rate": 9.996612835390594e-07, "loss": 0.0005, "num_tokens": 942360.0, "reward": 0.712890625, "reward_std": 0.011505991220474243, "rewards//mean": 0.712890625, "rewards//std": 0.0495036356151104, "step": 109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.022, "grad_norm": 3.0552749633789062, "kl": 0.018995069694938138, "learning_rate": 9.996495043490283e-07, "loss": 0.0008, "num_tokens": 951000.0, "reward": 0.74951171875, "reward_std": 0.012679621577262878, "rewards//mean": 0.74951171875, "rewards//std": 0.04841279238462448, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0222, "grad_norm": 2.890538215637207, "kl": 0.017626553802983835, "learning_rate": 9.996375239002368e-07, "loss": 0.0007, "num_tokens": 959688.0, "reward": 0.68048095703125, "reward_std": 0.012808963656425476, "rewards//mean": 0.68048095703125, "rewards//std": 0.06301399320363998, "step": 111 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0224, "grad_norm": 3.0831246376037598, "kl": 0.01691323594423011, "learning_rate": 9.996253421975102e-07, "loss": 0.0007, "num_tokens": 968352.0, "reward": 0.72821044921875, "reward_std": 0.013352443464100361, "rewards//mean": 0.72821044921875, "rewards//std": 0.047239430248737335, "step": 112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0226, "grad_norm": 3.094242572784424, "kl": 0.017311632866039872, "learning_rate": 9.996129592457556e-07, "loss": 0.0007, "num_tokens": 976936.0, "reward": 0.72747802734375, "reward_std": 0.01294796634465456, "rewards//mean": 0.72747802734375, "rewards//std": 0.04731083661317825, "step": 113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0228, "grad_norm": 3.1343140602111816, "kl": 0.01906962221255526, "learning_rate": 9.996003750499607e-07, "loss": 0.0008, "num_tokens": 985552.0, "reward": 0.7279052734375, "reward_std": 0.013005741871893406, "rewards//mean": 0.7279052734375, "rewards//std": 0.040284812450408936, "step": 114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.023, "grad_norm": 2.9134888648986816, "kl": 0.015599035657942295, "learning_rate": 9.995875896151944e-07, "loss": 0.0006, "num_tokens": 994264.0, "reward": 0.7315673828125, "reward_std": 0.01569860428571701, "rewards//mean": 0.7315673828125, "rewards//std": 0.03527295961976051, "step": 115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0232, "grad_norm": 2.9819822311401367, "kl": 0.021106205589603633, "learning_rate": 9.99574602946607e-07, "loss": 0.0008, "num_tokens": 1002832.0, "reward": 0.72247314453125, "reward_std": 0.014688769355416298, "rewards//mean": 0.72247314453125, "rewards//std": 0.03920132666826248, "step": 116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0234, "grad_norm": 2.9172799587249756, "kl": 0.019246674666646868, "learning_rate": 9.99561415049429e-07, "loss": 0.0008, "num_tokens": 1011448.0, "reward": 0.72149658203125, "reward_std": 0.015304185450077057, "rewards//mean": 0.72149658203125, "rewards//std": 0.03445466607809067, "step": 117 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0236, "grad_norm": 2.9187514781951904, "kl": 0.02077232871670276, "learning_rate": 9.99548025928973e-07, "loss": 0.0008, "num_tokens": 1020104.0, "reward": 0.71710205078125, "reward_std": 0.015331927686929703, "rewards//mean": 0.71710205078125, "rewards//std": 0.03905509039759636, "step": 118 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0238, "grad_norm": 2.793119430541992, "kl": 0.01499595059431158, "learning_rate": 9.995344355906318e-07, "loss": 0.0006, "num_tokens": 1028696.0, "reward": 0.72967529296875, "reward_std": 0.010924267582595348, "rewards//mean": 0.72967529296875, "rewards//std": 0.026722678914666176, "step": 119 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.024, "grad_norm": 2.850172519683838, "kl": 0.016416702361311764, "learning_rate": 9.995206440398796e-07, "loss": 0.0007, "num_tokens": 1037384.0, "reward": 0.71246337890625, "reward_std": 0.011186916381120682, "rewards//mean": 0.71246337890625, "rewards//std": 0.03915302827954292, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0242, "grad_norm": 3.1275367736816406, "kl": 0.019138285715598613, "learning_rate": 9.995066512822718e-07, "loss": 0.0008, "num_tokens": 1046176.0, "reward": 0.7113037109375, "reward_std": 0.012143395841121674, "rewards//mean": 0.7113037109375, "rewards//std": 0.040131207555532455, "step": 121 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0244, "grad_norm": 2.948559284210205, "kl": 0.023869303578976542, "learning_rate": 9.994924573234446e-07, "loss": 0.001, "num_tokens": 1054816.0, "reward": 0.72149658203125, "reward_std": 0.01686747930943966, "rewards//mean": 0.72149658203125, "rewards//std": 0.05836670845746994, "step": 122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0246, "grad_norm": 2.984609603881836, "kl": 0.021968798915622756, "learning_rate": 9.994780621691154e-07, "loss": 0.0009, "num_tokens": 1063496.0, "reward": 0.7352294921875, "reward_std": 0.01451707910746336, "rewards//mean": 0.7352294921875, "rewards//std": 0.041131168603897095, "step": 123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0248, "grad_norm": 2.9818739891052246, "kl": 0.024137288215570152, "learning_rate": 9.994634658250824e-07, "loss": 0.001, "num_tokens": 1072104.0, "reward": 0.74267578125, "reward_std": 0.012883680872619152, "rewards//mean": 0.74267578125, "rewards//std": 0.03866676986217499, "step": 124 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.025, "grad_norm": 2.7461442947387695, "kl": 0.0202828252222389, "learning_rate": 9.994486682972252e-07, "loss": 0.0008, "num_tokens": 1080752.0, "reward": 0.7342529296875, "reward_std": 0.013745720498263836, "rewards//mean": 0.7342529296875, "rewards//std": 0.0387197807431221, "step": 125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0252, "grad_norm": 2.9171698093414307, "kl": 0.022235158481635153, "learning_rate": 9.99433669591504e-07, "loss": 0.0009, "num_tokens": 1089368.0, "reward": 0.72381591796875, "reward_std": 0.012552469968795776, "rewards//mean": 0.72381591796875, "rewards//std": 0.0521097406744957, "step": 126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0254, "grad_norm": 2.6717565059661865, "kl": 0.01895753329154104, "learning_rate": 9.994184697139604e-07, "loss": 0.0008, "num_tokens": 1097992.0, "reward": 0.73980712890625, "reward_std": 0.011169027537107468, "rewards//mean": 0.73980712890625, "rewards//std": 0.026416730135679245, "step": 127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0256, "grad_norm": 2.8336143493652344, "kl": 0.02507422905182466, "learning_rate": 9.99403068670717e-07, "loss": 0.001, "num_tokens": 1106576.0, "reward": 0.7236328125, "reward_std": 0.01609623059630394, "rewards//mean": 0.7236328125, "rewards//std": 0.04100455343723297, "step": 128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0258, "grad_norm": 2.910475969314575, "kl": 0.02157846326008439, "learning_rate": 9.993874664679772e-07, "loss": 0.0009, "num_tokens": 1115160.0, "reward": 0.71417236328125, "reward_std": 0.012449681758880615, "rewards//mean": 0.71417236328125, "rewards//std": 0.044957876205444336, "step": 129 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.026, "grad_norm": 2.733341693878174, "kl": 0.018211158807389438, "learning_rate": 9.993716631120258e-07, "loss": 0.0007, "num_tokens": 1123808.0, "reward": 0.7242431640625, "reward_std": 0.01214703917503357, "rewards//mean": 0.7242431640625, "rewards//std": 0.0530952624976635, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0262, "grad_norm": 2.6225228309631348, "kl": 0.029907271615229547, "learning_rate": 9.99355658609228e-07, "loss": 0.0012, "num_tokens": 1132528.0, "reward": 0.716552734375, "reward_std": 0.01618252508342266, "rewards//mean": 0.716552734375, "rewards//std": 0.04933639243245125, "step": 131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0264, "grad_norm": 2.5123965740203857, "kl": 0.03157387347891927, "learning_rate": 9.993394529660306e-07, "loss": 0.0013, "num_tokens": 1141160.0, "reward": 0.73797607421875, "reward_std": 0.012247065082192421, "rewards//mean": 0.73797607421875, "rewards//std": 0.038407862186431885, "step": 132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0266, "grad_norm": 2.660794973373413, "kl": 0.030565934779588133, "learning_rate": 9.993230461889615e-07, "loss": 0.0012, "num_tokens": 1149744.0, "reward": 0.72650146484375, "reward_std": 0.012983234599232674, "rewards//mean": 0.72650146484375, "rewards//std": 0.0365544818341732, "step": 133 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0268, "grad_norm": 2.8524365425109863, "kl": 0.028304936771746725, "learning_rate": 9.993064382846289e-07, "loss": 0.0011, "num_tokens": 1158344.0, "reward": 0.72186279296875, "reward_std": 0.014238173142075539, "rewards//mean": 0.72186279296875, "rewards//std": 0.04674139618873596, "step": 134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.027, "grad_norm": 2.772726535797119, "kl": 0.028303387050982565, "learning_rate": 9.992896292597228e-07, "loss": 0.0011, "num_tokens": 1166920.0, "reward": 0.69146728515625, "reward_std": 0.01865844428539276, "rewards//mean": 0.69146728515625, "rewards//std": 0.03763829916715622, "step": 135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0272, "grad_norm": 2.759188652038574, "kl": 0.027150681242346764, "learning_rate": 9.992726191210137e-07, "loss": 0.0011, "num_tokens": 1175528.0, "reward": 0.7373046875, "reward_std": 0.011674551293253899, "rewards//mean": 0.7373046875, "rewards//std": 0.039217203855514526, "step": 136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0274, "grad_norm": 2.642434597015381, "kl": 0.027422477258369327, "learning_rate": 9.992554078753533e-07, "loss": 0.0011, "num_tokens": 1184144.0, "reward": 0.7293701171875, "reward_std": 0.01614459417760372, "rewards//mean": 0.7293701171875, "rewards//std": 0.05587686598300934, "step": 137 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0276, "grad_norm": 2.7088518142700195, "kl": 0.02350567257963121, "learning_rate": 9.992379955296745e-07, "loss": 0.0009, "num_tokens": 1192832.0, "reward": 0.73870849609375, "reward_std": 0.01621604897081852, "rewards//mean": 0.73870849609375, "rewards//std": 0.044315218925476074, "step": 138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0278, "grad_norm": 2.5840044021606445, "kl": 0.03757087094709277, "learning_rate": 9.992203820909905e-07, "loss": 0.0015, "num_tokens": 1201472.0, "reward": 0.71124267578125, "reward_std": 0.012270974926650524, "rewards//mean": 0.71124267578125, "rewards//std": 0.05214052274823189, "step": 139 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.028, "grad_norm": 2.7272589206695557, "kl": 0.03142879402730614, "learning_rate": 9.992025675663965e-07, "loss": 0.0013, "num_tokens": 1210176.0, "reward": 0.7528076171875, "reward_std": 0.012411152943968773, "rewards//mean": 0.7528076171875, "rewards//std": 0.03444615378975868, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0282, "grad_norm": 2.7319650650024414, "kl": 0.028858942445367575, "learning_rate": 9.991845519630676e-07, "loss": 0.0012, "num_tokens": 1218872.0, "reward": 0.7288818359375, "reward_std": 0.012398846447467804, "rewards//mean": 0.7288818359375, "rewards//std": 0.03197569400072098, "step": 141 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0284, "grad_norm": 3.004795789718628, "kl": 0.0333898447570391, "learning_rate": 9.991663352882613e-07, "loss": 0.0013, "num_tokens": 1227624.0, "reward": 0.73577880859375, "reward_std": 0.01382610946893692, "rewards//mean": 0.73577880859375, "rewards//std": 0.041894879192113876, "step": 142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0286, "grad_norm": 2.6800928115844727, "kl": 0.03263287292793393, "learning_rate": 9.991479175493148e-07, "loss": 0.0013, "num_tokens": 1236264.0, "reward": 0.74127197265625, "reward_std": 0.012902824208140373, "rewards//mean": 0.74127197265625, "rewards//std": 0.036892492324113846, "step": 143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0288, "grad_norm": 2.9141860008239746, "kl": 0.027082599932327867, "learning_rate": 9.991292987536468e-07, "loss": 0.0011, "num_tokens": 1244984.0, "reward": 0.71527099609375, "reward_std": 0.01237812452018261, "rewards//mean": 0.71527099609375, "rewards//std": 0.06423194706439972, "step": 144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.029, "grad_norm": 2.536769151687622, "kl": 0.03280840802472085, "learning_rate": 9.991104789087569e-07, "loss": 0.0013, "num_tokens": 1253544.0, "reward": 0.69854736328125, "reward_std": 0.015090405941009521, "rewards//mean": 0.69854736328125, "rewards//std": 0.0490887388586998, "step": 145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0292, "grad_norm": 2.6251041889190674, "kl": 0.03762986068613827, "learning_rate": 9.990914580222255e-07, "loss": 0.0015, "num_tokens": 1262272.0, "reward": 0.75982666015625, "reward_std": 0.013205408118665218, "rewards//mean": 0.75982666015625, "rewards//std": 0.04166662320494652, "step": 146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0294, "grad_norm": 2.7780847549438477, "kl": 0.029165279876906425, "learning_rate": 9.990722361017149e-07, "loss": 0.0012, "num_tokens": 1270984.0, "reward": 0.737548828125, "reward_std": 0.013255832716822624, "rewards//mean": 0.737548828125, "rewards//std": 0.04294831305742264, "step": 147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0296, "grad_norm": 2.728917360305786, "kl": 0.03120983554981649, "learning_rate": 9.990528131549671e-07, "loss": 0.0012, "num_tokens": 1279664.0, "reward": 0.726806640625, "reward_std": 0.016245994716882706, "rewards//mean": 0.726806640625, "rewards//std": 0.04418252781033516, "step": 148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0298, "grad_norm": 2.602877140045166, "kl": 0.03489436558447778, "learning_rate": 9.990331891898058e-07, "loss": 0.0014, "num_tokens": 1288360.0, "reward": 0.72125244140625, "reward_std": 0.015676885843276978, "rewards//mean": 0.72125244140625, "rewards//std": 0.04169785603880882, "step": 149 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.03, "grad_norm": 2.4549598693847656, "kl": 0.030647181847598404, "learning_rate": 9.990133642141357e-07, "loss": 0.0012, "num_tokens": 1297032.0, "reward": 0.74334716796875, "reward_std": 0.013695335015654564, "rewards//mean": 0.74334716796875, "rewards//std": 0.03441246226429939, "step": 150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0302, "grad_norm": 2.680105209350586, "kl": 0.03840164153371006, "learning_rate": 9.989933382359422e-07, "loss": 0.0015, "num_tokens": 1305632.0, "reward": 0.71258544921875, "reward_std": 0.012445718050003052, "rewards//mean": 0.71258544921875, "rewards//std": 0.04514201357960701, "step": 151 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0304, "grad_norm": 2.376408100128174, "kl": 0.03482830408029258, "learning_rate": 9.989731112632916e-07, "loss": 0.0014, "num_tokens": 1314272.0, "reward": 0.72320556640625, "reward_std": 0.009584907442331314, "rewards//mean": 0.72320556640625, "rewards//std": 0.04050455242395401, "step": 152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0306, "grad_norm": 2.7264437675476074, "kl": 0.038398853910621256, "learning_rate": 9.989526833043316e-07, "loss": 0.0015, "num_tokens": 1322960.0, "reward": 0.76617431640625, "reward_std": 0.01662430167198181, "rewards//mean": 0.76617431640625, "rewards//std": 0.05045284330844879, "step": 153 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0308, "grad_norm": 2.79830002784729, "kl": 0.045701892115175724, "learning_rate": 9.989320543672903e-07, "loss": 0.0018, "num_tokens": 1331608.0, "reward": 0.7325439453125, "reward_std": 0.015923475846648216, "rewards//mean": 0.7325439453125, "rewards//std": 0.051498621702194214, "step": 154 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.031, "grad_norm": 2.336735486984253, "kl": 0.0385560302529484, "learning_rate": 9.989112244604771e-07, "loss": 0.0015, "num_tokens": 1340352.0, "reward": 0.7415771484375, "reward_std": 0.010023964568972588, "rewards//mean": 0.7415771484375, "rewards//std": 0.04613660275936127, "step": 155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0312, "grad_norm": 2.459691286087036, "kl": 0.02748531079851091, "learning_rate": 9.988901935922825e-07, "loss": 0.0011, "num_tokens": 1349024.0, "reward": 0.72869873046875, "reward_std": 0.011690299957990646, "rewards//mean": 0.72869873046875, "rewards//std": 0.032983966171741486, "step": 156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0314, "grad_norm": 2.55267333984375, "kl": 0.03610196016961709, "learning_rate": 9.988689617711776e-07, "loss": 0.0014, "num_tokens": 1357544.0, "reward": 0.73175048828125, "reward_std": 0.014921002089977264, "rewards//mean": 0.73175048828125, "rewards//std": 0.043061964213848114, "step": 157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0316, "grad_norm": 2.3918230533599854, "kl": 0.042030082491692156, "learning_rate": 9.988475290057143e-07, "loss": 0.0017, "num_tokens": 1366224.0, "reward": 0.73443603515625, "reward_std": 0.011331222020089626, "rewards//mean": 0.73443603515625, "rewards//std": 0.04396059736609459, "step": 158 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0318, "grad_norm": 2.446019172668457, "kl": 0.040930149145424366, "learning_rate": 9.988258953045262e-07, "loss": 0.0016, "num_tokens": 1374848.0, "reward": 0.73486328125, "reward_std": 0.012925570830702782, "rewards//mean": 0.73486328125, "rewards//std": 0.043171197175979614, "step": 159 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.032, "grad_norm": 2.6987080574035645, "kl": 0.036448456230573356, "learning_rate": 9.988040606763272e-07, "loss": 0.0015, "num_tokens": 1383456.0, "reward": 0.7001953125, "reward_std": 0.010226636193692684, "rewards//mean": 0.7001953125, "rewards//std": 0.05315125733613968, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0322, "grad_norm": 2.816415309906006, "kl": 0.046790738590061665, "learning_rate": 9.98782025129912e-07, "loss": 0.0019, "num_tokens": 1392112.0, "reward": 0.73614501953125, "reward_std": 0.01567969098687172, "rewards//mean": 0.73614501953125, "rewards//std": 0.04891760274767876, "step": 161 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0324, "grad_norm": 2.4799399375915527, "kl": 0.050143138971179724, "learning_rate": 9.987597886741568e-07, "loss": 0.002, "num_tokens": 1400784.0, "reward": 0.7564697265625, "reward_std": 0.012243933975696564, "rewards//mean": 0.7564697265625, "rewards//std": 0.045272815972566605, "step": 162 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0326, "grad_norm": 2.50376558303833, "kl": 0.041093296254985034, "learning_rate": 9.987373513180184e-07, "loss": 0.0016, "num_tokens": 1409344.0, "reward": 0.74749755859375, "reward_std": 0.010359032079577446, "rewards//mean": 0.74749755859375, "rewards//std": 0.028775472193956375, "step": 163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0328, "grad_norm": 2.3595211505889893, "kl": 0.04736456251703203, "learning_rate": 9.987147130705347e-07, "loss": 0.0019, "num_tokens": 1417920.0, "reward": 0.7374267578125, "reward_std": 0.01212608814239502, "rewards//mean": 0.7374267578125, "rewards//std": 0.039145924150943756, "step": 164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.033, "grad_norm": 2.5590295791625977, "kl": 0.04534601035993546, "learning_rate": 9.98691873940824e-07, "loss": 0.0018, "num_tokens": 1426608.0, "reward": 0.72821044921875, "reward_std": 0.011726150289177895, "rewards//mean": 0.72821044921875, "rewards//std": 0.045850057154893875, "step": 165 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0332, "grad_norm": 2.226454019546509, "kl": 0.047674126049969345, "learning_rate": 9.98668833938086e-07, "loss": 0.0019, "num_tokens": 1435216.0, "reward": 0.72979736328125, "reward_std": 0.010330487042665482, "rewards//mean": 0.72979736328125, "rewards//std": 0.03575774282217026, "step": 166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0334, "grad_norm": 2.772519588470459, "kl": 0.03843457077164203, "learning_rate": 9.986455930716016e-07, "loss": 0.0015, "num_tokens": 1443832.0, "reward": 0.69500732421875, "reward_std": 0.01148445438593626, "rewards//mean": 0.69500732421875, "rewards//std": 0.059181105345487595, "step": 167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0336, "grad_norm": 2.318955183029175, "kl": 0.05194610939361155, "learning_rate": 9.986221513507318e-07, "loss": 0.0021, "num_tokens": 1452488.0, "reward": 0.75311279296875, "reward_std": 0.011434704065322876, "rewards//mean": 0.75311279296875, "rewards//std": 0.03438781946897507, "step": 168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0338, "grad_norm": 2.503162145614624, "kl": 0.043565331608988345, "learning_rate": 9.985985087849191e-07, "loss": 0.0017, "num_tokens": 1461184.0, "reward": 0.73974609375, "reward_std": 0.011233117431402206, "rewards//mean": 0.73974609375, "rewards//std": 0.042623523622751236, "step": 169 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.034, "grad_norm": 2.350789785385132, "kl": 0.03935895999893546, "learning_rate": 9.985746653836866e-07, "loss": 0.0016, "num_tokens": 1469920.0, "reward": 0.7506103515625, "reward_std": 0.011013064533472061, "rewards//mean": 0.7506103515625, "rewards//std": 0.044315777719020844, "step": 170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0342, "grad_norm": 2.4444878101348877, "kl": 0.04686792800202966, "learning_rate": 9.985506211566386e-07, "loss": 0.0019, "num_tokens": 1478560.0, "reward": 0.72991943359375, "reward_std": 0.014785466715693474, "rewards//mean": 0.72991943359375, "rewards//std": 0.04804300516843796, "step": 171 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0344, "grad_norm": 2.1867735385894775, "kl": 0.04372467682696879, "learning_rate": 9.9852637611346e-07, "loss": 0.0017, "num_tokens": 1487232.0, "reward": 0.6925048828125, "reward_std": 0.011685654520988464, "rewards//mean": 0.6925048828125, "rewards//std": 0.0518384724855423, "step": 172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0346, "grad_norm": 2.351771354675293, "kl": 0.052237617783248425, "learning_rate": 9.98501930263917e-07, "loss": 0.0021, "num_tokens": 1495848.0, "reward": 0.720947265625, "reward_std": 0.012654958292841911, "rewards//mean": 0.720947265625, "rewards//std": 0.04899648576974869, "step": 173 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0348, "grad_norm": 2.2505569458007812, "kl": 0.050110013922676444, "learning_rate": 9.984772836178556e-07, "loss": 0.002, "num_tokens": 1504680.0, "reward": 0.74395751953125, "reward_std": 0.012244937010109425, "rewards//mean": 0.74395751953125, "rewards//std": 0.04974732547998428, "step": 174 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.035, "grad_norm": 2.456139087677002, "kl": 0.0581817498896271, "learning_rate": 9.984524361852043e-07, "loss": 0.0023, "num_tokens": 1513360.0, "reward": 0.71875, "reward_std": 0.009412692859768867, "rewards//mean": 0.71875, "rewards//std": 0.05139990895986557, "step": 175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0352, "grad_norm": 2.341621160507202, "kl": 0.050122945569455624, "learning_rate": 9.984273879759712e-07, "loss": 0.002, "num_tokens": 1522112.0, "reward": 0.7413330078125, "reward_std": 0.010432751849293709, "rewards//mean": 0.7413330078125, "rewards//std": 0.0366072952747345, "step": 176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0354, "grad_norm": 2.1102633476257324, "kl": 0.050346111645922065, "learning_rate": 9.984021390002457e-07, "loss": 0.002, "num_tokens": 1530848.0, "reward": 0.71044921875, "reward_std": 0.010294707491993904, "rewards//mean": 0.71044921875, "rewards//std": 0.04748334363102913, "step": 177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0356, "grad_norm": 2.223541736602783, "kl": 0.048459484125487506, "learning_rate": 9.983766892681985e-07, "loss": 0.0019, "num_tokens": 1539528.0, "reward": 0.72589111328125, "reward_std": 0.012135764583945274, "rewards//mean": 0.72589111328125, "rewards//std": 0.03706156834959984, "step": 178 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0358, "grad_norm": 2.265479326248169, "kl": 0.04988991259597242, "learning_rate": 9.983510387900802e-07, "loss": 0.002, "num_tokens": 1548192.0, "reward": 0.70721435546875, "reward_std": 0.011628516018390656, "rewards//mean": 0.70721435546875, "rewards//std": 0.05118945613503456, "step": 179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.036, "grad_norm": 2.2445435523986816, "kl": 0.05180193926207721, "learning_rate": 9.983251875762232e-07, "loss": 0.0021, "num_tokens": 1556856.0, "reward": 0.73931884765625, "reward_std": 0.011930052191019058, "rewards//mean": 0.73931884765625, "rewards//std": 0.049146369099617004, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0362, "grad_norm": 1.988173007965088, "kl": 0.04608521016780287, "learning_rate": 9.982991356370403e-07, "loss": 0.0018, "num_tokens": 1565488.0, "reward": 0.7384033203125, "reward_std": 0.012451891787350178, "rewards//mean": 0.7384033203125, "rewards//std": 0.053000520914793015, "step": 181 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0364, "grad_norm": 2.1265347003936768, "kl": 0.05836852011270821, "learning_rate": 9.98272882983025e-07, "loss": 0.0023, "num_tokens": 1574184.0, "reward": 0.7396240234375, "reward_std": 0.012791775166988373, "rewards//mean": 0.7396240234375, "rewards//std": 0.0466274619102478, "step": 182 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0366, "grad_norm": 2.3052797317504883, "kl": 0.05570454359985888, "learning_rate": 9.982464296247522e-07, "loss": 0.0022, "num_tokens": 1582888.0, "reward": 0.737548828125, "reward_std": 0.011044729501008987, "rewards//mean": 0.737548828125, "rewards//std": 0.03410276025533676, "step": 183 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0368, "grad_norm": 2.3991541862487793, "kl": 0.04946473188465461, "learning_rate": 9.98219775572877e-07, "loss": 0.002, "num_tokens": 1591408.0, "reward": 0.754150390625, "reward_std": 0.010761473327875137, "rewards//mean": 0.754150390625, "rewards//std": 0.03539912402629852, "step": 184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.037, "grad_norm": 2.1395928859710693, "kl": 0.055561707355082035, "learning_rate": 9.981929208381357e-07, "loss": 0.0022, "num_tokens": 1600088.0, "reward": 0.767333984375, "reward_std": 0.012706330046057701, "rewards//mean": 0.767333984375, "rewards//std": 0.03725268691778183, "step": 185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0372, "grad_norm": 2.055741786956787, "kl": 0.05260586831718683, "learning_rate": 9.981658654313456e-07, "loss": 0.0021, "num_tokens": 1608712.0, "reward": 0.72021484375, "reward_std": 0.00924680382013321, "rewards//mean": 0.72021484375, "rewards//std": 0.038509856909513474, "step": 186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0374, "grad_norm": 2.327390432357788, "kl": 0.054784362437203526, "learning_rate": 9.981386093634045e-07, "loss": 0.0022, "num_tokens": 1617400.0, "reward": 0.7587890625, "reward_std": 0.010368096642196178, "rewards//mean": 0.7587890625, "rewards//std": 0.02377607673406601, "step": 187 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0376, "grad_norm": 2.1157243251800537, "kl": 0.059309986187145114, "learning_rate": 9.98111152645291e-07, "loss": 0.0024, "num_tokens": 1625992.0, "reward": 0.7705078125, "reward_std": 0.01089094765484333, "rewards//mean": 0.7705078125, "rewards//std": 0.040253423154354095, "step": 188 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0378, "grad_norm": 2.4641757011413574, "kl": 0.05123629025183618, "learning_rate": 9.98083495288065e-07, "loss": 0.002, "num_tokens": 1634576.0, "reward": 0.71209716796875, "reward_std": 0.015146300196647644, "rewards//mean": 0.71209716796875, "rewards//std": 0.04571184143424034, "step": 189 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.038, "grad_norm": 2.296196699142456, "kl": 0.05166075681336224, "learning_rate": 9.980556373028665e-07, "loss": 0.0021, "num_tokens": 1643200.0, "reward": 0.73907470703125, "reward_std": 0.008769819512963295, "rewards//mean": 0.73907470703125, "rewards//std": 0.03626636788249016, "step": 190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0382, "grad_norm": 2.1193857192993164, "kl": 0.05269998754374683, "learning_rate": 9.98027578700917e-07, "loss": 0.0021, "num_tokens": 1651848.0, "reward": 0.70281982421875, "reward_std": 0.010136201977729797, "rewards//mean": 0.70281982421875, "rewards//std": 0.03540205955505371, "step": 191 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0384, "grad_norm": 2.1637582778930664, "kl": 0.051795421401038766, "learning_rate": 9.979993194935182e-07, "loss": 0.0021, "num_tokens": 1660472.0, "reward": 0.739013671875, "reward_std": 0.01057550311088562, "rewards//mean": 0.739013671875, "rewards//std": 0.04138597846031189, "step": 192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0386, "grad_norm": 2.1885793209075928, "kl": 0.05414393660612404, "learning_rate": 9.979708596920529e-07, "loss": 0.0022, "num_tokens": 1669128.0, "reward": 0.7567138671875, "reward_std": 0.009756932035088539, "rewards//mean": 0.7567138671875, "rewards//std": 0.029720351099967957, "step": 193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0388, "grad_norm": 2.267918109893799, "kl": 0.05917287478223443, "learning_rate": 9.97942199307985e-07, "loss": 0.0024, "num_tokens": 1677784.0, "reward": 0.76678466796875, "reward_std": 0.013261063024401665, "rewards//mean": 0.76678466796875, "rewards//std": 0.047738611698150635, "step": 194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.039, "grad_norm": 2.1623001098632812, "kl": 0.05870238086208701, "learning_rate": 9.97913338352859e-07, "loss": 0.0023, "num_tokens": 1686448.0, "reward": 0.73870849609375, "reward_std": 0.008617502637207508, "rewards//mean": 0.73870849609375, "rewards//std": 0.026070643216371536, "step": 195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0392, "grad_norm": 2.185559034347534, "kl": 0.05918487114831805, "learning_rate": 9.978842768382998e-07, "loss": 0.0024, "num_tokens": 1695072.0, "reward": 0.7078857421875, "reward_std": 0.011204719543457031, "rewards//mean": 0.7078857421875, "rewards//std": 0.02731688879430294, "step": 196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0394, "grad_norm": 2.2516632080078125, "kl": 0.054595754481852055, "learning_rate": 9.978550147760131e-07, "loss": 0.0022, "num_tokens": 1703680.0, "reward": 0.74151611328125, "reward_std": 0.01452554203569889, "rewards//mean": 0.74151611328125, "rewards//std": 0.04260678216814995, "step": 197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0396, "grad_norm": 2.036886215209961, "kl": 0.05572623899206519, "learning_rate": 9.978255521777862e-07, "loss": 0.0022, "num_tokens": 1712304.0, "reward": 0.734130859375, "reward_std": 0.008719809353351593, "rewards//mean": 0.734130859375, "rewards//std": 0.043896544724702835, "step": 198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0398, "grad_norm": 2.373351573944092, "kl": 0.0579618860501796, "learning_rate": 9.977958890554866e-07, "loss": 0.0023, "num_tokens": 1720936.0, "reward": 0.71710205078125, "reward_std": 0.01314060389995575, "rewards//mean": 0.71710205078125, "rewards//std": 0.034784361720085144, "step": 199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.04, "grad_norm": 3.3971798419952393, "kl": 0.05702702864073217, "learning_rate": 9.97766025421062e-07, "loss": 0.0023, "num_tokens": 1729552.0, "reward": 0.71942138671875, "reward_std": 0.014108622446656227, "rewards//mean": 0.71942138671875, "rewards//std": 0.03779282793402672, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0402, "grad_norm": 1.8567906618118286, "kl": 0.05714975926093757, "learning_rate": 9.977359612865422e-07, "loss": 0.0023, "num_tokens": 1738184.0, "reward": 0.7197265625, "reward_std": 0.012525375932455063, "rewards//mean": 0.7197265625, "rewards//std": 0.035187240689992905, "step": 201 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0404, "grad_norm": 2.1849992275238037, "kl": 0.061090546660125256, "learning_rate": 9.977056966640367e-07, "loss": 0.0024, "num_tokens": 1746792.0, "reward": 0.736572265625, "reward_std": 0.016942109912633896, "rewards//mean": 0.736572265625, "rewards//std": 0.040252674371004105, "step": 202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0406, "grad_norm": 2.810636281967163, "kl": 0.055643110536038876, "learning_rate": 9.976752315657359e-07, "loss": 0.0022, "num_tokens": 1755408.0, "reward": 0.74658203125, "reward_std": 0.013593493029475212, "rewards//mean": 0.74658203125, "rewards//std": 0.039881665259599686, "step": 203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0408, "grad_norm": 2.388843059539795, "kl": 0.06320055993273854, "learning_rate": 9.976445660039117e-07, "loss": 0.0025, "num_tokens": 1764008.0, "reward": 0.7498779296875, "reward_std": 0.013801433145999908, "rewards//mean": 0.7498779296875, "rewards//std": 0.03442857041954994, "step": 204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.041, "grad_norm": 2.3890225887298584, "kl": 0.06138871354050934, "learning_rate": 9.976136999909155e-07, "loss": 0.0025, "num_tokens": 1772688.0, "reward": 0.73956298828125, "reward_std": 0.010558126494288445, "rewards//mean": 0.73956298828125, "rewards//std": 0.045245181769132614, "step": 205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0412, "grad_norm": 2.236191511154175, "kl": 0.06445245840586722, "learning_rate": 9.975826335391805e-07, "loss": 0.0026, "num_tokens": 1781256.0, "reward": 0.7501220703125, "reward_std": 0.012905829586088657, "rewards//mean": 0.7501220703125, "rewards//std": 0.03487589955329895, "step": 206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0414, "grad_norm": 2.0409834384918213, "kl": 0.061787127051502466, "learning_rate": 9.975513666612203e-07, "loss": 0.0025, "num_tokens": 1789976.0, "reward": 0.74481201171875, "reward_std": 0.011488648131489754, "rewards//mean": 0.74481201171875, "rewards//std": 0.05587073415517807, "step": 207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0416, "grad_norm": 2.1245434284210205, "kl": 0.05468311160802841, "learning_rate": 9.975198993696291e-07, "loss": 0.0022, "num_tokens": 1798664.0, "reward": 0.73492431640625, "reward_std": 0.013937927782535553, "rewards//mean": 0.73492431640625, "rewards//std": 0.0415378138422966, "step": 208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0418, "grad_norm": 2.1178219318389893, "kl": 0.05903259618207812, "learning_rate": 9.97488231677082e-07, "loss": 0.0024, "num_tokens": 1807424.0, "reward": 0.68988037109375, "reward_std": 0.01218400988727808, "rewards//mean": 0.68988037109375, "rewards//std": 0.04769261181354523, "step": 209 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.042, "grad_norm": 1.9095737934112549, "kl": 0.05956404539756477, "learning_rate": 9.974563635963347e-07, "loss": 0.0024, "num_tokens": 1816088.0, "reward": 0.73046875, "reward_std": 0.009744959883391857, "rewards//mean": 0.73046875, "rewards//std": 0.04048740491271019, "step": 210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0422, "grad_norm": 2.1075732707977295, "kl": 0.06340880901552737, "learning_rate": 9.974242951402235e-07, "loss": 0.0025, "num_tokens": 1824672.0, "reward": 0.6942138671875, "reward_std": 0.00884406641125679, "rewards//mean": 0.6942138671875, "rewards//std": 0.04520455375313759, "step": 211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0424, "grad_norm": 2.276707887649536, "kl": 0.05456857895478606, "learning_rate": 9.973920263216657e-07, "loss": 0.0022, "num_tokens": 1833248.0, "reward": 0.7916259765625, "reward_std": 0.011873690411448479, "rewards//mean": 0.7916259765625, "rewards//std": 0.027945812791585922, "step": 212 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0426, "grad_norm": 2.3808939456939697, "kl": 0.061990561662241817, "learning_rate": 9.97359557153659e-07, "loss": 0.0025, "num_tokens": 1841808.0, "reward": 0.73394775390625, "reward_std": 0.01026148721575737, "rewards//mean": 0.73394775390625, "rewards//std": 0.0428367480635643, "step": 213 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0428, "grad_norm": 2.4553043842315674, "kl": 0.061710300971753895, "learning_rate": 9.973268876492825e-07, "loss": 0.0025, "num_tokens": 1850392.0, "reward": 0.73028564453125, "reward_std": 0.015301035717129707, "rewards//mean": 0.73028564453125, "rewards//std": 0.04240093007683754, "step": 214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.043, "grad_norm": 2.1479804515838623, "kl": 0.06337157567031682, "learning_rate": 9.972940178216952e-07, "loss": 0.0025, "num_tokens": 1859016.0, "reward": 0.7620849609375, "reward_std": 0.011465619318187237, "rewards//mean": 0.7620849609375, "rewards//std": 0.04265884682536125, "step": 215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0432, "grad_norm": 2.276979923248291, "kl": 0.060141903813928366, "learning_rate": 9.972609476841365e-07, "loss": 0.0024, "num_tokens": 1867616.0, "reward": 0.7412109375, "reward_std": 0.008193857036530972, "rewards//mean": 0.7412109375, "rewards//std": 0.021133441478013992, "step": 216 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0434, "grad_norm": 2.192436933517456, "kl": 0.06401014095172286, "learning_rate": 9.97227677249928e-07, "loss": 0.0026, "num_tokens": 1876296.0, "reward": 0.745361328125, "reward_std": 0.010227463208138943, "rewards//mean": 0.745361328125, "rewards//std": 0.036384351551532745, "step": 217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0436, "grad_norm": 2.2693440914154053, "kl": 0.05632793949916959, "learning_rate": 9.971942065324702e-07, "loss": 0.0023, "num_tokens": 1884904.0, "reward": 0.7484130859375, "reward_std": 0.013199035078287125, "rewards//mean": 0.7484130859375, "rewards//std": 0.03698413819074631, "step": 218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0438, "grad_norm": 2.1643688678741455, "kl": 0.055894225370138884, "learning_rate": 9.971605355452457e-07, "loss": 0.0022, "num_tokens": 1893616.0, "reward": 0.72784423828125, "reward_std": 0.011329513974487782, "rewards//mean": 0.72784423828125, "rewards//std": 0.044684987515211105, "step": 219 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.044, "grad_norm": 2.3612186908721924, "kl": 0.07208431093022227, "learning_rate": 9.97126664301817e-07, "loss": 0.0029, "num_tokens": 1902160.0, "reward": 0.7008056640625, "reward_std": 0.014414534904062748, "rewards//mean": 0.7008056640625, "rewards//std": 0.04989214614033699, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0442, "grad_norm": 2.268941640853882, "kl": 0.06697740312665701, "learning_rate": 9.970925928158272e-07, "loss": 0.0027, "num_tokens": 1910880.0, "reward": 0.73345947265625, "reward_std": 0.012288164347410202, "rewards//mean": 0.73345947265625, "rewards//std": 0.03685308247804642, "step": 221 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0444, "grad_norm": 2.4280242919921875, "kl": 0.06854389910586178, "learning_rate": 9.970583211010007e-07, "loss": 0.0027, "num_tokens": 1919640.0, "reward": 0.70794677734375, "reward_std": 0.012792366556823254, "rewards//mean": 0.70794677734375, "rewards//std": 0.04355793073773384, "step": 222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0446, "grad_norm": 2.040374994277954, "kl": 0.05879002343863249, "learning_rate": 9.970238491711415e-07, "loss": 0.0024, "num_tokens": 1928296.0, "reward": 0.728271484375, "reward_std": 0.00894878152757883, "rewards//mean": 0.728271484375, "rewards//std": 0.03154986351728439, "step": 223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0448, "grad_norm": 2.0993146896362305, "kl": 0.06133840745314956, "learning_rate": 9.969891770401356e-07, "loss": 0.0025, "num_tokens": 1937088.0, "reward": 0.75421142578125, "reward_std": 0.010610558092594147, "rewards//mean": 0.75421142578125, "rewards//std": 0.03229665383696556, "step": 224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.045, "grad_norm": 2.6484577655792236, "kl": 0.0712521958630532, "learning_rate": 9.969543047219486e-07, "loss": 0.0029, "num_tokens": 1945688.0, "reward": 0.76324462890625, "reward_std": 0.016112633049488068, "rewards//mean": 0.76324462890625, "rewards//std": 0.04176894947886467, "step": 225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0452, "grad_norm": 2.331159830093384, "kl": 0.07147391699254513, "learning_rate": 9.96919232230627e-07, "loss": 0.0029, "num_tokens": 1954320.0, "reward": 0.7493896484375, "reward_std": 0.009902137331664562, "rewards//mean": 0.7493896484375, "rewards//std": 0.04099993780255318, "step": 226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0454, "grad_norm": 2.0388107299804688, "kl": 0.07578787580132484, "learning_rate": 9.968839595802981e-07, "loss": 0.003, "num_tokens": 1962944.0, "reward": 0.71954345703125, "reward_std": 0.010332188569009304, "rewards//mean": 0.71954345703125, "rewards//std": 0.0278586708009243, "step": 227 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0456, "grad_norm": 2.3314082622528076, "kl": 0.06498332298360765, "learning_rate": 9.968484867851697e-07, "loss": 0.0026, "num_tokens": 1971624.0, "reward": 0.76806640625, "reward_std": 0.013963586650788784, "rewards//mean": 0.76806640625, "rewards//std": 0.04706322029232979, "step": 228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0458, "grad_norm": 2.018660545349121, "kl": 0.07186876214109361, "learning_rate": 9.968128138595302e-07, "loss": 0.0029, "num_tokens": 1980280.0, "reward": 0.7147216796875, "reward_std": 0.010772351175546646, "rewards//mean": 0.7147216796875, "rewards//std": 0.03680853918194771, "step": 229 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.046, "grad_norm": 2.121382236480713, "kl": 0.06785159790888429, "learning_rate": 9.967769408177488e-07, "loss": 0.0027, "num_tokens": 1988880.0, "reward": 0.6982421875, "reward_std": 0.012861795723438263, "rewards//mean": 0.6982421875, "rewards//std": 0.0487145259976387, "step": 230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0462, "grad_norm": 2.435134172439575, "kl": 0.08035403094254434, "learning_rate": 9.967408676742751e-07, "loss": 0.0032, "num_tokens": 1997536.0, "reward": 0.7432861328125, "reward_std": 0.012507785111665726, "rewards//mean": 0.7432861328125, "rewards//std": 0.036315012723207474, "step": 231 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0464, "grad_norm": 2.347782611846924, "kl": 0.07359315385110676, "learning_rate": 9.967045944436393e-07, "loss": 0.0029, "num_tokens": 2006280.0, "reward": 0.7220458984375, "reward_std": 0.010751600377261639, "rewards//mean": 0.7220458984375, "rewards//std": 0.04129425063729286, "step": 232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0466, "grad_norm": 2.1992995738983154, "kl": 0.0674269802402705, "learning_rate": 9.96668121140452e-07, "loss": 0.0027, "num_tokens": 2015040.0, "reward": 0.7586669921875, "reward_std": 0.014760918915271759, "rewards//mean": 0.7586669921875, "rewards//std": 0.03706100583076477, "step": 233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0468, "grad_norm": 1.9348255395889282, "kl": 0.07445071451365948, "learning_rate": 9.966314477794052e-07, "loss": 0.003, "num_tokens": 2023640.0, "reward": 0.73541259765625, "reward_std": 0.009623780846595764, "rewards//mean": 0.73541259765625, "rewards//std": 0.03704645112156868, "step": 234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.047, "grad_norm": 2.164318323135376, "kl": 0.07863690797239542, "learning_rate": 9.965945743752705e-07, "loss": 0.0031, "num_tokens": 2032216.0, "reward": 0.7347412109375, "reward_std": 0.010528791695833206, "rewards//mean": 0.7347412109375, "rewards//std": 0.041150301694869995, "step": 235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0472, "grad_norm": 2.256127119064331, "kl": 0.0846207330469042, "learning_rate": 9.965575009429005e-07, "loss": 0.0034, "num_tokens": 2040856.0, "reward": 0.7567138671875, "reward_std": 0.012185719795525074, "rewards//mean": 0.7567138671875, "rewards//std": 0.038885194808244705, "step": 236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0474, "grad_norm": 2.367684841156006, "kl": 0.08532239287160337, "learning_rate": 9.965202274972286e-07, "loss": 0.0034, "num_tokens": 2049408.0, "reward": 0.71923828125, "reward_std": 0.009885359555482864, "rewards//mean": 0.71923828125, "rewards//std": 0.034121397882699966, "step": 237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0476, "grad_norm": 2.3444292545318604, "kl": 0.07901590317487717, "learning_rate": 9.964827540532684e-07, "loss": 0.0032, "num_tokens": 2058016.0, "reward": 0.7315673828125, "reward_std": 0.013179188594222069, "rewards//mean": 0.7315673828125, "rewards//std": 0.04617464542388916, "step": 238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0478, "grad_norm": 2.395791530609131, "kl": 0.08098797150887549, "learning_rate": 9.964450806261144e-07, "loss": 0.0032, "num_tokens": 2066648.0, "reward": 0.75616455078125, "reward_std": 0.012934434227645397, "rewards//mean": 0.75616455078125, "rewards//std": 0.029723087325692177, "step": 239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.048, "grad_norm": 2.1232640743255615, "kl": 0.0727109950967133, "learning_rate": 9.96407207230941e-07, "loss": 0.0029, "num_tokens": 2075336.0, "reward": 0.7333984375, "reward_std": 0.010355428792536259, "rewards//mean": 0.7333984375, "rewards//std": 0.03930357098579407, "step": 240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0482, "grad_norm": 2.341886043548584, "kl": 0.07593965763226151, "learning_rate": 9.963691338830042e-07, "loss": 0.003, "num_tokens": 2083952.0, "reward": 0.7322998046875, "reward_std": 0.011291599832475185, "rewards//mean": 0.7322998046875, "rewards//std": 0.02264885976910591, "step": 241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0484, "grad_norm": 2.3043925762176514, "kl": 0.08260205248370767, "learning_rate": 9.963308605976396e-07, "loss": 0.0033, "num_tokens": 2092624.0, "reward": 0.76416015625, "reward_std": 0.00958188809454441, "rewards//mean": 0.76416015625, "rewards//std": 0.02448371797800064, "step": 242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0486, "grad_norm": 2.2695934772491455, "kl": 0.07344825379550457, "learning_rate": 9.962923873902636e-07, "loss": 0.0029, "num_tokens": 2101160.0, "reward": 0.70928955078125, "reward_std": 0.010659701190888882, "rewards//mean": 0.70928955078125, "rewards//std": 0.044932279735803604, "step": 243 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0488, "grad_norm": 2.400766611099243, "kl": 0.08081559091806412, "learning_rate": 9.962537142763732e-07, "loss": 0.0032, "num_tokens": 2109792.0, "reward": 0.74383544921875, "reward_std": 0.010848671197891235, "rewards//mean": 0.74383544921875, "rewards//std": 0.026944944635033607, "step": 244 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.049, "grad_norm": 2.460304021835327, "kl": 0.08585498575121164, "learning_rate": 9.962148412715463e-07, "loss": 0.0034, "num_tokens": 2118552.0, "reward": 0.7547607421875, "reward_std": 0.012553266249597073, "rewards//mean": 0.7547607421875, "rewards//std": 0.03701522946357727, "step": 245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0492, "grad_norm": 2.2080492973327637, "kl": 0.0854458324611187, "learning_rate": 9.961757683914405e-07, "loss": 0.0034, "num_tokens": 2127248.0, "reward": 0.69549560546875, "reward_std": 0.01023833453655243, "rewards//mean": 0.69549560546875, "rewards//std": 0.04499489814043045, "step": 246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0494, "grad_norm": 2.5779333114624023, "kl": 0.07532225945033133, "learning_rate": 9.961364956517946e-07, "loss": 0.003, "num_tokens": 2135896.0, "reward": 0.75152587890625, "reward_std": 0.012623130343854427, "rewards//mean": 0.75152587890625, "rewards//std": 0.047596026211977005, "step": 247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0496, "grad_norm": 2.496809720993042, "kl": 0.08910441934131086, "learning_rate": 9.960970230684275e-07, "loss": 0.0036, "num_tokens": 2144536.0, "reward": 0.72540283203125, "reward_std": 0.013428730890154839, "rewards//mean": 0.72540283203125, "rewards//std": 0.04788045957684517, "step": 248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0498, "grad_norm": 2.400987148284912, "kl": 0.07812315365299582, "learning_rate": 9.960573506572389e-07, "loss": 0.0031, "num_tokens": 2153104.0, "reward": 0.75372314453125, "reward_std": 0.011331569403409958, "rewards//mean": 0.75372314453125, "rewards//std": 0.02425442449748516, "step": 249 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.05, "grad_norm": 2.398341178894043, "kl": 0.08263894519768655, "learning_rate": 9.960174784342087e-07, "loss": 0.0033, "num_tokens": 2161736.0, "reward": 0.7413330078125, "reward_std": 0.008479075506329536, "rewards//mean": 0.7413330078125, "rewards//std": 0.03599345684051514, "step": 250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0502, "grad_norm": 2.1341476440429688, "kl": 0.08822295162826777, "learning_rate": 9.959774064153975e-07, "loss": 0.0035, "num_tokens": 2170344.0, "reward": 0.7177734375, "reward_std": 0.008946191519498825, "rewards//mean": 0.7177734375, "rewards//std": 0.039610493928194046, "step": 251 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0504, "grad_norm": 2.428009271621704, "kl": 0.08810280309990048, "learning_rate": 9.959371346169465e-07, "loss": 0.0035, "num_tokens": 2179056.0, "reward": 0.7904052734375, "reward_std": 0.011364737525582314, "rewards//mean": 0.7904052734375, "rewards//std": 0.02987276203930378, "step": 252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0506, "grad_norm": 2.2430484294891357, "kl": 0.08622027351520956, "learning_rate": 9.95896663055077e-07, "loss": 0.0034, "num_tokens": 2187640.0, "reward": 0.71026611328125, "reward_std": 0.009076209738850594, "rewards//mean": 0.71026611328125, "rewards//std": 0.04143089801073074, "step": 253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0508, "grad_norm": 2.6776883602142334, "kl": 0.08731716056354344, "learning_rate": 9.958559917460907e-07, "loss": 0.0035, "num_tokens": 2196336.0, "reward": 0.75714111328125, "reward_std": 0.010459870100021362, "rewards//mean": 0.75714111328125, "rewards//std": 0.028164541348814964, "step": 254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.051, "grad_norm": 2.2391140460968018, "kl": 0.09201033553108573, "learning_rate": 9.958151207063703e-07, "loss": 0.0037, "num_tokens": 2205024.0, "reward": 0.7508544921875, "reward_std": 0.010757776908576488, "rewards//mean": 0.7508544921875, "rewards//std": 0.036147892475128174, "step": 255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0512, "grad_norm": 2.440457820892334, "kl": 0.08760909363627434, "learning_rate": 9.957740499523785e-07, "loss": 0.0035, "num_tokens": 2213608.0, "reward": 0.75689697265625, "reward_std": 0.013642119243741035, "rewards//mean": 0.75689697265625, "rewards//std": 0.04814309999346733, "step": 256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0514, "grad_norm": 2.3290514945983887, "kl": 0.09418594231829047, "learning_rate": 9.957327795006588e-07, "loss": 0.0038, "num_tokens": 2222264.0, "reward": 0.77484130859375, "reward_std": 0.011928251013159752, "rewards//mean": 0.77484130859375, "rewards//std": 0.04047763720154762, "step": 257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0516, "grad_norm": 2.230024576187134, "kl": 0.08470788644626737, "learning_rate": 9.956913093678348e-07, "loss": 0.0034, "num_tokens": 2230880.0, "reward": 0.72161865234375, "reward_std": 0.009003030136227608, "rewards//mean": 0.72161865234375, "rewards//std": 0.03698756545782089, "step": 258 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0518, "grad_norm": 2.324817419052124, "kl": 0.11008088244125247, "learning_rate": 9.956496395706105e-07, "loss": 0.0044, "num_tokens": 2239608.0, "reward": 0.7716064453125, "reward_std": 0.009707880206406116, "rewards//mean": 0.7716064453125, "rewards//std": 0.03827941045165062, "step": 259 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.052, "grad_norm": 2.1312897205352783, "kl": 0.09451142069883645, "learning_rate": 9.956077701257707e-07, "loss": 0.0038, "num_tokens": 2248296.0, "reward": 0.7510986328125, "reward_std": 0.008577214553952217, "rewards//mean": 0.7510986328125, "rewards//std": 0.026932932436466217, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0522, "grad_norm": 2.2232437133789062, "kl": 0.11112384242005646, "learning_rate": 9.955657010501806e-07, "loss": 0.0044, "num_tokens": 2256976.0, "reward": 0.72271728515625, "reward_std": 0.00968963373452425, "rewards//mean": 0.72271728515625, "rewards//std": 0.039515554904937744, "step": 261 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0524, "grad_norm": 2.0861966609954834, "kl": 0.10497553274035454, "learning_rate": 9.955234323607851e-07, "loss": 0.0042, "num_tokens": 2265672.0, "reward": 0.76654052734375, "reward_std": 0.009169444441795349, "rewards//mean": 0.76654052734375, "rewards//std": 0.035329725593328476, "step": 262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0526, "grad_norm": 2.0662031173706055, "kl": 0.09033403964713216, "learning_rate": 9.954809640746105e-07, "loss": 0.0036, "num_tokens": 2274336.0, "reward": 0.7734375, "reward_std": 0.009483709000051022, "rewards//mean": 0.7734375, "rewards//std": 0.03672310337424278, "step": 263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0528, "grad_norm": 2.315596103668213, "kl": 0.09967878996394575, "learning_rate": 9.954382962087627e-07, "loss": 0.004, "num_tokens": 2282984.0, "reward": 0.78326416015625, "reward_std": 0.011601641774177551, "rewards//mean": 0.78326416015625, "rewards//std": 0.032166559249162674, "step": 264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.053, "grad_norm": 2.2670955657958984, "kl": 0.10461910394951701, "learning_rate": 9.953954287804284e-07, "loss": 0.0042, "num_tokens": 2291520.0, "reward": 0.72039794921875, "reward_std": 0.010187996551394463, "rewards//mean": 0.72039794921875, "rewards//std": 0.03597552329301834, "step": 265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0532, "grad_norm": 2.1288392543792725, "kl": 0.11212815390899777, "learning_rate": 9.953523618068748e-07, "loss": 0.0045, "num_tokens": 2300080.0, "reward": 0.73370361328125, "reward_std": 0.00983397290110588, "rewards//mean": 0.73370361328125, "rewards//std": 0.02644364908337593, "step": 266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0534, "grad_norm": 2.3853437900543213, "kl": 0.12639968702569604, "learning_rate": 9.95309095305449e-07, "loss": 0.0051, "num_tokens": 2308656.0, "reward": 0.74774169921875, "reward_std": 0.011285347864031792, "rewards//mean": 0.74774169921875, "rewards//std": 0.03584738075733185, "step": 267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0536, "grad_norm": 2.105947494506836, "kl": 0.10734104830771685, "learning_rate": 9.952656292935788e-07, "loss": 0.0043, "num_tokens": 2317368.0, "reward": 0.76641845703125, "reward_std": 0.009294925257563591, "rewards//mean": 0.76641845703125, "rewards//std": 0.03633975610136986, "step": 268 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0538, "grad_norm": 2.4343624114990234, "kl": 0.11141107883304358, "learning_rate": 9.952219637887725e-07, "loss": 0.0045, "num_tokens": 2325992.0, "reward": 0.74627685546875, "reward_std": 0.013000589795410633, "rewards//mean": 0.74627685546875, "rewards//std": 0.0329802930355072, "step": 269 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.054, "grad_norm": 2.114595890045166, "kl": 0.11248049885034561, "learning_rate": 9.951780988086183e-07, "loss": 0.0045, "num_tokens": 2334616.0, "reward": 0.7659912109375, "reward_std": 0.008284421637654305, "rewards//mean": 0.7659912109375, "rewards//std": 0.024673106148838997, "step": 270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0542, "grad_norm": 2.3597919940948486, "kl": 0.10822209948673844, "learning_rate": 9.95134034370785e-07, "loss": 0.0043, "num_tokens": 2343264.0, "reward": 0.75213623046875, "reward_std": 0.008110977709293365, "rewards//mean": 0.75213623046875, "rewards//std": 0.02558121271431446, "step": 271 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0544, "grad_norm": 2.342583417892456, "kl": 0.10092776664532721, "learning_rate": 9.95089770493022e-07, "loss": 0.004, "num_tokens": 2351936.0, "reward": 0.73480224609375, "reward_std": 0.009873945266008377, "rewards//mean": 0.73480224609375, "rewards//std": 0.04320269823074341, "step": 272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0546, "grad_norm": 2.247030019760132, "kl": 0.11563214473426342, "learning_rate": 9.950453071931588e-07, "loss": 0.0046, "num_tokens": 2360560.0, "reward": 0.7540283203125, "reward_std": 0.009906873106956482, "rewards//mean": 0.7540283203125, "rewards//std": 0.03898783028125763, "step": 273 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0548, "grad_norm": 2.341111660003662, "kl": 0.11460674507543445, "learning_rate": 9.950006444891048e-07, "loss": 0.0046, "num_tokens": 2369160.0, "reward": 0.71240234375, "reward_std": 0.010771198198199272, "rewards//mean": 0.71240234375, "rewards//std": 0.033519841730594635, "step": 274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.055, "grad_norm": 2.476717948913574, "kl": 0.10514741018414497, "learning_rate": 9.949557823988506e-07, "loss": 0.0042, "num_tokens": 2377840.0, "reward": 0.7381591796875, "reward_std": 0.01162101048976183, "rewards//mean": 0.7381591796875, "rewards//std": 0.03822242096066475, "step": 275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0552, "grad_norm": 2.486793041229248, "kl": 0.11561599606648088, "learning_rate": 9.949107209404663e-07, "loss": 0.0046, "num_tokens": 2386504.0, "reward": 0.7628173828125, "reward_std": 0.008916687220335007, "rewards//mean": 0.7628173828125, "rewards//std": 0.021748732775449753, "step": 276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0554, "grad_norm": 2.1925792694091797, "kl": 0.11504006339237094, "learning_rate": 9.94865460132103e-07, "loss": 0.0046, "num_tokens": 2395128.0, "reward": 0.75469970703125, "reward_std": 0.013766275718808174, "rewards//mean": 0.75469970703125, "rewards//std": 0.03127463534474373, "step": 277 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0556, "grad_norm": 2.2684152126312256, "kl": 0.1263164347037673, "learning_rate": 9.948199999919912e-07, "loss": 0.0051, "num_tokens": 2403824.0, "reward": 0.71728515625, "reward_std": 0.008052065037190914, "rewards//mean": 0.71728515625, "rewards//std": 0.031269371509552, "step": 278 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0558, "grad_norm": 2.307429552078247, "kl": 0.1207714956253767, "learning_rate": 9.947743405384428e-07, "loss": 0.0048, "num_tokens": 2412440.0, "reward": 0.74017333984375, "reward_std": 0.010619981214404106, "rewards//mean": 0.74017333984375, "rewards//std": 0.037441518157720566, "step": 279 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.056, "grad_norm": 2.2332217693328857, "kl": 0.11981469811871648, "learning_rate": 9.947284817898492e-07, "loss": 0.0048, "num_tokens": 2421072.0, "reward": 0.7069091796875, "reward_std": 0.01122340653091669, "rewards//mean": 0.7069091796875, "rewards//std": 0.03928488865494728, "step": 280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0562, "grad_norm": 2.250086545944214, "kl": 0.12764271907508373, "learning_rate": 9.946824237646824e-07, "loss": 0.0051, "num_tokens": 2429736.0, "reward": 0.744873046875, "reward_std": 0.010046460665762424, "rewards//mean": 0.744873046875, "rewards//std": 0.03565817326307297, "step": 281 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0564, "grad_norm": 2.4999492168426514, "kl": 0.12754053669050336, "learning_rate": 9.946361664814943e-07, "loss": 0.0051, "num_tokens": 2438336.0, "reward": 0.73980712890625, "reward_std": 0.01213806588202715, "rewards//mean": 0.73980712890625, "rewards//std": 0.021334391087293625, "step": 282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0566, "grad_norm": 2.4783177375793457, "kl": 0.1202986822463572, "learning_rate": 9.945897099589173e-07, "loss": 0.0048, "num_tokens": 2446944.0, "reward": 0.72564697265625, "reward_std": 0.011273622512817383, "rewards//mean": 0.72564697265625, "rewards//std": 0.027434322983026505, "step": 283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0568, "grad_norm": 2.176853895187378, "kl": 0.12868809327483177, "learning_rate": 9.945430542156646e-07, "loss": 0.0051, "num_tokens": 2455528.0, "reward": 0.76263427734375, "reward_std": 0.01164940744638443, "rewards//mean": 0.76263427734375, "rewards//std": 0.02524586021900177, "step": 284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.057, "grad_norm": 2.0553863048553467, "kl": 0.11887755850329995, "learning_rate": 9.944961992705286e-07, "loss": 0.0048, "num_tokens": 2464104.0, "reward": 0.7296142578125, "reward_std": 0.007252207491546869, "rewards//mean": 0.7296142578125, "rewards//std": 0.03891632333397865, "step": 285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0572, "grad_norm": 2.3183748722076416, "kl": 0.11394174303859472, "learning_rate": 9.944491451423827e-07, "loss": 0.0046, "num_tokens": 2472768.0, "reward": 0.77862548828125, "reward_std": 0.010248874314129353, "rewards//mean": 0.77862548828125, "rewards//std": 0.021373379975557327, "step": 286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0574, "grad_norm": 2.401534080505371, "kl": 0.12894047936424613, "learning_rate": 9.944018918501805e-07, "loss": 0.0052, "num_tokens": 2481432.0, "reward": 0.73358154296875, "reward_std": 0.009259985759854317, "rewards//mean": 0.73358154296875, "rewards//std": 0.034647442400455475, "step": 287 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0576, "grad_norm": 2.3681552410125732, "kl": 0.13058429351076484, "learning_rate": 9.94354439412955e-07, "loss": 0.0052, "num_tokens": 2490120.0, "reward": 0.73504638671875, "reward_std": 0.010098876431584358, "rewards//mean": 0.73504638671875, "rewards//std": 0.023883353918790817, "step": 288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0578, "grad_norm": 2.335580348968506, "kl": 0.13276920886710286, "learning_rate": 9.943067878498209e-07, "loss": 0.0053, "num_tokens": 2498832.0, "reward": 0.7470703125, "reward_std": 0.009666764177381992, "rewards//mean": 0.7470703125, "rewards//std": 0.034583210945129395, "step": 289 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.058, "grad_norm": 3.374286651611328, "kl": 0.1374687599018216, "learning_rate": 9.942589371799714e-07, "loss": 0.0055, "num_tokens": 2507544.0, "reward": 0.75628662109375, "reward_std": 0.007982924580574036, "rewards//mean": 0.75628662109375, "rewards//std": 0.03333229571580887, "step": 290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0582, "grad_norm": 2.4196667671203613, "kl": 0.13316209614276886, "learning_rate": 9.94210887422681e-07, "loss": 0.0053, "num_tokens": 2516200.0, "reward": 0.75543212890625, "reward_std": 0.010620934888720512, "rewards//mean": 0.75543212890625, "rewards//std": 0.04492587968707085, "step": 291 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0584, "grad_norm": 2.6077113151550293, "kl": 0.13197654951363802, "learning_rate": 9.941626385973047e-07, "loss": 0.0053, "num_tokens": 2524768.0, "reward": 0.75946044921875, "reward_std": 0.00954778678715229, "rewards//mean": 0.75946044921875, "rewards//std": 0.028338147327303886, "step": 292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0586, "grad_norm": 2.331041097640991, "kl": 0.11738697905093431, "learning_rate": 9.941141907232763e-07, "loss": 0.0047, "num_tokens": 2533440.0, "reward": 0.753662109375, "reward_std": 0.009502648375928402, "rewards//mean": 0.753662109375, "rewards//std": 0.03564458340406418, "step": 293 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0588, "grad_norm": 2.3681371212005615, "kl": 0.1467471462674439, "learning_rate": 9.94065543820111e-07, "loss": 0.0059, "num_tokens": 2542128.0, "reward": 0.79052734375, "reward_std": 0.010154277086257935, "rewards//mean": 0.79052734375, "rewards//std": 0.02691522240638733, "step": 294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.059, "grad_norm": 2.214519739151001, "kl": 0.14778105821460485, "learning_rate": 9.94016697907404e-07, "loss": 0.0059, "num_tokens": 2550680.0, "reward": 0.73980712890625, "reward_std": 0.012334965169429779, "rewards//mean": 0.73980712890625, "rewards//std": 0.031967345625162125, "step": 295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0592, "grad_norm": 2.359705686569214, "kl": 0.135012770537287, "learning_rate": 9.9396765300483e-07, "loss": 0.0054, "num_tokens": 2559424.0, "reward": 0.7669677734375, "reward_std": 0.008275476284325123, "rewards//mean": 0.7669677734375, "rewards//std": 0.03668495640158653, "step": 296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0594, "grad_norm": 2.2671761512756348, "kl": 0.12718097073957324, "learning_rate": 9.939184091321444e-07, "loss": 0.0051, "num_tokens": 2568040.0, "reward": 0.78363037109375, "reward_std": 0.007923097349703312, "rewards//mean": 0.78363037109375, "rewards//std": 0.026158751919865608, "step": 297 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0596, "grad_norm": 2.3568453788757324, "kl": 0.14283269178122282, "learning_rate": 9.938689663091827e-07, "loss": 0.0057, "num_tokens": 2576776.0, "reward": 0.7615966796875, "reward_std": 0.00952947698533535, "rewards//mean": 0.7615966796875, "rewards//std": 0.034827250987291336, "step": 298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0598, "grad_norm": 2.3118374347686768, "kl": 0.13765425560995936, "learning_rate": 9.938193245558604e-07, "loss": 0.0055, "num_tokens": 2585392.0, "reward": 0.72393798828125, "reward_std": 0.011698382906615734, "rewards//mean": 0.72393798828125, "rewards//std": 0.0432247668504715, "step": 299 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.06, "grad_norm": 2.3777027130126953, "kl": 0.13236781675368547, "learning_rate": 9.937694838921733e-07, "loss": 0.0053, "num_tokens": 2594032.0, "reward": 0.7261962890625, "reward_std": 0.007660794071853161, "rewards//mean": 0.7261962890625, "rewards//std": 0.02233387529850006, "step": 300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0602, "grad_norm": 2.3749730587005615, "kl": 0.14261657977476716, "learning_rate": 9.93719444338197e-07, "loss": 0.0057, "num_tokens": 2602736.0, "reward": 0.725830078125, "reward_std": 0.012275228276848793, "rewards//mean": 0.725830078125, "rewards//std": 0.044357601553201675, "step": 301 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0604, "grad_norm": 2.3030948638916016, "kl": 0.14758447650820017, "learning_rate": 9.936692059140878e-07, "loss": 0.0059, "num_tokens": 2611384.0, "reward": 0.7725830078125, "reward_std": 0.008051641285419464, "rewards//mean": 0.7725830078125, "rewards//std": 0.02942551113665104, "step": 302 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0606, "grad_norm": 2.4487171173095703, "kl": 0.13396706525236368, "learning_rate": 9.936187686400814e-07, "loss": 0.0054, "num_tokens": 2620152.0, "reward": 0.75701904296875, "reward_std": 0.009920504875481129, "rewards//mean": 0.75701904296875, "rewards//std": 0.03601841628551483, "step": 303 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0608, "grad_norm": 2.4276504516601562, "kl": 0.14364641439169645, "learning_rate": 9.93568132536494e-07, "loss": 0.0057, "num_tokens": 2628816.0, "reward": 0.75531005859375, "reward_std": 0.01256647054105997, "rewards//mean": 0.75531005859375, "rewards//std": 0.030956953763961792, "step": 304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.061, "grad_norm": 2.934332847595215, "kl": 0.1464973008260131, "learning_rate": 9.935172976237217e-07, "loss": 0.0059, "num_tokens": 2637496.0, "reward": 0.75006103515625, "reward_std": 0.010656114667654037, "rewards//mean": 0.75006103515625, "rewards//std": 0.016754386946558952, "step": 305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0612, "grad_norm": 2.670234203338623, "kl": 0.13851862819865346, "learning_rate": 9.93466263922241e-07, "loss": 0.0055, "num_tokens": 2646104.0, "reward": 0.7593994140625, "reward_std": 0.010100295767188072, "rewards//mean": 0.7593994140625, "rewards//std": 0.028921151533722878, "step": 306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0614, "grad_norm": 2.3960487842559814, "kl": 0.14857173897325993, "learning_rate": 9.934150314526083e-07, "loss": 0.0059, "num_tokens": 2654744.0, "reward": 0.769775390625, "reward_std": 0.00879153236746788, "rewards//mean": 0.769775390625, "rewards//std": 0.015975650399923325, "step": 307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0616, "grad_norm": 2.3883743286132812, "kl": 0.14142659353092313, "learning_rate": 9.933636002354599e-07, "loss": 0.0057, "num_tokens": 2663376.0, "reward": 0.71966552734375, "reward_std": 0.009030375629663467, "rewards//mean": 0.71966552734375, "rewards//std": 0.02294394187629223, "step": 308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0618, "grad_norm": 2.4159791469573975, "kl": 0.13685584999620914, "learning_rate": 9.933119702915124e-07, "loss": 0.0055, "num_tokens": 2671952.0, "reward": 0.7314453125, "reward_std": 0.009831303730607033, "rewards//mean": 0.7314453125, "rewards//std": 0.025836244225502014, "step": 309 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.062, "grad_norm": 2.4496707916259766, "kl": 0.14309891825541854, "learning_rate": 9.93260141641562e-07, "loss": 0.0057, "num_tokens": 2680624.0, "reward": 0.74346923828125, "reward_std": 0.010039031505584717, "rewards//mean": 0.74346923828125, "rewards//std": 0.029430333524942398, "step": 310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0622, "grad_norm": 2.4669125080108643, "kl": 0.1311500621959567, "learning_rate": 9.932081143064858e-07, "loss": 0.0052, "num_tokens": 2689176.0, "reward": 0.7725830078125, "reward_std": 0.010175235569477081, "rewards//mean": 0.7725830078125, "rewards//std": 0.03233535587787628, "step": 311 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0624, "grad_norm": 2.6372668743133545, "kl": 0.14883148716762662, "learning_rate": 9.931558883072402e-07, "loss": 0.006, "num_tokens": 2697864.0, "reward": 0.7506103515625, "reward_std": 0.009078023955225945, "rewards//mean": 0.7506103515625, "rewards//std": 0.02745617926120758, "step": 312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0626, "grad_norm": 2.422214984893799, "kl": 0.13642366975545883, "learning_rate": 9.931034636648616e-07, "loss": 0.0055, "num_tokens": 2706480.0, "reward": 0.71978759765625, "reward_std": 0.008253131061792374, "rewards//mean": 0.71978759765625, "rewards//std": 0.02018669806420803, "step": 313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0628, "grad_norm": 2.507030487060547, "kl": 0.1547061800956726, "learning_rate": 9.930508404004666e-07, "loss": 0.0062, "num_tokens": 2715056.0, "reward": 0.7611083984375, "reward_std": 0.008002392947673798, "rewards//mean": 0.7611083984375, "rewards//std": 0.03237839788198471, "step": 314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.063, "grad_norm": 2.4126579761505127, "kl": 0.14070541644468904, "learning_rate": 9.929980185352525e-07, "loss": 0.0056, "num_tokens": 2723632.0, "reward": 0.75, "reward_std": 0.007926436141133308, "rewards//mean": 0.75, "rewards//std": 0.03435836732387543, "step": 315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0632, "grad_norm": 2.4670326709747314, "kl": 0.13856445252895355, "learning_rate": 9.929449980904951e-07, "loss": 0.0055, "num_tokens": 2732264.0, "reward": 0.728759765625, "reward_std": 0.00800327304750681, "rewards//mean": 0.728759765625, "rewards//std": 0.032090287655591965, "step": 316 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0634, "grad_norm": 2.4115591049194336, "kl": 0.1344844363629818, "learning_rate": 9.928917790875516e-07, "loss": 0.0054, "num_tokens": 2740960.0, "reward": 0.7650146484375, "reward_std": 0.008280552923679352, "rewards//mean": 0.7650146484375, "rewards//std": 0.030203351750969887, "step": 317 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0636, "grad_norm": 2.355797290802002, "kl": 0.14118980802595615, "learning_rate": 9.928383615478586e-07, "loss": 0.0056, "num_tokens": 2749528.0, "reward": 0.75140380859375, "reward_std": 0.010042013600468636, "rewards//mean": 0.75140380859375, "rewards//std": 0.023482587188482285, "step": 318 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0638, "grad_norm": 2.4152274131774902, "kl": 0.14794302312657237, "learning_rate": 9.927847454929322e-07, "loss": 0.0059, "num_tokens": 2758176.0, "reward": 0.7567138671875, "reward_std": 0.007422657683491707, "rewards//mean": 0.7567138671875, "rewards//std": 0.024572281166911125, "step": 319 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.064, "grad_norm": 2.4957797527313232, "kl": 0.14901667507365346, "learning_rate": 9.927309309443695e-07, "loss": 0.006, "num_tokens": 2766760.0, "reward": 0.76763916015625, "reward_std": 0.008163049817085266, "rewards//mean": 0.76763916015625, "rewards//std": 0.03143540024757385, "step": 320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0642, "grad_norm": 2.529953956604004, "kl": 0.14068567799404263, "learning_rate": 9.926769179238464e-07, "loss": 0.0056, "num_tokens": 2775344.0, "reward": 0.75244140625, "reward_std": 0.008808191865682602, "rewards//mean": 0.75244140625, "rewards//std": 0.03251486271619797, "step": 321 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0644, "grad_norm": 2.4482409954071045, "kl": 0.13553015561774373, "learning_rate": 9.926227064531199e-07, "loss": 0.0054, "num_tokens": 2783880.0, "reward": 0.75732421875, "reward_std": 0.009315178729593754, "rewards//mean": 0.75732421875, "rewards//std": 0.023255953565239906, "step": 322 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0646, "grad_norm": 2.410548210144043, "kl": 0.14313685148954391, "learning_rate": 9.925682965540263e-07, "loss": 0.0057, "num_tokens": 2792480.0, "reward": 0.7581787109375, "reward_std": 0.009847695007920265, "rewards//mean": 0.7581787109375, "rewards//std": 0.030062692239880562, "step": 323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0648, "grad_norm": 2.4038262367248535, "kl": 0.1418413147330284, "learning_rate": 9.925136882484815e-07, "loss": 0.0057, "num_tokens": 2801088.0, "reward": 0.7283935546875, "reward_std": 0.0071106343530118465, "rewards//mean": 0.7283935546875, "rewards//std": 0.029023557901382446, "step": 324 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.065, "grad_norm": 2.455026865005493, "kl": 0.14870468340814114, "learning_rate": 9.92458881558482e-07, "loss": 0.0059, "num_tokens": 2809768.0, "reward": 0.752197265625, "reward_std": 0.01029081642627716, "rewards//mean": 0.752197265625, "rewards//std": 0.027532432228326797, "step": 325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0652, "grad_norm": 2.627734661102295, "kl": 0.14815522264689207, "learning_rate": 9.92403876506104e-07, "loss": 0.0059, "num_tokens": 2818328.0, "reward": 0.76214599609375, "reward_std": 0.007829036563634872, "rewards//mean": 0.76214599609375, "rewards//std": 0.03401203081011772, "step": 326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0654, "grad_norm": 2.2990593910217285, "kl": 0.1547041591256857, "learning_rate": 9.923486731135033e-07, "loss": 0.0062, "num_tokens": 2826984.0, "reward": 0.73211669921875, "reward_std": 0.009885728359222412, "rewards//mean": 0.73211669921875, "rewards//std": 0.04717273265123367, "step": 327 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0656, "grad_norm": 2.4028618335723877, "kl": 0.14704480860382318, "learning_rate": 9.922932714029163e-07, "loss": 0.0059, "num_tokens": 2835544.0, "reward": 0.77032470703125, "reward_std": 0.008336121216416359, "rewards//mean": 0.77032470703125, "rewards//std": 0.022044189274311066, "step": 328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0658, "grad_norm": 2.481088638305664, "kl": 0.15337435388937593, "learning_rate": 9.92237671396658e-07, "loss": 0.0061, "num_tokens": 2844184.0, "reward": 0.78033447265625, "reward_std": 0.009491624310612679, "rewards//mean": 0.78033447265625, "rewards//std": 0.025861937552690506, "step": 329 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.066, "grad_norm": 2.3463478088378906, "kl": 0.15274662571027875, "learning_rate": 9.921818731171248e-07, "loss": 0.0061, "num_tokens": 2852824.0, "reward": 0.7298583984375, "reward_std": 0.007879456505179405, "rewards//mean": 0.7298583984375, "rewards//std": 0.01685134693980217, "step": 330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0662, "grad_norm": 2.4336092472076416, "kl": 0.16218566242605448, "learning_rate": 9.921258765867919e-07, "loss": 0.0065, "num_tokens": 2861568.0, "reward": 0.762451171875, "reward_std": 0.008910413831472397, "rewards//mean": 0.762451171875, "rewards//std": 0.027558811008930206, "step": 331 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0664, "grad_norm": 2.687967538833618, "kl": 0.18005005735903978, "learning_rate": 9.920696818282147e-07, "loss": 0.0072, "num_tokens": 2870168.0, "reward": 0.7740478515625, "reward_std": 0.006842412054538727, "rewards//mean": 0.7740478515625, "rewards//std": 0.025163955986499786, "step": 332 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0666, "grad_norm": 2.426886796951294, "kl": 0.1458109705708921, "learning_rate": 9.920132888640284e-07, "loss": 0.0058, "num_tokens": 2878752.0, "reward": 0.73150634765625, "reward_std": 0.008681036531925201, "rewards//mean": 0.73150634765625, "rewards//std": 0.03020353987812996, "step": 333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0668, "grad_norm": 2.489117383956909, "kl": 0.16056799283251166, "learning_rate": 9.919566977169485e-07, "loss": 0.0064, "num_tokens": 2887504.0, "reward": 0.73114013671875, "reward_std": 0.010405587032437325, "rewards//mean": 0.73114013671875, "rewards//std": 0.034376371651887894, "step": 334 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.067, "grad_norm": 2.3607702255249023, "kl": 0.14290315005928278, "learning_rate": 9.918999084097694e-07, "loss": 0.0057, "num_tokens": 2896176.0, "reward": 0.718017578125, "reward_std": 0.007494831457734108, "rewards//mean": 0.718017578125, "rewards//std": 0.022605039179325104, "step": 335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0672, "grad_norm": 2.3309366703033447, "kl": 0.13763791276142, "learning_rate": 9.91842920965366e-07, "loss": 0.0055, "num_tokens": 2904808.0, "reward": 0.7408447265625, "reward_std": 0.008373770862817764, "rewards//mean": 0.7408447265625, "rewards//std": 0.028170250356197357, "step": 336 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0674, "grad_norm": 2.3058981895446777, "kl": 0.15147208329290152, "learning_rate": 9.91785735406693e-07, "loss": 0.0061, "num_tokens": 2913544.0, "reward": 0.7340087890625, "reward_std": 0.007901324890553951, "rewards//mean": 0.7340087890625, "rewards//std": 0.033354826271533966, "step": 337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0676, "grad_norm": 2.5370399951934814, "kl": 0.16497230855748057, "learning_rate": 9.917283517567843e-07, "loss": 0.0066, "num_tokens": 2922208.0, "reward": 0.74591064453125, "reward_std": 0.007384471595287323, "rewards//mean": 0.74591064453125, "rewards//std": 0.025481605902314186, "step": 338 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0678, "grad_norm": 2.6054527759552, "kl": 0.17215240187942982, "learning_rate": 9.916707700387545e-07, "loss": 0.0069, "num_tokens": 2930904.0, "reward": 0.7353515625, "reward_std": 0.008231500163674355, "rewards//mean": 0.7353515625, "rewards//std": 0.03357398882508278, "step": 339 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.068, "grad_norm": 2.465613603591919, "kl": 0.1483508301898837, "learning_rate": 9.916129902757974e-07, "loss": 0.0059, "num_tokens": 2939600.0, "reward": 0.7542724609375, "reward_std": 0.00948614813387394, "rewards//mean": 0.7542724609375, "rewards//std": 0.022922571748495102, "step": 340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0682, "grad_norm": 2.374268054962158, "kl": 0.17610491486266255, "learning_rate": 9.915550124911866e-07, "loss": 0.007, "num_tokens": 2948432.0, "reward": 0.74737548828125, "reward_std": 0.0069858599454164505, "rewards//mean": 0.74737548828125, "rewards//std": 0.022756462916731834, "step": 341 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0684, "grad_norm": 2.5809059143066406, "kl": 0.15459762560203671, "learning_rate": 9.914968367082755e-07, "loss": 0.0062, "num_tokens": 2957032.0, "reward": 0.782470703125, "reward_std": 0.008777000941336155, "rewards//mean": 0.782470703125, "rewards//std": 0.016156556084752083, "step": 342 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0686, "grad_norm": 2.8978984355926514, "kl": 0.15157524775713682, "learning_rate": 9.914384629504973e-07, "loss": 0.0061, "num_tokens": 2965680.0, "reward": 0.75439453125, "reward_std": 0.009697170928120613, "rewards//mean": 0.75439453125, "rewards//std": 0.0314624160528183, "step": 343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0688, "grad_norm": 2.5209693908691406, "kl": 0.16175976116210222, "learning_rate": 9.913798912413652e-07, "loss": 0.0065, "num_tokens": 2974304.0, "reward": 0.70819091796875, "reward_std": 0.008540173061192036, "rewards//mean": 0.70819091796875, "rewards//std": 0.03589380159974098, "step": 344 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.069, "grad_norm": 2.5906119346618652, "kl": 0.15957222785800695, "learning_rate": 9.913211216044713e-07, "loss": 0.0064, "num_tokens": 2982920.0, "reward": 0.72332763671875, "reward_std": 0.008108312264084816, "rewards//mean": 0.72332763671875, "rewards//std": 0.038860805332660675, "step": 345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0692, "grad_norm": 2.638352870941162, "kl": 0.14657947700470686, "learning_rate": 9.912621540634886e-07, "loss": 0.0059, "num_tokens": 2991640.0, "reward": 0.76904296875, "reward_std": 0.006945888977497816, "rewards//mean": 0.76904296875, "rewards//std": 0.024503495544195175, "step": 346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0694, "grad_norm": 2.2758610248565674, "kl": 0.16795516945421696, "learning_rate": 9.91202988642169e-07, "loss": 0.0067, "num_tokens": 3000224.0, "reward": 0.7584228515625, "reward_std": 0.006969330366700888, "rewards//mean": 0.7584228515625, "rewards//std": 0.02684059739112854, "step": 347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0696, "grad_norm": 2.6884002685546875, "kl": 0.16269545536488295, "learning_rate": 9.911436253643443e-07, "loss": 0.0065, "num_tokens": 3008880.0, "reward": 0.74200439453125, "reward_std": 0.010088170878589153, "rewards//mean": 0.74200439453125, "rewards//std": 0.02847137860953808, "step": 348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0698, "grad_norm": 2.5450334548950195, "kl": 0.14913112577050924, "learning_rate": 9.91084064253926e-07, "loss": 0.006, "num_tokens": 3017536.0, "reward": 0.7344970703125, "reward_std": 0.0112111521884799, "rewards//mean": 0.7344970703125, "rewards//std": 0.029528219252824783, "step": 349 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.07, "grad_norm": 2.740511894226074, "kl": 0.16483196150511503, "learning_rate": 9.910243053349055e-07, "loss": 0.0066, "num_tokens": 3026160.0, "reward": 0.7740478515625, "reward_std": 0.010326296091079712, "rewards//mean": 0.7740478515625, "rewards//std": 0.02729693055152893, "step": 350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0702, "grad_norm": 2.612835168838501, "kl": 0.1619471926242113, "learning_rate": 9.909643486313533e-07, "loss": 0.0065, "num_tokens": 3034792.0, "reward": 0.7838134765625, "reward_std": 0.008278329856693745, "rewards//mean": 0.7838134765625, "rewards//std": 0.019749755039811134, "step": 351 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0704, "grad_norm": 4.1898603439331055, "kl": 0.16626066528260708, "learning_rate": 9.909041941674204e-07, "loss": 0.0067, "num_tokens": 3043432.0, "reward": 0.759033203125, "reward_std": 0.007817190140485764, "rewards//mean": 0.759033203125, "rewards//std": 0.026068393141031265, "step": 352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0706, "grad_norm": 2.5477583408355713, "kl": 0.164810037240386, "learning_rate": 9.908438419673366e-07, "loss": 0.0066, "num_tokens": 3052008.0, "reward": 0.7667236328125, "reward_std": 0.011191527359187603, "rewards//mean": 0.7667236328125, "rewards//std": 0.022867033258080482, "step": 353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0708, "grad_norm": 2.383877992630005, "kl": 0.1532201822847128, "learning_rate": 9.90783292055412e-07, "loss": 0.0061, "num_tokens": 3060680.0, "reward": 0.76751708984375, "reward_std": 0.008705323562026024, "rewards//mean": 0.76751708984375, "rewards//std": 0.02952020801603794, "step": 354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.071, "grad_norm": 2.4674792289733887, "kl": 0.16015919670462608, "learning_rate": 9.907225444560361e-07, "loss": 0.0064, "num_tokens": 3069312.0, "reward": 0.77276611328125, "reward_std": 0.008959580212831497, "rewards//mean": 0.77276611328125, "rewards//std": 0.026305902749300003, "step": 355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0712, "grad_norm": 2.8221089839935303, "kl": 0.16012147534638643, "learning_rate": 9.90661599193678e-07, "loss": 0.0064, "num_tokens": 3078032.0, "reward": 0.74908447265625, "reward_std": 0.009289544075727463, "rewards//mean": 0.74908447265625, "rewards//std": 0.019817376509308815, "step": 356 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0714, "grad_norm": 2.472399950027466, "kl": 0.14893962908536196, "learning_rate": 9.906004562928863e-07, "loss": 0.006, "num_tokens": 3086656.0, "reward": 0.7430419921875, "reward_std": 0.008085759356617928, "rewards//mean": 0.7430419921875, "rewards//std": 0.017846059054136276, "step": 357 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0716, "grad_norm": 2.5224549770355225, "kl": 0.16191406780853868, "learning_rate": 9.905391157782897e-07, "loss": 0.0065, "num_tokens": 3095184.0, "reward": 0.74932861328125, "reward_std": 0.011135554872453213, "rewards//mean": 0.74932861328125, "rewards//std": 0.03566278889775276, "step": 358 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0718, "grad_norm": 2.772547483444214, "kl": 0.17021930869668722, "learning_rate": 9.904775776745956e-07, "loss": 0.0068, "num_tokens": 3103768.0, "reward": 0.79095458984375, "reward_std": 0.008317381143569946, "rewards//mean": 0.79095458984375, "rewards//std": 0.02636510692536831, "step": 359 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.072, "grad_norm": 2.4869027137756348, "kl": 0.16306456178426743, "learning_rate": 9.904158420065922e-07, "loss": 0.0065, "num_tokens": 3112464.0, "reward": 0.76519775390625, "reward_std": 0.006278190761804581, "rewards//mean": 0.76519775390625, "rewards//std": 0.03004978597164154, "step": 360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0722, "grad_norm": 2.7342638969421387, "kl": 0.16009332658722997, "learning_rate": 9.903539087991461e-07, "loss": 0.0064, "num_tokens": 3121000.0, "reward": 0.72418212890625, "reward_std": 0.008755840361118317, "rewards//mean": 0.72418212890625, "rewards//std": 0.03662150725722313, "step": 361 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0724, "grad_norm": 2.810922622680664, "kl": 0.1547168125398457, "learning_rate": 9.902917780772042e-07, "loss": 0.0062, "num_tokens": 3129608.0, "reward": 0.77642822265625, "reward_std": 0.01392364501953125, "rewards//mean": 0.77642822265625, "rewards//std": 0.023206347599625587, "step": 362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0726, "grad_norm": 2.8092007637023926, "kl": 0.16291858348995447, "learning_rate": 9.902294498657929e-07, "loss": 0.0065, "num_tokens": 3138360.0, "reward": 0.7508544921875, "reward_std": 0.009465081617236137, "rewards//mean": 0.7508544921875, "rewards//std": 0.04253233224153519, "step": 363 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0728, "grad_norm": 2.507169723510742, "kl": 0.16634537931531668, "learning_rate": 9.901669241900176e-07, "loss": 0.0067, "num_tokens": 3146912.0, "reward": 0.73480224609375, "reward_std": 0.007271946407854557, "rewards//mean": 0.73480224609375, "rewards//std": 0.02672211267054081, "step": 364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.073, "grad_norm": 2.7649011611938477, "kl": 0.15464997477829456, "learning_rate": 9.90104201075064e-07, "loss": 0.0062, "num_tokens": 3155568.0, "reward": 0.71844482421875, "reward_std": 0.008541805669665337, "rewards//mean": 0.71844482421875, "rewards//std": 0.03847087174654007, "step": 365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0732, "grad_norm": 2.8290657997131348, "kl": 0.19138141721487045, "learning_rate": 9.900412805461966e-07, "loss": 0.0077, "num_tokens": 3164336.0, "reward": 0.76190185546875, "reward_std": 0.007410317659378052, "rewards//mean": 0.76190185546875, "rewards//std": 0.036752309650182724, "step": 366 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0734, "grad_norm": 2.7856574058532715, "kl": 0.15830178558826447, "learning_rate": 9.899781626287602e-07, "loss": 0.0063, "num_tokens": 3173144.0, "reward": 0.760009765625, "reward_std": 0.008688227273523808, "rewards//mean": 0.760009765625, "rewards//std": 0.02757638320326805, "step": 367 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0736, "grad_norm": 2.695427894592285, "kl": 0.17311879340559244, "learning_rate": 9.899148473481784e-07, "loss": 0.0069, "num_tokens": 3181768.0, "reward": 0.73980712890625, "reward_std": 0.006761234253644943, "rewards//mean": 0.73980712890625, "rewards//std": 0.02978668175637722, "step": 368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0738, "grad_norm": 2.656928062438965, "kl": 0.1619602907449007, "learning_rate": 9.898513347299547e-07, "loss": 0.0065, "num_tokens": 3190400.0, "reward": 0.714599609375, "reward_std": 0.007395018357783556, "rewards//mean": 0.714599609375, "rewards//std": 0.04246055334806442, "step": 369 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.074, "grad_norm": 2.7612054347991943, "kl": 0.16792897693812847, "learning_rate": 9.89787624799672e-07, "loss": 0.0067, "num_tokens": 3199032.0, "reward": 0.77362060546875, "reward_std": 0.009967046789824963, "rewards//mean": 0.77362060546875, "rewards//std": 0.029193807393312454, "step": 370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0742, "grad_norm": 2.7488813400268555, "kl": 0.17005004733800888, "learning_rate": 9.897237175829926e-07, "loss": 0.0068, "num_tokens": 3207600.0, "reward": 0.75494384765625, "reward_std": 0.009035948663949966, "rewards//mean": 0.75494384765625, "rewards//std": 0.031119847670197487, "step": 371 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0744, "grad_norm": 2.553189754486084, "kl": 0.16994115710258484, "learning_rate": 9.896596131056582e-07, "loss": 0.0068, "num_tokens": 3216240.0, "reward": 0.74267578125, "reward_std": 0.008026436902582645, "rewards//mean": 0.74267578125, "rewards//std": 0.03289256617426872, "step": 372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0746, "grad_norm": 2.361967086791992, "kl": 0.1540005928836763, "learning_rate": 9.895953113934903e-07, "loss": 0.0062, "num_tokens": 3224856.0, "reward": 0.8001708984375, "reward_std": 0.007031290326267481, "rewards//mean": 0.8001708984375, "rewards//std": 0.022404255345463753, "step": 373 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0748, "grad_norm": 2.7153866291046143, "kl": 0.15861506760120392, "learning_rate": 9.895308124723896e-07, "loss": 0.0063, "num_tokens": 3233608.0, "reward": 0.7701416015625, "reward_std": 0.01336634624749422, "rewards//mean": 0.7701416015625, "rewards//std": 0.03284627944231033, "step": 374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.075, "grad_norm": 2.4990200996398926, "kl": 0.1461920291185379, "learning_rate": 9.89466116368336e-07, "loss": 0.0058, "num_tokens": 3242224.0, "reward": 0.72381591796875, "reward_std": 0.009897821582853794, "rewards//mean": 0.72381591796875, "rewards//std": 0.043557584285736084, "step": 375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0752, "grad_norm": 2.413890838623047, "kl": 0.15592028107494116, "learning_rate": 9.894012231073895e-07, "loss": 0.0062, "num_tokens": 3250880.0, "reward": 0.75830078125, "reward_std": 0.00871911458671093, "rewards//mean": 0.75830078125, "rewards//std": 0.03690403327345848, "step": 376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0754, "grad_norm": 2.9090678691864014, "kl": 0.14931943733245134, "learning_rate": 9.893361327156884e-07, "loss": 0.006, "num_tokens": 3259592.0, "reward": 0.7562255859375, "reward_std": 0.00859127752482891, "rewards//mean": 0.7562255859375, "rewards//std": 0.03542540967464447, "step": 377 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0756, "grad_norm": 2.6068999767303467, "kl": 0.16098628798499703, "learning_rate": 9.89270845219452e-07, "loss": 0.0064, "num_tokens": 3268216.0, "reward": 0.7706298828125, "reward_std": 0.013982309028506279, "rewards//mean": 0.7706298828125, "rewards//std": 0.03481682017445564, "step": 378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0758, "grad_norm": 2.4730560779571533, "kl": 0.15955154318362474, "learning_rate": 9.892053606449774e-07, "loss": 0.0064, "num_tokens": 3276832.0, "reward": 0.77764892578125, "reward_std": 0.006985072512179613, "rewards//mean": 0.77764892578125, "rewards//std": 0.02482563443481922, "step": 379 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.076, "grad_norm": 2.840237855911255, "kl": 0.16714267572388053, "learning_rate": 9.891396790186422e-07, "loss": 0.0067, "num_tokens": 3285440.0, "reward": 0.73150634765625, "reward_std": 0.006967043504118919, "rewards//mean": 0.73150634765625, "rewards//std": 0.03609272837638855, "step": 380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0762, "grad_norm": 2.4098997116088867, "kl": 0.16293874941766262, "learning_rate": 9.890738003669027e-07, "loss": 0.0065, "num_tokens": 3294056.0, "reward": 0.77288818359375, "reward_std": 0.00870993360877037, "rewards//mean": 0.77288818359375, "rewards//std": 0.04149697721004486, "step": 381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0764, "grad_norm": 2.6688671112060547, "kl": 0.16645270492881536, "learning_rate": 9.89007724716295e-07, "loss": 0.0067, "num_tokens": 3302832.0, "reward": 0.75201416015625, "reward_std": 0.006812462583184242, "rewards//mean": 0.75201416015625, "rewards//std": 0.03431599214673042, "step": 382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0766, "grad_norm": 2.900796890258789, "kl": 0.1770854638889432, "learning_rate": 9.889414520934343e-07, "loss": 0.0071, "num_tokens": 3311488.0, "reward": 0.757080078125, "reward_std": 0.009493831545114517, "rewards//mean": 0.757080078125, "rewards//std": 0.026105530560016632, "step": 383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0768, "grad_norm": 2.7237555980682373, "kl": 0.16892468417063355, "learning_rate": 9.88874982525015e-07, "loss": 0.0068, "num_tokens": 3320128.0, "reward": 0.72174072265625, "reward_std": 0.00853054877370596, "rewards//mean": 0.72174072265625, "rewards//std": 0.039785850793123245, "step": 384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.077, "grad_norm": 2.570565938949585, "kl": 0.17061032820492983, "learning_rate": 9.888083160378112e-07, "loss": 0.0068, "num_tokens": 3328744.0, "reward": 0.7491455078125, "reward_std": 0.008467968553304672, "rewards//mean": 0.7491455078125, "rewards//std": 0.02584063820540905, "step": 385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0772, "grad_norm": 2.565676689147949, "kl": 0.15762492455542088, "learning_rate": 9.887414526586763e-07, "loss": 0.0063, "num_tokens": 3337352.0, "reward": 0.746826171875, "reward_std": 0.00936988927423954, "rewards//mean": 0.746826171875, "rewards//std": 0.030208613723516464, "step": 386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0774, "grad_norm": 2.5404958724975586, "kl": 0.1645594099536538, "learning_rate": 9.886743924145426e-07, "loss": 0.0066, "num_tokens": 3345952.0, "reward": 0.78057861328125, "reward_std": 0.007525625638663769, "rewards//mean": 0.78057861328125, "rewards//std": 0.029721051454544067, "step": 387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0776, "grad_norm": 2.538297414779663, "kl": 0.16444916930049658, "learning_rate": 9.886071353324222e-07, "loss": 0.0066, "num_tokens": 3354552.0, "reward": 0.73828125, "reward_std": 0.009832067415118217, "rewards//mean": 0.73828125, "rewards//std": 0.028039097785949707, "step": 388 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0778, "grad_norm": 2.6103405952453613, "kl": 0.17876138631254435, "learning_rate": 9.88539681439406e-07, "loss": 0.0072, "num_tokens": 3363192.0, "reward": 0.74212646484375, "reward_std": 0.009619161486625671, "rewards//mean": 0.74212646484375, "rewards//std": 0.03128479793667793, "step": 389 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.078, "grad_norm": 2.5139098167419434, "kl": 0.1818069638684392, "learning_rate": 9.884720307626646e-07, "loss": 0.0073, "num_tokens": 3371832.0, "reward": 0.75091552734375, "reward_std": 0.008223002776503563, "rewards//mean": 0.75091552734375, "rewards//std": 0.032842766493558884, "step": 390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0782, "grad_norm": 2.67838978767395, "kl": 0.15979084372520447, "learning_rate": 9.884041833294475e-07, "loss": 0.0064, "num_tokens": 3380448.0, "reward": 0.774658203125, "reward_std": 0.010571452789008617, "rewards//mean": 0.774658203125, "rewards//std": 0.024899380281567574, "step": 391 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0784, "grad_norm": 2.609266757965088, "kl": 0.17010739585384727, "learning_rate": 9.883361391670839e-07, "loss": 0.0068, "num_tokens": 3389040.0, "reward": 0.75555419921875, "reward_std": 0.010208028368651867, "rewards//mean": 0.75555419921875, "rewards//std": 0.02784671261906624, "step": 392 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0786, "grad_norm": 2.705087184906006, "kl": 0.1755635621957481, "learning_rate": 9.882678983029817e-07, "loss": 0.007, "num_tokens": 3397736.0, "reward": 0.748291015625, "reward_std": 0.00819418951869011, "rewards//mean": 0.748291015625, "rewards//std": 0.03141137585043907, "step": 393 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0788, "grad_norm": 2.8524513244628906, "kl": 0.17735581519082189, "learning_rate": 9.881994607646286e-07, "loss": 0.0071, "num_tokens": 3406328.0, "reward": 0.7484130859375, "reward_std": 0.009125860407948494, "rewards//mean": 0.7484130859375, "rewards//std": 0.011681877076625824, "step": 394 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.079, "grad_norm": 2.815516948699951, "kl": 0.17463888693600893, "learning_rate": 9.881308265795911e-07, "loss": 0.007, "num_tokens": 3414960.0, "reward": 0.73016357421875, "reward_std": 0.00860263966023922, "rewards//mean": 0.73016357421875, "rewards//std": 0.034648314118385315, "step": 395 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0792, "grad_norm": 3.0293967723846436, "kl": 0.18000991456210613, "learning_rate": 9.88061995775515e-07, "loss": 0.0072, "num_tokens": 3423696.0, "reward": 0.720458984375, "reward_std": 0.007856153883039951, "rewards//mean": 0.720458984375, "rewards//std": 0.03832308202981949, "step": 396 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0794, "grad_norm": 2.5878942012786865, "kl": 0.18182779476046562, "learning_rate": 9.879929683801253e-07, "loss": 0.0073, "num_tokens": 3432248.0, "reward": 0.75244140625, "reward_std": 0.007938584312796593, "rewards//mean": 0.75244140625, "rewards//std": 0.02669837884604931, "step": 397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0796, "grad_norm": 2.4721519947052, "kl": 0.16829326748847961, "learning_rate": 9.879237444212264e-07, "loss": 0.0067, "num_tokens": 3440928.0, "reward": 0.76171875, "reward_std": 0.008632577955722809, "rewards//mean": 0.76171875, "rewards//std": 0.01977158710360527, "step": 398 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0798, "grad_norm": 2.558156967163086, "kl": 0.17831025645136833, "learning_rate": 9.878543239267014e-07, "loss": 0.0071, "num_tokens": 3449560.0, "reward": 0.75030517578125, "reward_std": 0.007332679349929094, "rewards//mean": 0.75030517578125, "rewards//std": 0.023222649469971657, "step": 399 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.08, "grad_norm": 2.3982725143432617, "kl": 0.1726833526045084, "learning_rate": 9.877847069245133e-07, "loss": 0.0069, "num_tokens": 3458264.0, "reward": 0.7664794921875, "reward_std": 0.0074759190902113914, "rewards//mean": 0.7664794921875, "rewards//std": 0.023963475599884987, "step": 400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0802, "grad_norm": 2.6396090984344482, "kl": 0.18833619728684425, "learning_rate": 9.877148934427035e-07, "loss": 0.0075, "num_tokens": 3466896.0, "reward": 0.7677001953125, "reward_std": 0.006921134889125824, "rewards//mean": 0.7677001953125, "rewards//std": 0.029165079817175865, "step": 401 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0804, "grad_norm": 2.3063488006591797, "kl": 0.18033870682120323, "learning_rate": 9.876448835093929e-07, "loss": 0.0072, "num_tokens": 3475584.0, "reward": 0.76837158203125, "reward_std": 0.00596450874581933, "rewards//mean": 0.76837158203125, "rewards//std": 0.033730026334524155, "step": 402 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0806, "grad_norm": 2.6989567279815674, "kl": 0.19588027335703373, "learning_rate": 9.875746771527815e-07, "loss": 0.0078, "num_tokens": 3484184.0, "reward": 0.77716064453125, "reward_std": 0.008181614801287651, "rewards//mean": 0.77716064453125, "rewards//std": 0.02728992886841297, "step": 403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0808, "grad_norm": 2.426013469696045, "kl": 0.18516316264867783, "learning_rate": 9.875042744011486e-07, "loss": 0.0074, "num_tokens": 3492760.0, "reward": 0.76885986328125, "reward_std": 0.007055317983031273, "rewards//mean": 0.76885986328125, "rewards//std": 0.017928466200828552, "step": 404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.081, "grad_norm": 2.4702301025390625, "kl": 0.1746185226365924, "learning_rate": 9.874336752828522e-07, "loss": 0.007, "num_tokens": 3501344.0, "reward": 0.76171875, "reward_std": 0.010278567671775818, "rewards//mean": 0.76171875, "rewards//std": 0.0270722433924675, "step": 405 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0812, "grad_norm": 2.7907369136810303, "kl": 0.18580629862844944, "learning_rate": 9.873628798263295e-07, "loss": 0.0074, "num_tokens": 3510008.0, "reward": 0.7650146484375, "reward_std": 0.006237374618649483, "rewards//mean": 0.7650146484375, "rewards//std": 0.034500602632761, "step": 406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0814, "grad_norm": 2.4231181144714355, "kl": 0.18027489073574543, "learning_rate": 9.872918880600973e-07, "loss": 0.0072, "num_tokens": 3518752.0, "reward": 0.764892578125, "reward_std": 0.005956288427114487, "rewards//mean": 0.764892578125, "rewards//std": 0.016305776312947273, "step": 407 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0816, "grad_norm": 2.4755072593688965, "kl": 0.19561395607888699, "learning_rate": 9.87220700012751e-07, "loss": 0.0078, "num_tokens": 3527400.0, "reward": 0.762939453125, "reward_std": 0.006552073638886213, "rewards//mean": 0.762939453125, "rewards//std": 0.036029815673828125, "step": 408 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0818, "grad_norm": 2.598135232925415, "kl": 0.18279425613582134, "learning_rate": 9.871493157129647e-07, "loss": 0.0073, "num_tokens": 3536064.0, "reward": 0.74993896484375, "reward_std": 0.008556658402085304, "rewards//mean": 0.74993896484375, "rewards//std": 0.02316194958984852, "step": 409 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.082, "grad_norm": 2.7481443881988525, "kl": 0.18953238148242235, "learning_rate": 9.870777351894926e-07, "loss": 0.0076, "num_tokens": 3544736.0, "reward": 0.71820068359375, "reward_std": 0.008066165260970592, "rewards//mean": 0.71820068359375, "rewards//std": 0.028888877481222153, "step": 410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0822, "grad_norm": 2.787365198135376, "kl": 0.1878298930823803, "learning_rate": 9.870059584711668e-07, "loss": 0.0075, "num_tokens": 3553424.0, "reward": 0.74188232421875, "reward_std": 0.007019890006631613, "rewards//mean": 0.74188232421875, "rewards//std": 0.02889254502952099, "step": 411 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0824, "grad_norm": 2.81783390045166, "kl": 0.19163534324616194, "learning_rate": 9.869339855868991e-07, "loss": 0.0077, "num_tokens": 3562040.0, "reward": 0.7288818359375, "reward_std": 0.009509474039077759, "rewards//mean": 0.7288818359375, "rewards//std": 0.03359359875321388, "step": 412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0826, "grad_norm": 2.587827205657959, "kl": 0.2044460643082857, "learning_rate": 9.868618165656804e-07, "loss": 0.0082, "num_tokens": 3570696.0, "reward": 0.74566650390625, "reward_std": 0.008736889809370041, "rewards//mean": 0.74566650390625, "rewards//std": 0.03351707383990288, "step": 413 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0828, "grad_norm": 2.6200897693634033, "kl": 0.18804363813251257, "learning_rate": 9.8678945143658e-07, "loss": 0.0075, "num_tokens": 3579256.0, "reward": 0.7288818359375, "reward_std": 0.009151293896138668, "rewards//mean": 0.7288818359375, "rewards//std": 0.03662879392504692, "step": 414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.083, "grad_norm": 2.506772994995117, "kl": 0.1931857457384467, "learning_rate": 9.86716890228747e-07, "loss": 0.0077, "num_tokens": 3587928.0, "reward": 0.72314453125, "reward_std": 0.006545564159750938, "rewards//mean": 0.72314453125, "rewards//std": 0.03633522614836693, "step": 415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0832, "grad_norm": 2.445132255554199, "kl": 0.20388566982001066, "learning_rate": 9.866441329714087e-07, "loss": 0.0082, "num_tokens": 3596568.0, "reward": 0.761474609375, "reward_std": 0.005879071541130543, "rewards//mean": 0.761474609375, "rewards//std": 0.020397162064909935, "step": 416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0834, "grad_norm": 2.6903209686279297, "kl": 0.20185576286166906, "learning_rate": 9.86571179693872e-07, "loss": 0.0081, "num_tokens": 3605248.0, "reward": 0.78009033203125, "reward_std": 0.010339450091123581, "rewards//mean": 0.78009033203125, "rewards//std": 0.026325460523366928, "step": 417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0836, "grad_norm": 2.6082499027252197, "kl": 0.1881139650940895, "learning_rate": 9.86498030425522e-07, "loss": 0.0075, "num_tokens": 3613840.0, "reward": 0.753173828125, "reward_std": 0.009717400185763836, "rewards//mean": 0.753173828125, "rewards//std": 0.033479172736406326, "step": 418 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0838, "grad_norm": 2.484720230102539, "kl": 0.19344091974198818, "learning_rate": 9.864246851958237e-07, "loss": 0.0077, "num_tokens": 3622464.0, "reward": 0.75836181640625, "reward_std": 0.010392667725682259, "rewards//mean": 0.75836181640625, "rewards//std": 0.04072520136833191, "step": 419 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.084, "grad_norm": 2.8422417640686035, "kl": 0.18848030921071768, "learning_rate": 9.863511440343205e-07, "loss": 0.0075, "num_tokens": 3631032.0, "reward": 0.74420166015625, "reward_std": 0.007764378562569618, "rewards//mean": 0.74420166015625, "rewards//std": 0.029569394886493683, "step": 420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0842, "grad_norm": 2.5308332443237305, "kl": 0.1926432317122817, "learning_rate": 9.862774069706345e-07, "loss": 0.0077, "num_tokens": 3639672.0, "reward": 0.74664306640625, "reward_std": 0.0067763011902570724, "rewards//mean": 0.74664306640625, "rewards//std": 0.030432218685746193, "step": 421 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0844, "grad_norm": 2.7227256298065186, "kl": 0.22925530839711428, "learning_rate": 9.862034740344671e-07, "loss": 0.0092, "num_tokens": 3648336.0, "reward": 0.771484375, "reward_std": 0.008542067371308804, "rewards//mean": 0.771484375, "rewards//std": 0.023302771151065826, "step": 422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0846, "grad_norm": 2.769712448120117, "kl": 0.19543266110122204, "learning_rate": 9.861293452555986e-07, "loss": 0.0078, "num_tokens": 3656896.0, "reward": 0.79376220703125, "reward_std": 0.007219559047371149, "rewards//mean": 0.79376220703125, "rewards//std": 0.03004222735762596, "step": 423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0848, "grad_norm": 2.576681613922119, "kl": 0.19337232876569033, "learning_rate": 9.86055020663888e-07, "loss": 0.0077, "num_tokens": 3665504.0, "reward": 0.733154296875, "reward_std": 0.006978962570428848, "rewards//mean": 0.733154296875, "rewards//std": 0.02180885523557663, "step": 424 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.085, "grad_norm": 2.465479850769043, "kl": 0.1821321528404951, "learning_rate": 9.859805002892731e-07, "loss": 0.0073, "num_tokens": 3674080.0, "reward": 0.73333740234375, "reward_std": 0.007003418169915676, "rewards//mean": 0.73333740234375, "rewards//std": 0.03618655353784561, "step": 425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0852, "grad_norm": 2.6620612144470215, "kl": 0.20288071408867836, "learning_rate": 9.859057841617708e-07, "loss": 0.0081, "num_tokens": 3682640.0, "reward": 0.7447509765625, "reward_std": 0.008478684350848198, "rewards//mean": 0.7447509765625, "rewards//std": 0.03240830451250076, "step": 426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0854, "grad_norm": 2.6503520011901855, "kl": 0.19767640065401793, "learning_rate": 9.858308723114768e-07, "loss": 0.0079, "num_tokens": 3691304.0, "reward": 0.78692626953125, "reward_std": 0.008378500118851662, "rewards//mean": 0.78692626953125, "rewards//std": 0.027062665671110153, "step": 427 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0856, "grad_norm": 2.59274959564209, "kl": 0.20678219757974148, "learning_rate": 9.857557647685655e-07, "loss": 0.0083, "num_tokens": 3700024.0, "reward": 0.75311279296875, "reward_std": 0.00883854366838932, "rewards//mean": 0.75311279296875, "rewards//std": 0.03340170904994011, "step": 428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0858, "grad_norm": 2.514921188354492, "kl": 0.1842403383925557, "learning_rate": 9.856804615632901e-07, "loss": 0.0074, "num_tokens": 3708784.0, "reward": 0.75360107421875, "reward_std": 0.009444896131753922, "rewards//mean": 0.75360107421875, "rewards//std": 0.029522772878408432, "step": 429 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.086, "grad_norm": 2.777625799179077, "kl": 0.19502401165664196, "learning_rate": 9.856049627259832e-07, "loss": 0.0078, "num_tokens": 3717392.0, "reward": 0.76678466796875, "reward_std": 0.007297373842447996, "rewards//mean": 0.76678466796875, "rewards//std": 0.02159462496638298, "step": 430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0862, "grad_norm": 2.677000045776367, "kl": 0.18325078207999468, "learning_rate": 9.85529268287055e-07, "loss": 0.0073, "num_tokens": 3726008.0, "reward": 0.74859619140625, "reward_std": 0.007576840464025736, "rewards//mean": 0.74859619140625, "rewards//std": 0.02463035099208355, "step": 431 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0864, "grad_norm": 2.5007803440093994, "kl": 0.19161915499716997, "learning_rate": 9.854533782769959e-07, "loss": 0.0077, "num_tokens": 3734584.0, "reward": 0.74884033203125, "reward_std": 0.007008885033428669, "rewards//mean": 0.74884033203125, "rewards//std": 0.02085285261273384, "step": 432 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0866, "grad_norm": 2.587076187133789, "kl": 0.2080300049856305, "learning_rate": 9.853772927263739e-07, "loss": 0.0083, "num_tokens": 3743296.0, "reward": 0.7801513671875, "reward_std": 0.007318174000829458, "rewards//mean": 0.7801513671875, "rewards//std": 0.03165404871106148, "step": 433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0868, "grad_norm": 2.896596908569336, "kl": 0.1951505597680807, "learning_rate": 9.853010116658366e-07, "loss": 0.0078, "num_tokens": 3752000.0, "reward": 0.7440185546875, "reward_std": 0.00882251188158989, "rewards//mean": 0.7440185546875, "rewards//std": 0.02630971372127533, "step": 434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.087, "grad_norm": 2.714966297149658, "kl": 0.19744882453233004, "learning_rate": 9.852245351261097e-07, "loss": 0.0079, "num_tokens": 3760584.0, "reward": 0.71649169921875, "reward_std": 0.009333435446023941, "rewards//mean": 0.71649169921875, "rewards//std": 0.04489992558956146, "step": 435 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0872, "grad_norm": 2.624218702316284, "kl": 0.19562861416488886, "learning_rate": 9.851478631379982e-07, "loss": 0.0078, "num_tokens": 3769168.0, "reward": 0.742919921875, "reward_std": 0.008637506514787674, "rewards//mean": 0.742919921875, "rewards//std": 0.03229343146085739, "step": 436 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0874, "grad_norm": 2.5103540420532227, "kl": 0.19844850152730942, "learning_rate": 9.850709957323854e-07, "loss": 0.0079, "num_tokens": 3777808.0, "reward": 0.74249267578125, "reward_std": 0.007015228737145662, "rewards//mean": 0.74249267578125, "rewards//std": 0.02400221675634384, "step": 437 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0876, "grad_norm": 2.376951217651367, "kl": 0.19819010514765978, "learning_rate": 9.849939329402336e-07, "loss": 0.0079, "num_tokens": 3786392.0, "reward": 0.7506103515625, "reward_std": 0.007007762789726257, "rewards//mean": 0.7506103515625, "rewards//std": 0.019626734778285027, "step": 438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0878, "grad_norm": 2.7214441299438477, "kl": 0.20057359337806702, "learning_rate": 9.849166747925834e-07, "loss": 0.008, "num_tokens": 3795064.0, "reward": 0.77362060546875, "reward_std": 0.008020205423235893, "rewards//mean": 0.77362060546875, "rewards//std": 0.02572754956781864, "step": 439 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.088, "grad_norm": 2.4217584133148193, "kl": 0.20243626087903976, "learning_rate": 9.848392213205547e-07, "loss": 0.0081, "num_tokens": 3803632.0, "reward": 0.7454833984375, "reward_std": 0.007380123715847731, "rewards//mean": 0.7454833984375, "rewards//std": 0.040875695645809174, "step": 440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0882, "grad_norm": 2.661808729171753, "kl": 0.18222533399239182, "learning_rate": 9.847615725553455e-07, "loss": 0.0073, "num_tokens": 3812296.0, "reward": 0.7913818359375, "reward_std": 0.008068419992923737, "rewards//mean": 0.7913818359375, "rewards//std": 0.022720929235219955, "step": 441 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0884, "grad_norm": 2.587676525115967, "kl": 0.191447539255023, "learning_rate": 9.84683728528233e-07, "loss": 0.0077, "num_tokens": 3820952.0, "reward": 0.7510986328125, "reward_std": 0.009070787578821182, "rewards//mean": 0.7510986328125, "rewards//std": 0.0328776054084301, "step": 442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0886, "grad_norm": 2.9863452911376953, "kl": 0.20910645369440317, "learning_rate": 9.846056892705727e-07, "loss": 0.0084, "num_tokens": 3829712.0, "reward": 0.71697998046875, "reward_std": 0.008707192726433277, "rewards//mean": 0.71697998046875, "rewards//std": 0.032932523638010025, "step": 443 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0888, "grad_norm": 2.7540276050567627, "kl": 0.2015506774187088, "learning_rate": 9.845274548137985e-07, "loss": 0.0081, "num_tokens": 3838296.0, "reward": 0.73382568359375, "reward_std": 0.007177338935434818, "rewards//mean": 0.73382568359375, "rewards//std": 0.012887472286820412, "step": 444 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.089, "grad_norm": 2.6551339626312256, "kl": 0.2096218653023243, "learning_rate": 9.844490251894236e-07, "loss": 0.0084, "num_tokens": 3846960.0, "reward": 0.7542724609375, "reward_std": 0.007436465937644243, "rewards//mean": 0.7542724609375, "rewards//std": 0.02651832066476345, "step": 445 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0892, "grad_norm": 2.52319073677063, "kl": 0.19700186513364315, "learning_rate": 9.843704004290392e-07, "loss": 0.0079, "num_tokens": 3855688.0, "reward": 0.7493896484375, "reward_std": 0.006621568463742733, "rewards//mean": 0.7493896484375, "rewards//std": 0.03030742146074772, "step": 446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0894, "grad_norm": 2.7591910362243652, "kl": 0.2017053822055459, "learning_rate": 9.842915805643156e-07, "loss": 0.0081, "num_tokens": 3864312.0, "reward": 0.7664794921875, "reward_std": 0.008353838697075844, "rewards//mean": 0.7664794921875, "rewards//std": 0.023245209828019142, "step": 447 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0896, "grad_norm": 2.4190142154693604, "kl": 0.1878850907087326, "learning_rate": 9.84212565627001e-07, "loss": 0.0075, "num_tokens": 3872992.0, "reward": 0.7568359375, "reward_std": 0.009020679630339146, "rewards//mean": 0.7568359375, "rewards//std": 0.02271326445043087, "step": 448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0898, "grad_norm": 2.5284149646759033, "kl": 0.21311705745756626, "learning_rate": 9.841333556489232e-07, "loss": 0.0085, "num_tokens": 3881712.0, "reward": 0.76666259765625, "reward_std": 0.006807057186961174, "rewards//mean": 0.76666259765625, "rewards//std": 0.027252739295363426, "step": 449 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.09, "grad_norm": 2.463273763656616, "kl": 0.1890072450041771, "learning_rate": 9.840539506619872e-07, "loss": 0.0076, "num_tokens": 3890312.0, "reward": 0.74871826171875, "reward_std": 0.0067598591558635235, "rewards//mean": 0.74871826171875, "rewards//std": 0.03121940791606903, "step": 450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0902, "grad_norm": 2.689650774002075, "kl": 0.20761975646018982, "learning_rate": 9.83974350698178e-07, "loss": 0.0083, "num_tokens": 3899016.0, "reward": 0.7344970703125, "reward_std": 0.008867532014846802, "rewards//mean": 0.7344970703125, "rewards//std": 0.0433250293135643, "step": 451 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0904, "grad_norm": 2.30403733253479, "kl": 0.2046855976805091, "learning_rate": 9.838945557895584e-07, "loss": 0.0082, "num_tokens": 3907752.0, "reward": 0.777587890625, "reward_std": 0.007500022184103727, "rewards//mean": 0.777587890625, "rewards//std": 0.018582813441753387, "step": 452 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0906, "grad_norm": 2.616736650466919, "kl": 0.21534956805408, "learning_rate": 9.838145659682692e-07, "loss": 0.0086, "num_tokens": 3916496.0, "reward": 0.7713623046875, "reward_std": 0.008273258805274963, "rewards//mean": 0.7713623046875, "rewards//std": 0.03000018745660782, "step": 453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0908, "grad_norm": 2.856215715408325, "kl": 0.1973020858131349, "learning_rate": 9.83734381266531e-07, "loss": 0.0079, "num_tokens": 3925008.0, "reward": 0.73614501953125, "reward_std": 0.011645911261439323, "rewards//mean": 0.73614501953125, "rewards//std": 0.03672264143824577, "step": 454 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.091, "grad_norm": 2.643582582473755, "kl": 0.19773272797465324, "learning_rate": 9.836540017166419e-07, "loss": 0.0079, "num_tokens": 3933696.0, "reward": 0.73785400390625, "reward_std": 0.0073912725783884525, "rewards//mean": 0.73785400390625, "rewards//std": 0.032764315605163574, "step": 455 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0912, "grad_norm": 2.69028902053833, "kl": 0.2114551505073905, "learning_rate": 9.835734273509785e-07, "loss": 0.0085, "num_tokens": 3942304.0, "reward": 0.779541015625, "reward_std": 0.006727224215865135, "rewards//mean": 0.779541015625, "rewards//std": 0.022128576412796974, "step": 456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0914, "grad_norm": 2.5662038326263428, "kl": 0.1863793758675456, "learning_rate": 9.834926582019966e-07, "loss": 0.0075, "num_tokens": 3950968.0, "reward": 0.7479248046875, "reward_std": 0.007806117180734873, "rewards//mean": 0.7479248046875, "rewards//std": 0.02594820410013199, "step": 457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0916, "grad_norm": 2.378325939178467, "kl": 0.18959766253829002, "learning_rate": 9.834116943022297e-07, "loss": 0.0076, "num_tokens": 3959640.0, "reward": 0.74444580078125, "reward_std": 0.007687430828809738, "rewards//mean": 0.74444580078125, "rewards//std": 0.032067108899354935, "step": 458 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0918, "grad_norm": 2.345531702041626, "kl": 0.19898757711052895, "learning_rate": 9.8333053568429e-07, "loss": 0.008, "num_tokens": 3968256.0, "reward": 0.77471923828125, "reward_std": 0.009729254059493542, "rewards//mean": 0.77471923828125, "rewards//std": 0.02229410409927368, "step": 459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.092, "grad_norm": 2.5729498863220215, "kl": 0.19167259149253368, "learning_rate": 9.832491823808686e-07, "loss": 0.0077, "num_tokens": 3977024.0, "reward": 0.77337646484375, "reward_std": 0.008461561053991318, "rewards//mean": 0.77337646484375, "rewards//std": 0.02044006995856762, "step": 460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0922, "grad_norm": 2.286104679107666, "kl": 0.20634953398257494, "learning_rate": 9.831676344247342e-07, "loss": 0.0083, "num_tokens": 3985664.0, "reward": 0.72589111328125, "reward_std": 0.00819789431989193, "rewards//mean": 0.72589111328125, "rewards//std": 0.03155507892370224, "step": 461 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0924, "grad_norm": 2.541599750518799, "kl": 0.1981488335877657, "learning_rate": 9.830858918487346e-07, "loss": 0.0079, "num_tokens": 3994480.0, "reward": 0.760009765625, "reward_std": 0.008511878550052643, "rewards//mean": 0.760009765625, "rewards//std": 0.02578815445303917, "step": 462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0926, "grad_norm": 2.3062171936035156, "kl": 0.18402158375829458, "learning_rate": 9.830039546857952e-07, "loss": 0.0074, "num_tokens": 4003112.0, "reward": 0.73883056640625, "reward_std": 0.00948390644043684, "rewards//mean": 0.73883056640625, "rewards//std": 0.0378524586558342, "step": 463 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0928, "grad_norm": 2.393819808959961, "kl": 0.2080188114196062, "learning_rate": 9.829218229689209e-07, "loss": 0.0083, "num_tokens": 4011720.0, "reward": 0.744384765625, "reward_std": 0.008310135453939438, "rewards//mean": 0.744384765625, "rewards//std": 0.03354422003030777, "step": 464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.093, "grad_norm": 2.644294500350952, "kl": 0.20960933715105057, "learning_rate": 9.828394967311938e-07, "loss": 0.0084, "num_tokens": 4020384.0, "reward": 0.7265625, "reward_std": 0.009460685774683952, "rewards//mean": 0.7265625, "rewards//std": 0.03722097724676132, "step": 465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0932, "grad_norm": 2.4498705863952637, "kl": 0.19817031733691692, "learning_rate": 9.827569760057754e-07, "loss": 0.0079, "num_tokens": 4029016.0, "reward": 0.76434326171875, "reward_std": 0.010624798946082592, "rewards//mean": 0.76434326171875, "rewards//std": 0.020740758627653122, "step": 466 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0934, "grad_norm": 2.5488860607147217, "kl": 0.18394710682332516, "learning_rate": 9.826742608259047e-07, "loss": 0.0074, "num_tokens": 4037688.0, "reward": 0.781982421875, "reward_std": 0.010184954851865768, "rewards//mean": 0.781982421875, "rewards//std": 0.03223337233066559, "step": 467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0936, "grad_norm": 2.507493495941162, "kl": 0.19846187718212605, "learning_rate": 9.825913512248995e-07, "loss": 0.0079, "num_tokens": 4046304.0, "reward": 0.74517822265625, "reward_std": 0.007261100225150585, "rewards//mean": 0.74517822265625, "rewards//std": 0.023946020752191544, "step": 468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0938, "grad_norm": 2.748115301132202, "kl": 0.19336541090160608, "learning_rate": 9.825082472361556e-07, "loss": 0.0077, "num_tokens": 4054992.0, "reward": 0.71734619140625, "reward_std": 0.007476060185581446, "rewards//mean": 0.71734619140625, "rewards//std": 0.03136115521192551, "step": 469 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.094, "grad_norm": 2.452084541320801, "kl": 0.20964460261166096, "learning_rate": 9.824249488931475e-07, "loss": 0.0084, "num_tokens": 4063624.0, "reward": 0.7545166015625, "reward_std": 0.008906117640435696, "rewards//mean": 0.7545166015625, "rewards//std": 0.027163511142134666, "step": 470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0942, "grad_norm": 2.641265630722046, "kl": 0.19954989198595285, "learning_rate": 9.82341456229428e-07, "loss": 0.008, "num_tokens": 4072256.0, "reward": 0.77545166015625, "reward_std": 0.007894563488662243, "rewards//mean": 0.77545166015625, "rewards//std": 0.02620847150683403, "step": 471 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0944, "grad_norm": 2.660736083984375, "kl": 0.19904836919158697, "learning_rate": 9.822577692786272e-07, "loss": 0.008, "num_tokens": 4080920.0, "reward": 0.75482177734375, "reward_std": 0.008569098077714443, "rewards//mean": 0.75482177734375, "rewards//std": 0.022247202694416046, "step": 472 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0946, "grad_norm": 2.474003553390503, "kl": 0.17544310074299574, "learning_rate": 9.821738880744547e-07, "loss": 0.007, "num_tokens": 4089496.0, "reward": 0.73590087890625, "reward_std": 0.01006229966878891, "rewards//mean": 0.73590087890625, "rewards//std": 0.030430229380726814, "step": 473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0948, "grad_norm": 2.616037130355835, "kl": 0.19754528906196356, "learning_rate": 9.820898126506979e-07, "loss": 0.0079, "num_tokens": 4098072.0, "reward": 0.74658203125, "reward_std": 0.00810934603214264, "rewards//mean": 0.74658203125, "rewards//std": 0.02306772582232952, "step": 474 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.095, "grad_norm": 2.8660085201263428, "kl": 0.19248671364039183, "learning_rate": 9.820055430412219e-07, "loss": 0.0077, "num_tokens": 4106672.0, "reward": 0.70306396484375, "reward_std": 0.011695638298988342, "rewards//mean": 0.70306396484375, "rewards//std": 0.040045253932476044, "step": 475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0952, "grad_norm": 2.6622281074523926, "kl": 0.19109869375824928, "learning_rate": 9.81921079279971e-07, "loss": 0.0076, "num_tokens": 4115384.0, "reward": 0.744873046875, "reward_std": 0.009246986359357834, "rewards//mean": 0.744873046875, "rewards//std": 0.02397768571972847, "step": 476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0954, "grad_norm": 2.4832065105438232, "kl": 0.17988140787929296, "learning_rate": 9.81836421400967e-07, "loss": 0.0072, "num_tokens": 4124064.0, "reward": 0.73858642578125, "reward_std": 0.007093518041074276, "rewards//mean": 0.73858642578125, "rewards//std": 0.03044365718960762, "step": 477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0956, "grad_norm": 2.5871036052703857, "kl": 0.18412148859351873, "learning_rate": 9.817515694383102e-07, "loss": 0.0074, "num_tokens": 4132704.0, "reward": 0.74658203125, "reward_std": 0.010728240013122559, "rewards//mean": 0.74658203125, "rewards//std": 0.04038258269429207, "step": 478 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0958, "grad_norm": 2.662102460861206, "kl": 0.19116327725350857, "learning_rate": 9.816665234261786e-07, "loss": 0.0076, "num_tokens": 4141304.0, "reward": 0.76910400390625, "reward_std": 0.008907128125429153, "rewards//mean": 0.76910400390625, "rewards//std": 0.030226586386561394, "step": 479 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.096, "grad_norm": 2.553114175796509, "kl": 0.19382564909756184, "learning_rate": 9.81581283398829e-07, "loss": 0.0078, "num_tokens": 4149840.0, "reward": 0.779541015625, "reward_std": 0.010796980001032352, "rewards//mean": 0.779541015625, "rewards//std": 0.03200715780258179, "step": 480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0962, "grad_norm": 2.7164504528045654, "kl": 0.20583349466323853, "learning_rate": 9.814958493905962e-07, "loss": 0.0082, "num_tokens": 4158448.0, "reward": 0.7186279296875, "reward_std": 0.007095596753060818, "rewards//mean": 0.7186279296875, "rewards//std": 0.037109989672899246, "step": 481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0964, "grad_norm": 2.745314836502075, "kl": 0.1839658385142684, "learning_rate": 9.814102214358926e-07, "loss": 0.0074, "num_tokens": 4167072.0, "reward": 0.738525390625, "reward_std": 0.009960859082639217, "rewards//mean": 0.738525390625, "rewards//std": 0.03948724642395973, "step": 482 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0966, "grad_norm": 2.495893716812134, "kl": 0.19855735450983047, "learning_rate": 9.813243995692097e-07, "loss": 0.0079, "num_tokens": 4175656.0, "reward": 0.7696533203125, "reward_std": 0.010801002383232117, "rewards//mean": 0.7696533203125, "rewards//std": 0.028872955590486526, "step": 483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0968, "grad_norm": 2.618044376373291, "kl": 0.20143710542470217, "learning_rate": 9.81238383825116e-07, "loss": 0.0081, "num_tokens": 4184232.0, "reward": 0.741943359375, "reward_std": 0.009049217216670513, "rewards//mean": 0.741943359375, "rewards//std": 0.025741152465343475, "step": 484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.097, "grad_norm": 2.513794183731079, "kl": 0.21006046421825886, "learning_rate": 9.81152174238259e-07, "loss": 0.0084, "num_tokens": 4192904.0, "reward": 0.72381591796875, "reward_std": 0.00830012746155262, "rewards//mean": 0.72381591796875, "rewards//std": 0.032827552407979965, "step": 485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0972, "grad_norm": 2.6051175594329834, "kl": 0.18188222404569387, "learning_rate": 9.810657708433635e-07, "loss": 0.0073, "num_tokens": 4201520.0, "reward": 0.73992919921875, "reward_std": 0.012143421918153763, "rewards//mean": 0.73992919921875, "rewards//std": 0.023102398961782455, "step": 486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0974, "grad_norm": 2.344669818878174, "kl": 0.20672739390283823, "learning_rate": 9.809791736752332e-07, "loss": 0.0083, "num_tokens": 4210128.0, "reward": 0.76385498046875, "reward_std": 0.007372348103672266, "rewards//mean": 0.76385498046875, "rewards//std": 0.025873055681586266, "step": 487 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0976, "grad_norm": 2.6771304607391357, "kl": 0.19675791915506124, "learning_rate": 9.808923827687492e-07, "loss": 0.0079, "num_tokens": 4218696.0, "reward": 0.732177734375, "reward_std": 0.009250756353139877, "rewards//mean": 0.732177734375, "rewards//std": 0.032651446759700775, "step": 488 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0978, "grad_norm": 2.732707977294922, "kl": 0.20094926748424768, "learning_rate": 9.80805398158871e-07, "loss": 0.008, "num_tokens": 4227304.0, "reward": 0.77362060546875, "reward_std": 0.010892328806221485, "rewards//mean": 0.77362060546875, "rewards//std": 0.029110724106431007, "step": 489 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.098, "grad_norm": 2.597862958908081, "kl": 0.2329936372116208, "learning_rate": 9.80718219880636e-07, "loss": 0.0093, "num_tokens": 4235968.0, "reward": 0.739990234375, "reward_std": 0.007593792397528887, "rewards//mean": 0.739990234375, "rewards//std": 0.03062269650399685, "step": 490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0982, "grad_norm": 2.4808294773101807, "kl": 0.20427713077515364, "learning_rate": 9.806308479691594e-07, "loss": 0.0082, "num_tokens": 4244616.0, "reward": 0.74407958984375, "reward_std": 0.008033126592636108, "rewards//mean": 0.74407958984375, "rewards//std": 0.028433604165911674, "step": 491 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0984, "grad_norm": 2.5725882053375244, "kl": 0.2031407980248332, "learning_rate": 9.805432824596347e-07, "loss": 0.0081, "num_tokens": 4253168.0, "reward": 0.7191162109375, "reward_std": 0.008553249761462212, "rewards//mean": 0.7191162109375, "rewards//std": 0.03302644565701485, "step": 492 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0986, "grad_norm": 2.347768783569336, "kl": 0.19026768300682306, "learning_rate": 9.804555233873332e-07, "loss": 0.0076, "num_tokens": 4261768.0, "reward": 0.75909423828125, "reward_std": 0.008124874904751778, "rewards//mean": 0.75909423828125, "rewards//std": 0.024676403030753136, "step": 493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0988, "grad_norm": 2.4123456478118896, "kl": 0.19286382384598255, "learning_rate": 9.803675707876048e-07, "loss": 0.0077, "num_tokens": 4270328.0, "reward": 0.72900390625, "reward_std": 0.008946258574724197, "rewards//mean": 0.72900390625, "rewards//std": 0.03177647665143013, "step": 494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.099, "grad_norm": 2.5037314891815186, "kl": 0.20618513878434896, "learning_rate": 9.80279424695876e-07, "loss": 0.0082, "num_tokens": 4279016.0, "reward": 0.74676513671875, "reward_std": 0.008192609064280987, "rewards//mean": 0.74676513671875, "rewards//std": 0.022867942228913307, "step": 495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0992, "grad_norm": 2.2900962829589844, "kl": 0.19684153143316507, "learning_rate": 9.801910851476524e-07, "loss": 0.0079, "num_tokens": 4287632.0, "reward": 0.7484130859375, "reward_std": 0.008924616500735283, "rewards//mean": 0.7484130859375, "rewards//std": 0.032028671354055405, "step": 496 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0994, "grad_norm": 2.4447147846221924, "kl": 0.2063806215301156, "learning_rate": 9.80102552178517e-07, "loss": 0.0083, "num_tokens": 4296256.0, "reward": 0.73052978515625, "reward_std": 0.006515363696962595, "rewards//mean": 0.73052978515625, "rewards//std": 0.02408469282090664, "step": 497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0996, "grad_norm": 2.3006718158721924, "kl": 0.20592305064201355, "learning_rate": 9.800138258241309e-07, "loss": 0.0082, "num_tokens": 4304944.0, "reward": 0.76043701171875, "reward_std": 0.007967970333993435, "rewards//mean": 0.76043701171875, "rewards//std": 0.038851458579301834, "step": 498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.0998, "grad_norm": 2.6404166221618652, "kl": 0.1996441464871168, "learning_rate": 9.799249061202334e-07, "loss": 0.008, "num_tokens": 4313560.0, "reward": 0.72857666015625, "reward_std": 0.008022921159863472, "rewards//mean": 0.72857666015625, "rewards//std": 0.029675686731934547, "step": 499 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1, "grad_norm": 2.557433605194092, "kl": 0.1909168316051364, "learning_rate": 9.798357931026412e-07, "loss": 0.0076, "num_tokens": 4322208.0, "reward": 0.76361083984375, "reward_std": 0.007821734994649887, "rewards//mean": 0.76361083984375, "rewards//std": 0.025229064747691154, "step": 500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1002, "grad_norm": 2.2387688159942627, "kl": 0.19993739295750856, "learning_rate": 9.797464868072486e-07, "loss": 0.008, "num_tokens": 4330888.0, "reward": 0.75006103515625, "reward_std": 0.007695702835917473, "rewards//mean": 0.75006103515625, "rewards//std": 0.028172602877020836, "step": 501 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1004, "grad_norm": 2.7008557319641113, "kl": 0.19917786587029696, "learning_rate": 9.796569872700287e-07, "loss": 0.008, "num_tokens": 4339480.0, "reward": 0.7283935546875, "reward_std": 0.011237498372793198, "rewards//mean": 0.7283935546875, "rewards//std": 0.039598070085048676, "step": 502 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1006, "grad_norm": 2.4576563835144043, "kl": 0.2003437550738454, "learning_rate": 9.795672945270316e-07, "loss": 0.008, "num_tokens": 4348208.0, "reward": 0.71051025390625, "reward_std": 0.007731921039521694, "rewards//mean": 0.71051025390625, "rewards//std": 0.03753923252224922, "step": 503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1008, "grad_norm": 2.464399814605713, "kl": 0.1987034259364009, "learning_rate": 9.794774086143857e-07, "loss": 0.0079, "num_tokens": 4356904.0, "reward": 0.75030517578125, "reward_std": 0.008051925338804722, "rewards//mean": 0.75030517578125, "rewards//std": 0.025308741256594658, "step": 504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.101, "grad_norm": 2.399820327758789, "kl": 0.19125232845544815, "learning_rate": 9.79387329568297e-07, "loss": 0.0077, "num_tokens": 4365528.0, "reward": 0.75128173828125, "reward_std": 0.010841385461390018, "rewards//mean": 0.75128173828125, "rewards//std": 0.0275955218821764, "step": 505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1012, "grad_norm": 2.731841564178467, "kl": 0.20694850012660027, "learning_rate": 9.792970574250493e-07, "loss": 0.0083, "num_tokens": 4374120.0, "reward": 0.73114013671875, "reward_std": 0.009580669924616814, "rewards//mean": 0.73114013671875, "rewards//std": 0.020978784188628197, "step": 506 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1014, "grad_norm": 2.4787447452545166, "kl": 0.18850557878613472, "learning_rate": 9.79206592221004e-07, "loss": 0.0075, "num_tokens": 4382808.0, "reward": 0.759033203125, "reward_std": 0.010051162913441658, "rewards//mean": 0.759033203125, "rewards//std": 0.022892504930496216, "step": 507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1016, "grad_norm": 2.734656572341919, "kl": 0.19833427853882313, "learning_rate": 9.791159339926008e-07, "loss": 0.0079, "num_tokens": 4391536.0, "reward": 0.76953125, "reward_std": 0.009782131761312485, "rewards//mean": 0.76953125, "rewards//std": 0.027134796604514122, "step": 508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1018, "grad_norm": 2.514674425125122, "kl": 0.2131390981376171, "learning_rate": 9.790250827763565e-07, "loss": 0.0085, "num_tokens": 4400152.0, "reward": 0.75579833984375, "reward_std": 0.009274055249989033, "rewards//mean": 0.75579833984375, "rewards//std": 0.037097085267305374, "step": 509 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.102, "grad_norm": 2.802415609359741, "kl": 0.1829777229577303, "learning_rate": 9.789340386088662e-07, "loss": 0.0073, "num_tokens": 4408712.0, "reward": 0.72381591796875, "reward_std": 0.00830510351806879, "rewards//mean": 0.72381591796875, "rewards//std": 0.03307013213634491, "step": 510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1022, "grad_norm": 2.1923043727874756, "kl": 0.19948930945247412, "learning_rate": 9.788428015268026e-07, "loss": 0.008, "num_tokens": 4417376.0, "reward": 0.7576904296875, "reward_std": 0.00736301438882947, "rewards//mean": 0.7576904296875, "rewards//std": 0.023488791659474373, "step": 511 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1024, "grad_norm": 2.5372860431671143, "kl": 0.20013653114438057, "learning_rate": 9.787513715669157e-07, "loss": 0.008, "num_tokens": 4426096.0, "reward": 0.75469970703125, "reward_std": 0.009306471794843674, "rewards//mean": 0.75469970703125, "rewards//std": 0.0311115775257349, "step": 512 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1026, "grad_norm": 2.5075161457061768, "kl": 0.21732639521360397, "learning_rate": 9.786597487660335e-07, "loss": 0.0087, "num_tokens": 4434696.0, "reward": 0.74029541015625, "reward_std": 0.011468115262687206, "rewards//mean": 0.74029541015625, "rewards//std": 0.036023881286382675, "step": 513 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1028, "grad_norm": 2.4096226692199707, "kl": 0.2048845263198018, "learning_rate": 9.78567933161062e-07, "loss": 0.0082, "num_tokens": 4443352.0, "reward": 0.760986328125, "reward_std": 0.007425335701555014, "rewards//mean": 0.760986328125, "rewards//std": 0.020633282139897346, "step": 514 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.103, "grad_norm": 2.4639785289764404, "kl": 0.19642549473792315, "learning_rate": 9.78475924788984e-07, "loss": 0.0079, "num_tokens": 4452008.0, "reward": 0.7655029296875, "reward_std": 0.00605900539085269, "rewards//mean": 0.7655029296875, "rewards//std": 0.026829317212104797, "step": 515 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1032, "grad_norm": 2.3441789150238037, "kl": 0.19151584152132273, "learning_rate": 9.783837236868609e-07, "loss": 0.0077, "num_tokens": 4460584.0, "reward": 0.75494384765625, "reward_std": 0.007256645243614912, "rewards//mean": 0.75494384765625, "rewards//std": 0.013319531455636024, "step": 516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1034, "grad_norm": 2.703347682952881, "kl": 0.1911890273913741, "learning_rate": 9.782913298918308e-07, "loss": 0.0076, "num_tokens": 4469208.0, "reward": 0.77349853515625, "reward_std": 0.010297955945134163, "rewards//mean": 0.77349853515625, "rewards//std": 0.023994015529751778, "step": 517 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1036, "grad_norm": 2.3911008834838867, "kl": 0.2087996369227767, "learning_rate": 9.781987434411106e-07, "loss": 0.0084, "num_tokens": 4477880.0, "reward": 0.7352294921875, "reward_std": 0.006590959616005421, "rewards//mean": 0.7352294921875, "rewards//std": 0.02513747289776802, "step": 518 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1038, "grad_norm": 2.5501441955566406, "kl": 0.19869405683130026, "learning_rate": 9.781059643719936e-07, "loss": 0.0079, "num_tokens": 4486520.0, "reward": 0.7608642578125, "reward_std": 0.00740392180159688, "rewards//mean": 0.7608642578125, "rewards//std": 0.030540280044078827, "step": 519 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.104, "grad_norm": 2.442887306213379, "kl": 0.2156991232186556, "learning_rate": 9.780129927218511e-07, "loss": 0.0086, "num_tokens": 4495064.0, "reward": 0.75372314453125, "reward_std": 0.008730370551347733, "rewards//mean": 0.75372314453125, "rewards//std": 0.027449216693639755, "step": 520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1042, "grad_norm": 2.282963275909424, "kl": 0.19761809334158897, "learning_rate": 9.779198285281326e-07, "loss": 0.0079, "num_tokens": 4503728.0, "reward": 0.73199462890625, "reward_std": 0.008112097159028053, "rewards//mean": 0.73199462890625, "rewards//std": 0.03141709789633751, "step": 521 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1044, "grad_norm": 2.3329858779907227, "kl": 0.2080096947029233, "learning_rate": 9.77826471828364e-07, "loss": 0.0083, "num_tokens": 4512344.0, "reward": 0.76318359375, "reward_std": 0.007059612777084112, "rewards//mean": 0.76318359375, "rewards//std": 0.025233939290046692, "step": 522 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1046, "grad_norm": 2.312187910079956, "kl": 0.18228135351091623, "learning_rate": 9.777329226601501e-07, "loss": 0.0073, "num_tokens": 4520896.0, "reward": 0.7510986328125, "reward_std": 0.008694940246641636, "rewards//mean": 0.7510986328125, "rewards//std": 0.020850403234362602, "step": 523 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1048, "grad_norm": 2.6797306537628174, "kl": 0.1950625041499734, "learning_rate": 9.776391810611718e-07, "loss": 0.0078, "num_tokens": 4529480.0, "reward": 0.73638916015625, "reward_std": 0.008760758675634861, "rewards//mean": 0.73638916015625, "rewards//std": 0.022107960656285286, "step": 524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.105, "grad_norm": 2.5822930335998535, "kl": 0.20127020124346018, "learning_rate": 9.775452470691885e-07, "loss": 0.0081, "num_tokens": 4538064.0, "reward": 0.75830078125, "reward_std": 0.007956849411129951, "rewards//mean": 0.75830078125, "rewards//std": 0.020487504079937935, "step": 525 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1052, "grad_norm": 2.515317440032959, "kl": 0.19474153500050306, "learning_rate": 9.774511207220368e-07, "loss": 0.0078, "num_tokens": 4546688.0, "reward": 0.778076171875, "reward_std": 0.008999904617667198, "rewards//mean": 0.778076171875, "rewards//std": 0.024517083540558815, "step": 526 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1054, "grad_norm": 2.553143262863159, "kl": 0.20256055518984795, "learning_rate": 9.77356802057631e-07, "loss": 0.0081, "num_tokens": 4555432.0, "reward": 0.78192138671875, "reward_std": 0.008142570033669472, "rewards//mean": 0.78192138671875, "rewards//std": 0.024024909362196922, "step": 527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1056, "grad_norm": 2.4554619789123535, "kl": 0.1910470724105835, "learning_rate": 9.77262291113962e-07, "loss": 0.0076, "num_tokens": 4564064.0, "reward": 0.773681640625, "reward_std": 0.007831105962395668, "rewards//mean": 0.773681640625, "rewards//std": 0.022998061031103134, "step": 528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1058, "grad_norm": 2.4189770221710205, "kl": 0.219999672845006, "learning_rate": 9.771675879290996e-07, "loss": 0.0088, "num_tokens": 4572752.0, "reward": 0.75238037109375, "reward_std": 0.006505347788333893, "rewards//mean": 0.75238037109375, "rewards//std": 0.024196309968829155, "step": 529 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.106, "grad_norm": 2.3663268089294434, "kl": 0.19590437784790993, "learning_rate": 9.770726925411897e-07, "loss": 0.0078, "num_tokens": 4581432.0, "reward": 0.78173828125, "reward_std": 0.007341471035033464, "rewards//mean": 0.78173828125, "rewards//std": 0.02225545234978199, "step": 530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1062, "grad_norm": 2.2811291217803955, "kl": 0.20613388065248728, "learning_rate": 9.769776049884563e-07, "loss": 0.0082, "num_tokens": 4590056.0, "reward": 0.75042724609375, "reward_std": 0.008027258329093456, "rewards//mean": 0.75042724609375, "rewards//std": 0.030585041269659996, "step": 531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1064, "grad_norm": 2.760805130004883, "kl": 0.20790709182620049, "learning_rate": 9.768823253092008e-07, "loss": 0.0083, "num_tokens": 4598776.0, "reward": 0.74578857421875, "reward_std": 0.008729961700737476, "rewards//mean": 0.74578857421875, "rewards//std": 0.020163437351584435, "step": 532 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1066, "grad_norm": 2.5373082160949707, "kl": 0.2122226096689701, "learning_rate": 9.767868535418014e-07, "loss": 0.0085, "num_tokens": 4607464.0, "reward": 0.769775390625, "reward_std": 0.009555306285619736, "rewards//mean": 0.769775390625, "rewards//std": 0.03326871246099472, "step": 533 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1068, "grad_norm": 2.4272828102111816, "kl": 0.2178900595754385, "learning_rate": 9.766911897247146e-07, "loss": 0.0087, "num_tokens": 4616104.0, "reward": 0.7227783203125, "reward_std": 0.007462595589458942, "rewards//mean": 0.7227783203125, "rewards//std": 0.03189416229724884, "step": 534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.107, "grad_norm": 2.4738922119140625, "kl": 0.2237789435312152, "learning_rate": 9.765953338964734e-07, "loss": 0.009, "num_tokens": 4624760.0, "reward": 0.756591796875, "reward_std": 0.0054986486211419106, "rewards//mean": 0.756591796875, "rewards//std": 0.02899773046374321, "step": 535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1072, "grad_norm": 2.638998031616211, "kl": 0.23875396233052015, "learning_rate": 9.76499286095689e-07, "loss": 0.0096, "num_tokens": 4633392.0, "reward": 0.750732421875, "reward_std": 0.0086929090321064, "rewards//mean": 0.750732421875, "rewards//std": 0.027240585535764694, "step": 536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1074, "grad_norm": 2.5916919708251953, "kl": 0.2088939379900694, "learning_rate": 9.764030463610488e-07, "loss": 0.0084, "num_tokens": 4642072.0, "reward": 0.7579345703125, "reward_std": 0.008034957572817802, "rewards//mean": 0.7579345703125, "rewards//std": 0.024626431986689568, "step": 537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1076, "grad_norm": 2.7041096687316895, "kl": 0.20330458413809538, "learning_rate": 9.763066147313189e-07, "loss": 0.0081, "num_tokens": 4650720.0, "reward": 0.75616455078125, "reward_std": 0.007733976934105158, "rewards//mean": 0.75616455078125, "rewards//std": 0.024278128519654274, "step": 538 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1078, "grad_norm": 2.899296283721924, "kl": 0.21454135701060295, "learning_rate": 9.762099912453412e-07, "loss": 0.0086, "num_tokens": 4659312.0, "reward": 0.7630615234375, "reward_std": 0.011397436261177063, "rewards//mean": 0.7630615234375, "rewards//std": 0.03030742146074772, "step": 539 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.108, "grad_norm": 2.7333781719207764, "kl": 0.20671376585960388, "learning_rate": 9.76113175942036e-07, "loss": 0.0083, "num_tokens": 4667896.0, "reward": 0.74896240234375, "reward_std": 0.008332246914505959, "rewards//mean": 0.74896240234375, "rewards//std": 0.024805502966046333, "step": 540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1082, "grad_norm": 2.7489192485809326, "kl": 0.22079274151474237, "learning_rate": 9.760161688604007e-07, "loss": 0.0088, "num_tokens": 4676488.0, "reward": 0.75518798828125, "reward_std": 0.008646560832858086, "rewards//mean": 0.75518798828125, "rewards//std": 0.022315820679068565, "step": 541 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1084, "grad_norm": 2.610943555831909, "kl": 0.20998156070709229, "learning_rate": 9.759189700395095e-07, "loss": 0.0084, "num_tokens": 4685128.0, "reward": 0.755126953125, "reward_std": 0.009915570728480816, "rewards//mean": 0.755126953125, "rewards//std": 0.03310084342956543, "step": 542 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1086, "grad_norm": 2.5846469402313232, "kl": 0.1863661976531148, "learning_rate": 9.758215795185138e-07, "loss": 0.0075, "num_tokens": 4693824.0, "reward": 0.7552490234375, "reward_std": 0.007276731543242931, "rewards//mean": 0.7552490234375, "rewards//std": 0.04481577128171921, "step": 543 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1088, "grad_norm": 2.8750290870666504, "kl": 0.2036522412672639, "learning_rate": 9.757239973366428e-07, "loss": 0.0081, "num_tokens": 4702472.0, "reward": 0.75543212890625, "reward_std": 0.007248344831168652, "rewards//mean": 0.75543212890625, "rewards//std": 0.03401559218764305, "step": 544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.109, "grad_norm": 2.3670542240142822, "kl": 0.21758887451142073, "learning_rate": 9.756262235332028e-07, "loss": 0.0087, "num_tokens": 4711104.0, "reward": 0.78314208984375, "reward_std": 0.006803870666772127, "rewards//mean": 0.78314208984375, "rewards//std": 0.020751703530550003, "step": 545 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1092, "grad_norm": 2.6635184288024902, "kl": 0.21388847939670086, "learning_rate": 9.755282581475767e-07, "loss": 0.0086, "num_tokens": 4719688.0, "reward": 0.7152099609375, "reward_std": 0.009670565836131573, "rewards//mean": 0.7152099609375, "rewards//std": 0.037010323256254196, "step": 546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1094, "grad_norm": 2.5421857833862305, "kl": 0.19934120308607817, "learning_rate": 9.754301012192253e-07, "loss": 0.008, "num_tokens": 4728272.0, "reward": 0.75390625, "reward_std": 0.00879538431763649, "rewards//mean": 0.75390625, "rewards//std": 0.022851470857858658, "step": 547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1096, "grad_norm": 2.671503782272339, "kl": 0.20683455374091864, "learning_rate": 9.753317527876856e-07, "loss": 0.0083, "num_tokens": 4736888.0, "reward": 0.7462158203125, "reward_std": 0.007687820121645927, "rewards//mean": 0.7462158203125, "rewards//std": 0.028830982744693756, "step": 548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1098, "grad_norm": 2.4200358390808105, "kl": 0.20728019066154957, "learning_rate": 9.75233212892573e-07, "loss": 0.0083, "num_tokens": 4745456.0, "reward": 0.76397705078125, "reward_std": 0.010222984477877617, "rewards//mean": 0.76397705078125, "rewards//std": 0.023579727858304977, "step": 549 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.11, "grad_norm": 2.7071242332458496, "kl": 0.23281041253358126, "learning_rate": 9.75134481573579e-07, "loss": 0.0093, "num_tokens": 4754000.0, "reward": 0.7882080078125, "reward_std": 0.00951644778251648, "rewards//mean": 0.7882080078125, "rewards//std": 0.021926190704107285, "step": 550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1102, "grad_norm": 2.5958337783813477, "kl": 0.19507870636880398, "learning_rate": 9.750355588704727e-07, "loss": 0.0078, "num_tokens": 4762808.0, "reward": 0.75701904296875, "reward_std": 0.00885623786598444, "rewards//mean": 0.75701904296875, "rewards//std": 0.029848111793398857, "step": 551 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1104, "grad_norm": 2.472012996673584, "kl": 0.20976228173822165, "learning_rate": 9.749364448231e-07, "loss": 0.0084, "num_tokens": 4771488.0, "reward": 0.73773193359375, "reward_std": 0.008529557846486568, "rewards//mean": 0.73773193359375, "rewards//std": 0.02323894202709198, "step": 552 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1106, "grad_norm": 2.439462900161743, "kl": 0.21856457088142633, "learning_rate": 9.748371394713841e-07, "loss": 0.0087, "num_tokens": 4780144.0, "reward": 0.74310302734375, "reward_std": 0.007541933096945286, "rewards//mean": 0.74310302734375, "rewards//std": 0.025569967925548553, "step": 553 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1108, "grad_norm": 2.706909656524658, "kl": 0.1947801075875759, "learning_rate": 9.747376428553253e-07, "loss": 0.0078, "num_tokens": 4788744.0, "reward": 0.7108154296875, "reward_std": 0.010071827098727226, "rewards//mean": 0.7108154296875, "rewards//std": 0.036030445247888565, "step": 554 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.111, "grad_norm": 2.587865114212036, "kl": 0.23300563916563988, "learning_rate": 9.746379550150008e-07, "loss": 0.0093, "num_tokens": 4797400.0, "reward": 0.7410888671875, "reward_std": 0.008374853990972042, "rewards//mean": 0.7410888671875, "rewards//std": 0.02552945725619793, "step": 555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1112, "grad_norm": 2.6079254150390625, "kl": 0.21863503381609917, "learning_rate": 9.745380759905647e-07, "loss": 0.0087, "num_tokens": 4806144.0, "reward": 0.79052734375, "reward_std": 0.007233437150716782, "rewards//mean": 0.79052734375, "rewards//std": 0.02630539983510971, "step": 556 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1114, "grad_norm": 3.000269889831543, "kl": 0.20787948928773403, "learning_rate": 9.744380058222482e-07, "loss": 0.0083, "num_tokens": 4814792.0, "reward": 0.763916015625, "reward_std": 0.011713992804288864, "rewards//mean": 0.763916015625, "rewards//std": 0.022955898195505142, "step": 557 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1116, "grad_norm": 2.738211154937744, "kl": 0.20346645265817642, "learning_rate": 9.743377445503597e-07, "loss": 0.0081, "num_tokens": 4823488.0, "reward": 0.7716064453125, "reward_std": 0.007158038206398487, "rewards//mean": 0.7716064453125, "rewards//std": 0.03104759007692337, "step": 558 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1118, "grad_norm": 2.745866298675537, "kl": 0.20644012559205294, "learning_rate": 9.742372922152845e-07, "loss": 0.0083, "num_tokens": 4832160.0, "reward": 0.77496337890625, "reward_std": 0.013732454739511013, "rewards//mean": 0.77496337890625, "rewards//std": 0.03590012714266777, "step": 559 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.112, "grad_norm": 2.579831838607788, "kl": 0.20006072893738747, "learning_rate": 9.74136648857485e-07, "loss": 0.008, "num_tokens": 4840776.0, "reward": 0.7215576171875, "reward_std": 0.00754170399159193, "rewards//mean": 0.7215576171875, "rewards//std": 0.03687099739909172, "step": 560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1122, "grad_norm": 2.6222269535064697, "kl": 0.21760489791631699, "learning_rate": 9.740358145174997e-07, "loss": 0.0087, "num_tokens": 4849480.0, "reward": 0.7720947265625, "reward_std": 0.010821899399161339, "rewards//mean": 0.7720947265625, "rewards//std": 0.031313635408878326, "step": 561 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1124, "grad_norm": 2.5947396755218506, "kl": 0.21706217341125011, "learning_rate": 9.73934789235945e-07, "loss": 0.0087, "num_tokens": 4858184.0, "reward": 0.7625732421875, "reward_std": 0.008415354415774345, "rewards//mean": 0.7625732421875, "rewards//std": 0.03192072734236717, "step": 562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1126, "grad_norm": 2.336047410964966, "kl": 0.18568948656320572, "learning_rate": 9.73833573053514e-07, "loss": 0.0074, "num_tokens": 4866856.0, "reward": 0.77105712890625, "reward_std": 0.00783691368997097, "rewards//mean": 0.77105712890625, "rewards//std": 0.022494863718748093, "step": 563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1128, "grad_norm": 2.3330979347229004, "kl": 0.20027238689363003, "learning_rate": 9.737321660109766e-07, "loss": 0.008, "num_tokens": 4875376.0, "reward": 0.73626708984375, "reward_std": 0.008788051083683968, "rewards//mean": 0.73626708984375, "rewards//std": 0.03000037744641304, "step": 564 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.113, "grad_norm": 2.5522613525390625, "kl": 0.2055955696851015, "learning_rate": 9.73630568149179e-07, "loss": 0.0082, "num_tokens": 4884008.0, "reward": 0.721435546875, "reward_std": 0.007860198616981506, "rewards//mean": 0.721435546875, "rewards//std": 0.03621755167841911, "step": 565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1132, "grad_norm": 2.5428450107574463, "kl": 0.19873892422765493, "learning_rate": 9.735287795090454e-07, "loss": 0.0079, "num_tokens": 4892672.0, "reward": 0.77001953125, "reward_std": 0.009354211390018463, "rewards//mean": 0.77001953125, "rewards//std": 0.025794023647904396, "step": 566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1134, "grad_norm": 2.505075454711914, "kl": 0.20103717595338821, "learning_rate": 9.734268001315759e-07, "loss": 0.008, "num_tokens": 4901272.0, "reward": 0.73175048828125, "reward_std": 0.011893498711287975, "rewards//mean": 0.73175048828125, "rewards//std": 0.03574080392718315, "step": 567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1136, "grad_norm": 2.3751235008239746, "kl": 0.23137819208204746, "learning_rate": 9.733246300578482e-07, "loss": 0.0093, "num_tokens": 4909840.0, "reward": 0.76470947265625, "reward_std": 0.008945251815021038, "rewards//mean": 0.76470947265625, "rewards//std": 0.023465821519494057, "step": 568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1138, "grad_norm": 2.253753423690796, "kl": 0.18717945273965597, "learning_rate": 9.73222269329016e-07, "loss": 0.0075, "num_tokens": 4918384.0, "reward": 0.75146484375, "reward_std": 0.0072065722197294235, "rewards//mean": 0.75146484375, "rewards//std": 0.016374332830309868, "step": 569 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.114, "grad_norm": 2.406771183013916, "kl": 0.20669802837073803, "learning_rate": 9.731197179863103e-07, "loss": 0.0083, "num_tokens": 4927096.0, "reward": 0.77728271484375, "reward_std": 0.00788349099457264, "rewards//mean": 0.77728271484375, "rewards//std": 0.030165426433086395, "step": 570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1142, "grad_norm": 2.6489546298980713, "kl": 0.21308070421218872, "learning_rate": 9.730169760710385e-07, "loss": 0.0085, "num_tokens": 4935776.0, "reward": 0.74444580078125, "reward_std": 0.007337694056332111, "rewards//mean": 0.74444580078125, "rewards//std": 0.025256050750613213, "step": 571 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1144, "grad_norm": 2.2121670246124268, "kl": 0.20004741102457047, "learning_rate": 9.729140436245856e-07, "loss": 0.008, "num_tokens": 4944400.0, "reward": 0.73980712890625, "reward_std": 0.007258987985551357, "rewards//mean": 0.73980712890625, "rewards//std": 0.02781897597014904, "step": 572 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1146, "grad_norm": 2.572361946105957, "kl": 0.21485054399818182, "learning_rate": 9.728109206884125e-07, "loss": 0.0086, "num_tokens": 4953000.0, "reward": 0.75616455078125, "reward_std": 0.00930515956133604, "rewards//mean": 0.75616455078125, "rewards//std": 0.02749660424888134, "step": 573 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1148, "grad_norm": 2.499732494354248, "kl": 0.18966595456004143, "learning_rate": 9.72707607304057e-07, "loss": 0.0076, "num_tokens": 4961696.0, "reward": 0.76312255859375, "reward_std": 0.006110053043812513, "rewards//mean": 0.76312255859375, "rewards//std": 0.026850959286093712, "step": 574 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.115, "grad_norm": 2.362006664276123, "kl": 0.18402161914855242, "learning_rate": 9.726041035131338e-07, "loss": 0.0074, "num_tokens": 4970296.0, "reward": 0.741943359375, "reward_std": 0.008008334785699844, "rewards//mean": 0.741943359375, "rewards//std": 0.024831196293234825, "step": 575 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1152, "grad_norm": 2.7022228240966797, "kl": 0.24209712725132704, "learning_rate": 9.72500409357334e-07, "loss": 0.0097, "num_tokens": 4979040.0, "reward": 0.75054931640625, "reward_std": 0.008825918659567833, "rewards//mean": 0.75054931640625, "rewards//std": 0.04422084242105484, "step": 576 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1154, "grad_norm": 2.5268378257751465, "kl": 0.21274339873343706, "learning_rate": 9.723965248784262e-07, "loss": 0.0085, "num_tokens": 4987720.0, "reward": 0.79443359375, "reward_std": 0.00982908345758915, "rewards//mean": 0.79443359375, "rewards//std": 0.018525702878832817, "step": 577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1156, "grad_norm": 2.417515277862549, "kl": 0.21194756217300892, "learning_rate": 9.722924501182546e-07, "loss": 0.0085, "num_tokens": 4996352.0, "reward": 0.7261962890625, "reward_std": 0.0078034717589616776, "rewards//mean": 0.7261962890625, "rewards//std": 0.03793460503220558, "step": 578 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1158, "grad_norm": 2.5283761024475098, "kl": 0.18611767049878836, "learning_rate": 9.721881851187405e-07, "loss": 0.0074, "num_tokens": 5005168.0, "reward": 0.75762939453125, "reward_std": 0.010058899410068989, "rewards//mean": 0.75762939453125, "rewards//std": 0.03418472781777382, "step": 579 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.116, "grad_norm": 1.9342117309570312, "kl": 0.1932758465409279, "learning_rate": 9.720837299218818e-07, "loss": 0.0077, "num_tokens": 5013744.0, "reward": 0.75384521484375, "reward_std": 0.005680167116224766, "rewards//mean": 0.75384521484375, "rewards//std": 0.025079822167754173, "step": 580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1162, "grad_norm": 2.3345181941986084, "kl": 0.20730253774672747, "learning_rate": 9.719790845697532e-07, "loss": 0.0083, "num_tokens": 5022304.0, "reward": 0.7408447265625, "reward_std": 0.009484879672527313, "rewards//mean": 0.7408447265625, "rewards//std": 0.031383171677589417, "step": 581 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1164, "grad_norm": 2.366091728210449, "kl": 0.20875695161521435, "learning_rate": 9.71874249104506e-07, "loss": 0.0084, "num_tokens": 5030944.0, "reward": 0.70733642578125, "reward_std": 0.00794148351997137, "rewards//mean": 0.70733642578125, "rewards//std": 0.029824772849678993, "step": 582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1166, "grad_norm": 2.5741488933563232, "kl": 0.19299913849681616, "learning_rate": 9.717692235683674e-07, "loss": 0.0077, "num_tokens": 5039632.0, "reward": 0.76080322265625, "reward_std": 0.007838273420929909, "rewards//mean": 0.76080322265625, "rewards//std": 0.026335233822464943, "step": 583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1168, "grad_norm": 2.336000919342041, "kl": 0.20232780231162906, "learning_rate": 9.716640080036423e-07, "loss": 0.0081, "num_tokens": 5048256.0, "reward": 0.75787353515625, "reward_std": 0.011394470930099487, "rewards//mean": 0.75787353515625, "rewards//std": 0.020225655287504196, "step": 584 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.117, "grad_norm": 2.3802225589752197, "kl": 0.2070111045613885, "learning_rate": 9.715586024527109e-07, "loss": 0.0083, "num_tokens": 5056808.0, "reward": 0.757080078125, "reward_std": 0.007744330447167158, "rewards//mean": 0.757080078125, "rewards//std": 0.0297480970621109, "step": 585 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1172, "grad_norm": 2.345366954803467, "kl": 0.19000516179949045, "learning_rate": 9.714530069580308e-07, "loss": 0.0076, "num_tokens": 5065400.0, "reward": 0.7744140625, "reward_std": 0.007273450493812561, "rewards//mean": 0.7744140625, "rewards//std": 0.030425317585468292, "step": 586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1174, "grad_norm": 2.5098626613616943, "kl": 0.21227385383099318, "learning_rate": 9.71347221562136e-07, "loss": 0.0085, "num_tokens": 5074064.0, "reward": 0.78564453125, "reward_std": 0.006442628800868988, "rewards//mean": 0.78564453125, "rewards//std": 0.026406485587358475, "step": 587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1176, "grad_norm": 2.6274147033691406, "kl": 0.2008717842400074, "learning_rate": 9.712412463076367e-07, "loss": 0.008, "num_tokens": 5082720.0, "reward": 0.7396240234375, "reward_std": 0.01041447278112173, "rewards//mean": 0.7396240234375, "rewards//std": 0.0307280533015728, "step": 588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1178, "grad_norm": 2.2818710803985596, "kl": 0.2134570386260748, "learning_rate": 9.711350812372196e-07, "loss": 0.0085, "num_tokens": 5091344.0, "reward": 0.73651123046875, "reward_std": 0.00837304349988699, "rewards//mean": 0.73651123046875, "rewards//std": 0.02319786697626114, "step": 589 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.118, "grad_norm": 2.303163528442383, "kl": 0.22205129265785217, "learning_rate": 9.710287263936483e-07, "loss": 0.0089, "num_tokens": 5100024.0, "reward": 0.76922607421875, "reward_std": 0.009423606097698212, "rewards//mean": 0.76922607421875, "rewards//std": 0.01837875321507454, "step": 590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1182, "grad_norm": 2.4563488960266113, "kl": 0.21712797321379185, "learning_rate": 9.709221818197623e-07, "loss": 0.0087, "num_tokens": 5108640.0, "reward": 0.736572265625, "reward_std": 0.008645838126540184, "rewards//mean": 0.736572265625, "rewards//std": 0.023050658404827118, "step": 591 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1184, "grad_norm": 2.8191616535186768, "kl": 0.2220251578837633, "learning_rate": 9.708154475584777e-07, "loss": 0.0089, "num_tokens": 5117224.0, "reward": 0.737548828125, "reward_std": 0.008272893726825714, "rewards//mean": 0.737548828125, "rewards//std": 0.029478194192051888, "step": 592 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1186, "grad_norm": 2.3526864051818848, "kl": 0.22833502758294344, "learning_rate": 9.707085236527873e-07, "loss": 0.0091, "num_tokens": 5125776.0, "reward": 0.78204345703125, "reward_std": 0.00702026579529047, "rewards//mean": 0.78204345703125, "rewards//std": 0.020448213443160057, "step": 593 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1188, "grad_norm": 2.4170947074890137, "kl": 0.2121505094692111, "learning_rate": 9.706014101457599e-07, "loss": 0.0085, "num_tokens": 5134408.0, "reward": 0.776611328125, "reward_std": 0.007152588106691837, "rewards//mean": 0.776611328125, "rewards//std": 0.020773665979504585, "step": 594 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.119, "grad_norm": 2.487192153930664, "kl": 0.24224482756108046, "learning_rate": 9.704941070805405e-07, "loss": 0.0097, "num_tokens": 5143040.0, "reward": 0.7618408203125, "reward_std": 0.007607296109199524, "rewards//mean": 0.7618408203125, "rewards//std": 0.020422007888555527, "step": 595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1192, "grad_norm": 2.591724395751953, "kl": 0.22526742238551378, "learning_rate": 9.70386614500351e-07, "loss": 0.009, "num_tokens": 5151704.0, "reward": 0.7568359375, "reward_std": 0.007447593845427036, "rewards//mean": 0.7568359375, "rewards//std": 0.020587746053934097, "step": 596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1194, "grad_norm": 2.794344186782837, "kl": 0.2177692949771881, "learning_rate": 9.702789324484896e-07, "loss": 0.0087, "num_tokens": 5160400.0, "reward": 0.750732421875, "reward_std": 0.010969633236527443, "rewards//mean": 0.750732421875, "rewards//std": 0.021952755749225616, "step": 597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1196, "grad_norm": 2.484254837036133, "kl": 0.22438115626573563, "learning_rate": 9.701710609683305e-07, "loss": 0.009, "num_tokens": 5169024.0, "reward": 0.7562255859375, "reward_std": 0.007083019241690636, "rewards//mean": 0.7562255859375, "rewards//std": 0.026011131703853607, "step": 598 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1198, "grad_norm": 2.862764358520508, "kl": 0.21322147641330957, "learning_rate": 9.700630001033243e-07, "loss": 0.0085, "num_tokens": 5177672.0, "reward": 0.759765625, "reward_std": 0.006926173344254494, "rewards//mean": 0.759765625, "rewards//std": 0.025132955983281136, "step": 599 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.12, "grad_norm": 2.6096386909484863, "kl": 0.22782303858548403, "learning_rate": 9.699547498969978e-07, "loss": 0.0091, "num_tokens": 5186400.0, "reward": 0.7332763671875, "reward_std": 0.007928953506052494, "rewards//mean": 0.7332763671875, "rewards//std": 0.03339655324816704, "step": 600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1202, "grad_norm": 2.481480121612549, "kl": 0.2197017315775156, "learning_rate": 9.698463103929541e-07, "loss": 0.0088, "num_tokens": 5195048.0, "reward": 0.73583984375, "reward_std": 0.00684010935947299, "rewards//mean": 0.73583984375, "rewards//std": 0.020237719640135765, "step": 601 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1204, "grad_norm": 2.598886013031006, "kl": 0.248580701649189, "learning_rate": 9.69737681634873e-07, "loss": 0.0099, "num_tokens": 5203712.0, "reward": 0.7767333984375, "reward_std": 0.008067253045737743, "rewards//mean": 0.7767333984375, "rewards//std": 0.0184171162545681, "step": 602 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1206, "grad_norm": 2.55953311920166, "kl": 0.236043612472713, "learning_rate": 9.696288636665097e-07, "loss": 0.0094, "num_tokens": 5212352.0, "reward": 0.766845703125, "reward_std": 0.007209985051304102, "rewards//mean": 0.766845703125, "rewards//std": 0.02043275348842144, "step": 603 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1208, "grad_norm": 2.678175210952759, "kl": 0.231639688834548, "learning_rate": 9.695198565316964e-07, "loss": 0.0093, "num_tokens": 5220912.0, "reward": 0.76904296875, "reward_std": 0.009503869339823723, "rewards//mean": 0.76904296875, "rewards//std": 0.03473345935344696, "step": 604 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.121, "grad_norm": 2.3178458213806152, "kl": 0.2136409254744649, "learning_rate": 9.69410660274341e-07, "loss": 0.0085, "num_tokens": 5229616.0, "reward": 0.75732421875, "reward_std": 0.0072447145357728004, "rewards//mean": 0.75732421875, "rewards//std": 0.020675791427493095, "step": 605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1212, "grad_norm": 2.709106922149658, "kl": 0.24635965004563332, "learning_rate": 9.693012749384277e-07, "loss": 0.0099, "num_tokens": 5238192.0, "reward": 0.73809814453125, "reward_std": 0.01116020604968071, "rewards//mean": 0.73809814453125, "rewards//std": 0.028742311522364616, "step": 606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1214, "grad_norm": 2.685481071472168, "kl": 0.23443083930760622, "learning_rate": 9.691917005680173e-07, "loss": 0.0094, "num_tokens": 5246720.0, "reward": 0.7750244140625, "reward_std": 0.00865564402192831, "rewards//mean": 0.7750244140625, "rewards//std": 0.029171306639909744, "step": 607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1216, "grad_norm": 2.558425188064575, "kl": 0.22762812487781048, "learning_rate": 9.690819372072456e-07, "loss": 0.0091, "num_tokens": 5255328.0, "reward": 0.74798583984375, "reward_std": 0.009355241432785988, "rewards//mean": 0.74798583984375, "rewards//std": 0.024773750454187393, "step": 608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1218, "grad_norm": 2.848205089569092, "kl": 0.2626179587095976, "learning_rate": 9.68971984900326e-07, "loss": 0.0105, "num_tokens": 5263992.0, "reward": 0.7454833984375, "reward_std": 0.007315634749829769, "rewards//mean": 0.7454833984375, "rewards//std": 0.02932450734078884, "step": 609 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.122, "grad_norm": 2.914008855819702, "kl": 0.21053088549524546, "learning_rate": 9.688618436915468e-07, "loss": 0.0084, "num_tokens": 5272632.0, "reward": 0.745849609375, "reward_std": 0.008161933161318302, "rewards//mean": 0.745849609375, "rewards//std": 0.025064200162887573, "step": 610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1222, "grad_norm": 2.6426734924316406, "kl": 0.26074648182839155, "learning_rate": 9.68751513625273e-07, "loss": 0.0104, "num_tokens": 5281264.0, "reward": 0.76171875, "reward_std": 0.009589259512722492, "rewards//mean": 0.76171875, "rewards//std": 0.024793371558189392, "step": 611 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1224, "grad_norm": 3.003979206085205, "kl": 0.23192260041832924, "learning_rate": 9.686409947459457e-07, "loss": 0.0093, "num_tokens": 5290096.0, "reward": 0.7344970703125, "reward_std": 0.008901823312044144, "rewards//mean": 0.7344970703125, "rewards//std": 0.029090242460370064, "step": 612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1226, "grad_norm": 2.669278860092163, "kl": 0.2331440532580018, "learning_rate": 9.685302870980817e-07, "loss": 0.0093, "num_tokens": 5298720.0, "reward": 0.7698974609375, "reward_std": 0.008679483085870743, "rewards//mean": 0.7698974609375, "rewards//std": 0.025303132832050323, "step": 613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1228, "grad_norm": 3.0592589378356934, "kl": 0.24727703537791967, "learning_rate": 9.684193907262742e-07, "loss": 0.0099, "num_tokens": 5307344.0, "reward": 0.743408203125, "reward_std": 0.011624716222286224, "rewards//mean": 0.743408203125, "rewards//std": 0.03264402970671654, "step": 614 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.123, "grad_norm": 2.7208240032196045, "kl": 0.24731595907360315, "learning_rate": 9.68308305675192e-07, "loss": 0.0099, "num_tokens": 5315880.0, "reward": 0.7630615234375, "reward_std": 0.00930720753967762, "rewards//mean": 0.7630615234375, "rewards//std": 0.025922521948814392, "step": 615 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1232, "grad_norm": 2.5424563884735107, "kl": 0.2442295728251338, "learning_rate": 9.681970319895802e-07, "loss": 0.0098, "num_tokens": 5324592.0, "reward": 0.76751708984375, "reward_std": 0.009459644556045532, "rewards//mean": 0.76751708984375, "rewards//std": 0.02340639755129814, "step": 616 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1234, "grad_norm": 2.7117526531219482, "kl": 0.23817005194723606, "learning_rate": 9.6808556971426e-07, "loss": 0.0095, "num_tokens": 5333288.0, "reward": 0.75787353515625, "reward_std": 0.007240240462124348, "rewards//mean": 0.75787353515625, "rewards//std": 0.020943395793437958, "step": 617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1236, "grad_norm": 2.368464231491089, "kl": 0.23345900233834982, "learning_rate": 9.679739188941283e-07, "loss": 0.0093, "num_tokens": 5341944.0, "reward": 0.75018310546875, "reward_std": 0.008537301793694496, "rewards//mean": 0.75018310546875, "rewards//std": 0.02675211802124977, "step": 618 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1238, "grad_norm": 2.8834733963012695, "kl": 0.2347724586725235, "learning_rate": 9.678620795741582e-07, "loss": 0.0094, "num_tokens": 5350712.0, "reward": 0.8011474609375, "reward_std": 0.0081472247838974, "rewards//mean": 0.8011474609375, "rewards//std": 0.01927182264626026, "step": 619 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.124, "grad_norm": 2.6222214698791504, "kl": 0.23652953561395407, "learning_rate": 9.677500517993982e-07, "loss": 0.0095, "num_tokens": 5359400.0, "reward": 0.75927734375, "reward_std": 0.0071931458078324795, "rewards//mean": 0.75927734375, "rewards//std": 0.02862468920648098, "step": 620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1242, "grad_norm": 2.447474718093872, "kl": 0.2144069541245699, "learning_rate": 9.676378356149732e-07, "loss": 0.0086, "num_tokens": 5368040.0, "reward": 0.74359130859375, "reward_std": 0.005447229836136103, "rewards//mean": 0.74359130859375, "rewards//std": 0.035368263721466064, "step": 621 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1244, "grad_norm": 3.263336181640625, "kl": 0.24912102241069078, "learning_rate": 9.675254310660841e-07, "loss": 0.01, "num_tokens": 5376656.0, "reward": 0.73675537109375, "reward_std": 0.009956108406186104, "rewards//mean": 0.73675537109375, "rewards//std": 0.03202790021896362, "step": 622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1246, "grad_norm": 2.958742618560791, "kl": 0.2529023764654994, "learning_rate": 9.674128381980071e-07, "loss": 0.0101, "num_tokens": 5385312.0, "reward": 0.74224853515625, "reward_std": 0.006682121194899082, "rewards//mean": 0.74224853515625, "rewards//std": 0.024054504930973053, "step": 623 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1248, "grad_norm": 2.909976005554199, "kl": 0.2629548078402877, "learning_rate": 9.67300057056095e-07, "loss": 0.0105, "num_tokens": 5393888.0, "reward": 0.75115966796875, "reward_std": 0.007848279550671577, "rewards//mean": 0.75115966796875, "rewards//std": 0.025100937113165855, "step": 624 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.125, "grad_norm": 2.7330803871154785, "kl": 0.24556253291666508, "learning_rate": 9.671870876857758e-07, "loss": 0.0098, "num_tokens": 5402496.0, "reward": 0.76007080078125, "reward_std": 0.008249986916780472, "rewards//mean": 0.76007080078125, "rewards//std": 0.02755819447338581, "step": 625 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1252, "grad_norm": 2.7917697429656982, "kl": 0.266143218614161, "learning_rate": 9.670739301325534e-07, "loss": 0.0106, "num_tokens": 5411160.0, "reward": 0.76275634765625, "reward_std": 0.007092661689966917, "rewards//mean": 0.76275634765625, "rewards//std": 0.04186524078249931, "step": 626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1254, "grad_norm": 2.990387439727783, "kl": 0.2544079925864935, "learning_rate": 9.669605844420078e-07, "loss": 0.0102, "num_tokens": 5419800.0, "reward": 0.78546142578125, "reward_std": 0.007669455837458372, "rewards//mean": 0.78546142578125, "rewards//std": 0.026569290086627007, "step": 627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1256, "grad_norm": 3.2159271240234375, "kl": 0.28000779543071985, "learning_rate": 9.668470506597946e-07, "loss": 0.0112, "num_tokens": 5428536.0, "reward": 0.73834228515625, "reward_std": 0.01008367445319891, "rewards//mean": 0.73834228515625, "rewards//std": 0.021637342870235443, "step": 628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1258, "grad_norm": 3.7808144092559814, "kl": 0.28342091385275126, "learning_rate": 9.667333288316453e-07, "loss": 0.0113, "num_tokens": 5437144.0, "reward": 0.78680419921875, "reward_std": 0.007455417886376381, "rewards//mean": 0.78680419921875, "rewards//std": 0.02565566450357437, "step": 629 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.126, "grad_norm": 3.1781973838806152, "kl": 0.34094157721847296, "learning_rate": 9.66619419003367e-07, "loss": 0.0136, "num_tokens": 5445856.0, "reward": 0.75860595703125, "reward_std": 0.010273408144712448, "rewards//mean": 0.75860595703125, "rewards//std": 0.02941489778459072, "step": 630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1262, "grad_norm": 4.200908660888672, "kl": 0.31012608483433723, "learning_rate": 9.665053212208426e-07, "loss": 0.0124, "num_tokens": 5454512.0, "reward": 0.72613525390625, "reward_std": 0.01104994397610426, "rewards//mean": 0.72613525390625, "rewards//std": 0.03484218940138817, "step": 631 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1264, "grad_norm": 3.3702046871185303, "kl": 0.29447157122194767, "learning_rate": 9.663910355300304e-07, "loss": 0.0118, "num_tokens": 5463144.0, "reward": 0.7642822265625, "reward_std": 0.00872592069208622, "rewards//mean": 0.7642822265625, "rewards//std": 0.02257656119763851, "step": 632 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1266, "grad_norm": 3.8665194511413574, "kl": 0.2957277214154601, "learning_rate": 9.66276561976965e-07, "loss": 0.0118, "num_tokens": 5471768.0, "reward": 0.750244140625, "reward_std": 0.00933067500591278, "rewards//mean": 0.750244140625, "rewards//std": 0.019610146060585976, "step": 633 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1268, "grad_norm": 3.341804027557373, "kl": 0.2521153260022402, "learning_rate": 9.661619006077561e-07, "loss": 0.0101, "num_tokens": 5480384.0, "reward": 0.75299072265625, "reward_std": 0.007986579090356827, "rewards//mean": 0.75299072265625, "rewards//std": 0.021505415439605713, "step": 634 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.127, "grad_norm": 3.5835232734680176, "kl": 0.33276358991861343, "learning_rate": 9.660470514685895e-07, "loss": 0.0133, "num_tokens": 5488984.0, "reward": 0.73980712890625, "reward_std": 0.008749248459935188, "rewards//mean": 0.73980712890625, "rewards//std": 0.024534286931157112, "step": 635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1272, "grad_norm": 3.281963348388672, "kl": 0.26525493059307337, "learning_rate": 9.659320146057262e-07, "loss": 0.0106, "num_tokens": 5497624.0, "reward": 0.81622314453125, "reward_std": 0.008145196363329887, "rewards//mean": 0.81622314453125, "rewards//std": 0.019604645669460297, "step": 636 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1274, "grad_norm": 3.1588807106018066, "kl": 0.3334341961890459, "learning_rate": 9.65816790065503e-07, "loss": 0.0133, "num_tokens": 5506312.0, "reward": 0.74041748046875, "reward_std": 0.008933894336223602, "rewards//mean": 0.74041748046875, "rewards//std": 0.025957170873880386, "step": 637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1276, "grad_norm": 4.310033798217773, "kl": 0.3234546445310116, "learning_rate": 9.657013778943327e-07, "loss": 0.0129, "num_tokens": 5514912.0, "reward": 0.73516845703125, "reward_std": 0.010786399245262146, "rewards//mean": 0.73516845703125, "rewards//std": 0.019138913601636887, "step": 638 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1278, "grad_norm": 3.3058111667633057, "kl": 0.26327971275895834, "learning_rate": 9.65585778138703e-07, "loss": 0.0105, "num_tokens": 5523536.0, "reward": 0.78564453125, "reward_std": 0.006881622597575188, "rewards//mean": 0.78564453125, "rewards//std": 0.02183798886835575, "step": 639 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.128, "grad_norm": 5.251916885375977, "kl": 0.3826856277883053, "learning_rate": 9.654699908451776e-07, "loss": 0.0153, "num_tokens": 5532184.0, "reward": 0.730224609375, "reward_std": 0.011996911838650703, "rewards//mean": 0.730224609375, "rewards//std": 0.02739132009446621, "step": 640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1282, "grad_norm": 6.175998687744141, "kl": 0.5581840332597494, "learning_rate": 9.653540160603955e-07, "loss": 0.0223, "num_tokens": 5540808.0, "reward": 0.777587890625, "reward_std": 0.017355050891637802, "rewards//mean": 0.777587890625, "rewards//std": 0.033652350306510925, "step": 641 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1284, "grad_norm": 3.777766466140747, "kl": 0.45173448603600264, "learning_rate": 9.652378538310713e-07, "loss": 0.0181, "num_tokens": 5549448.0, "reward": 0.72308349609375, "reward_std": 0.010328175500035286, "rewards//mean": 0.72308349609375, "rewards//std": 0.030106153339147568, "step": 642 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1286, "grad_norm": 4.543262481689453, "kl": 0.37777620926499367, "learning_rate": 9.651215042039953e-07, "loss": 0.0151, "num_tokens": 5558080.0, "reward": 0.784912109375, "reward_std": 0.00804133340716362, "rewards//mean": 0.784912109375, "rewards//std": 0.024118687957525253, "step": 643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1288, "grad_norm": 5.618739128112793, "kl": 0.41111266426742077, "learning_rate": 9.650049672260333e-07, "loss": 0.0164, "num_tokens": 5566712.0, "reward": 0.777587890625, "reward_std": 0.009358042851090431, "rewards//mean": 0.777587890625, "rewards//std": 0.025352440774440765, "step": 644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.129, "grad_norm": 4.500043869018555, "kl": 0.4685266828164458, "learning_rate": 9.648882429441256e-07, "loss": 0.0187, "num_tokens": 5575304.0, "reward": 0.76861572265625, "reward_std": 0.012531119398772717, "rewards//mean": 0.76861572265625, "rewards//std": 0.028879445046186447, "step": 645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1292, "grad_norm": 4.578500747680664, "kl": 0.4764310959726572, "learning_rate": 9.647713314052895e-07, "loss": 0.0191, "num_tokens": 5584080.0, "reward": 0.7603759765625, "reward_std": 0.010578645393252373, "rewards//mean": 0.7603759765625, "rewards//std": 0.030641227960586548, "step": 646 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1294, "grad_norm": 5.9160237312316895, "kl": 0.40606721490621567, "learning_rate": 9.646542326566168e-07, "loss": 0.0162, "num_tokens": 5592720.0, "reward": 0.76458740234375, "reward_std": 0.0073903873562812805, "rewards//mean": 0.76458740234375, "rewards//std": 0.01999380625784397, "step": 647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1296, "grad_norm": 4.43061637878418, "kl": 0.4973544552922249, "learning_rate": 9.645369467452745e-07, "loss": 0.0199, "num_tokens": 5601432.0, "reward": 0.7669677734375, "reward_std": 0.008010782301425934, "rewards//mean": 0.7669677734375, "rewards//std": 0.012856454588472843, "step": 648 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1298, "grad_norm": 5.037925720214844, "kl": 0.5347670819610357, "learning_rate": 9.644194737185057e-07, "loss": 0.0214, "num_tokens": 5610040.0, "reward": 0.7193603515625, "reward_std": 0.009534362703561783, "rewards//mean": 0.7193603515625, "rewards//std": 0.025828920304775238, "step": 649 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.13, "grad_norm": 5.004450798034668, "kl": 0.516712237149477, "learning_rate": 9.643018136236286e-07, "loss": 0.0207, "num_tokens": 5618752.0, "reward": 0.73358154296875, "reward_std": 0.007627109065651894, "rewards//mean": 0.73358154296875, "rewards//std": 0.02682219259440899, "step": 650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1302, "grad_norm": 5.551060676574707, "kl": 0.47126047126948833, "learning_rate": 9.641839665080363e-07, "loss": 0.0189, "num_tokens": 5627320.0, "reward": 0.759521484375, "reward_std": 0.011706827208399773, "rewards//mean": 0.759521484375, "rewards//std": 0.01942399889230728, "step": 651 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1304, "grad_norm": 6.075092792510986, "kl": 0.5999239925295115, "learning_rate": 9.640659324191978e-07, "loss": 0.024, "num_tokens": 5635952.0, "reward": 0.77130126953125, "reward_std": 0.010738062672317028, "rewards//mean": 0.77130126953125, "rewards//std": 0.029699651524424553, "step": 652 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1306, "grad_norm": 5.55996036529541, "kl": 0.5250953948125243, "learning_rate": 9.639477114046572e-07, "loss": 0.021, "num_tokens": 5644616.0, "reward": 0.75323486328125, "reward_std": 0.009308221749961376, "rewards//mean": 0.75323486328125, "rewards//std": 0.021435614675283432, "step": 653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1308, "grad_norm": 4.967021465301514, "kl": 0.4518276769667864, "learning_rate": 9.63829303512034e-07, "loss": 0.0181, "num_tokens": 5653272.0, "reward": 0.7772216796875, "reward_std": 0.008542926982045174, "rewards//mean": 0.7772216796875, "rewards//std": 0.022433962672948837, "step": 654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.131, "grad_norm": 6.460659027099609, "kl": 0.5484109930694103, "learning_rate": 9.63710708789023e-07, "loss": 0.0219, "num_tokens": 5661888.0, "reward": 0.7843017578125, "reward_std": 0.009501149877905846, "rewards//mean": 0.7843017578125, "rewards//std": 0.018021628260612488, "step": 655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1312, "grad_norm": 7.013893127441406, "kl": 0.7591905985027552, "learning_rate": 9.635919272833937e-07, "loss": 0.0304, "num_tokens": 5670504.0, "reward": 0.7396240234375, "reward_std": 0.012006670236587524, "rewards//mean": 0.7396240234375, "rewards//std": 0.029880868270993233, "step": 656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1314, "grad_norm": 6.466878414154053, "kl": 0.701962748542428, "learning_rate": 9.634729590429916e-07, "loss": 0.0281, "num_tokens": 5679168.0, "reward": 0.763427734375, "reward_std": 0.011417343281209469, "rewards//mean": 0.763427734375, "rewards//std": 0.029780646786093712, "step": 657 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1316, "grad_norm": 5.745469570159912, "kl": 0.6625272277742624, "learning_rate": 9.63353804115737e-07, "loss": 0.0265, "num_tokens": 5687896.0, "reward": 0.769287109375, "reward_std": 0.007695325184613466, "rewards//mean": 0.769287109375, "rewards//std": 0.02231387048959732, "step": 658 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1318, "grad_norm": 7.092409133911133, "kl": 0.759245615452528, "learning_rate": 9.632344625496255e-07, "loss": 0.0304, "num_tokens": 5696520.0, "reward": 0.767822265625, "reward_std": 0.011308575049042702, "rewards//mean": 0.767822265625, "rewards//std": 0.016541730612516403, "step": 659 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.132, "grad_norm": 5.445815086364746, "kl": 0.3996684094890952, "learning_rate": 9.63114934392728e-07, "loss": 0.016, "num_tokens": 5705160.0, "reward": 0.7298583984375, "reward_std": 0.005815407261252403, "rewards//mean": 0.7298583984375, "rewards//std": 0.02091708965599537, "step": 660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1322, "grad_norm": 5.452217102050781, "kl": 0.6336718033999205, "learning_rate": 9.6299521969319e-07, "loss": 0.0253, "num_tokens": 5713848.0, "reward": 0.77862548828125, "reward_std": 0.00826502125710249, "rewards//mean": 0.77862548828125, "rewards//std": 0.017908191308379173, "step": 661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1324, "grad_norm": 6.652682304382324, "kl": 0.6581048853695393, "learning_rate": 9.628753184992333e-07, "loss": 0.0263, "num_tokens": 5722440.0, "reward": 0.77764892578125, "reward_std": 0.013120852410793304, "rewards//mean": 0.77764892578125, "rewards//std": 0.02943238988518715, "step": 662 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1326, "grad_norm": 5.127926349639893, "kl": 0.7042171955108643, "learning_rate": 9.627552308591533e-07, "loss": 0.0282, "num_tokens": 5731056.0, "reward": 0.7740478515625, "reward_std": 0.008530810475349426, "rewards//mean": 0.7740478515625, "rewards//std": 0.02753545716404915, "step": 663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1328, "grad_norm": 5.415334701538086, "kl": 0.842731736600399, "learning_rate": 9.62634956821322e-07, "loss": 0.0337, "num_tokens": 5739704.0, "reward": 0.73822021484375, "reward_std": 0.009494063444435596, "rewards//mean": 0.73822021484375, "rewards//std": 0.0287428367882967, "step": 664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.133, "grad_norm": 6.197539329528809, "kl": 0.8267682623118162, "learning_rate": 9.625144964341852e-07, "loss": 0.0331, "num_tokens": 5748272.0, "reward": 0.759521484375, "reward_std": 0.011234431527554989, "rewards//mean": 0.759521484375, "rewards//std": 0.027664074674248695, "step": 665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1332, "grad_norm": 5.348916530609131, "kl": 0.7823239136487246, "learning_rate": 9.623938497462645e-07, "loss": 0.0313, "num_tokens": 5756944.0, "reward": 0.7764892578125, "reward_std": 0.009534705430269241, "rewards//mean": 0.7764892578125, "rewards//std": 0.024204833433032036, "step": 666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1334, "grad_norm": 8.233922958374023, "kl": 1.2101830318570137, "learning_rate": 9.622730168061567e-07, "loss": 0.0484, "num_tokens": 5765464.0, "reward": 0.74365234375, "reward_std": 0.012155747041106224, "rewards//mean": 0.74365234375, "rewards//std": 0.028403842821717262, "step": 667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1336, "grad_norm": 7.8597798347473145, "kl": 0.7464694883674383, "learning_rate": 9.621519976625326e-07, "loss": 0.0299, "num_tokens": 5774152.0, "reward": 0.7633056640625, "reward_std": 0.012880093418061733, "rewards//mean": 0.7633056640625, "rewards//std": 0.028837284073233604, "step": 668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1338, "grad_norm": 6.381985187530518, "kl": 0.6748458743095398, "learning_rate": 9.620307923641392e-07, "loss": 0.027, "num_tokens": 5782856.0, "reward": 0.75762939453125, "reward_std": 0.00775592215359211, "rewards//mean": 0.75762939453125, "rewards//std": 0.029168905690312386, "step": 669 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.134, "grad_norm": 4.677013874053955, "kl": 1.0138772446662188, "learning_rate": 9.61909400959798e-07, "loss": 0.0406, "num_tokens": 5791512.0, "reward": 0.7457275390625, "reward_std": 0.010004445910453796, "rewards//mean": 0.7457275390625, "rewards//std": 0.03177051991224289, "step": 670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1342, "grad_norm": 7.821444034576416, "kl": 1.091262711212039, "learning_rate": 9.617878234984054e-07, "loss": 0.0437, "num_tokens": 5800024.0, "reward": 0.7340087890625, "reward_std": 0.011750814504921436, "rewards//mean": 0.7340087890625, "rewards//std": 0.023844420909881592, "step": 671 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1344, "grad_norm": 5.484076023101807, "kl": 0.47971958946436644, "learning_rate": 9.616660600289327e-07, "loss": 0.0192, "num_tokens": 5808632.0, "reward": 0.7623291015625, "reward_std": 0.009718211367726326, "rewards//mean": 0.7623291015625, "rewards//std": 0.02152203768491745, "step": 672 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1346, "grad_norm": 6.548192977905273, "kl": 0.6700717397034168, "learning_rate": 9.615441106004262e-07, "loss": 0.0268, "num_tokens": 5817208.0, "reward": 0.75799560546875, "reward_std": 0.009721084497869015, "rewards//mean": 0.75799560546875, "rewards//std": 0.02633235976099968, "step": 673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1348, "grad_norm": 8.142129898071289, "kl": 1.1573685612529516, "learning_rate": 9.614219752620072e-07, "loss": 0.0463, "num_tokens": 5826000.0, "reward": 0.76007080078125, "reward_std": 0.013684765435755253, "rewards//mean": 0.76007080078125, "rewards//std": 0.026234449818730354, "step": 674 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.135, "grad_norm": 10.413251876831055, "kl": 0.9288819264620543, "learning_rate": 9.612996540628717e-07, "loss": 0.0372, "num_tokens": 5834584.0, "reward": 0.75897216796875, "reward_std": 0.013641918078064919, "rewards//mean": 0.75897216796875, "rewards//std": 0.02662734128534794, "step": 675 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1352, "grad_norm": 6.3422393798828125, "kl": 0.916122330352664, "learning_rate": 9.611771470522907e-07, "loss": 0.0366, "num_tokens": 5843224.0, "reward": 0.729736328125, "reward_std": 0.010156441479921341, "rewards//mean": 0.729736328125, "rewards//std": 0.021630434319376945, "step": 676 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1354, "grad_norm": 6.724777698516846, "kl": 0.9932534657418728, "learning_rate": 9.6105445427961e-07, "loss": 0.0397, "num_tokens": 5851800.0, "reward": 0.7030029296875, "reward_std": 0.010675765573978424, "rewards//mean": 0.7030029296875, "rewards//std": 0.02326083369553089, "step": 677 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1356, "grad_norm": 5.712681770324707, "kl": 1.0585048235952854, "learning_rate": 9.609315757942502e-07, "loss": 0.0423, "num_tokens": 5860408.0, "reward": 0.743896484375, "reward_std": 0.00818407628685236, "rewards//mean": 0.743896484375, "rewards//std": 0.0249673780053854, "step": 678 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1358, "grad_norm": 5.611710071563721, "kl": 0.8865110967308283, "learning_rate": 9.608085116457068e-07, "loss": 0.0355, "num_tokens": 5869048.0, "reward": 0.760498046875, "reward_std": 0.008227774873375893, "rewards//mean": 0.760498046875, "rewards//std": 0.02861093543469906, "step": 679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.136, "grad_norm": 7.391247749328613, "kl": 1.793531896546483, "learning_rate": 9.606852618835502e-07, "loss": 0.0717, "num_tokens": 5877704.0, "reward": 0.76129150390625, "reward_std": 0.013443115167319775, "rewards//mean": 0.76129150390625, "rewards//std": 0.029619522392749786, "step": 680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1362, "grad_norm": 5.805166721343994, "kl": 1.892662312835455, "learning_rate": 9.60561826557425e-07, "loss": 0.0757, "num_tokens": 5886232.0, "reward": 0.743408203125, "reward_std": 0.009377227164804935, "rewards//mean": 0.743408203125, "rewards//std": 0.02087833546102047, "step": 681 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1364, "grad_norm": 6.622632026672363, "kl": 1.2632559463381767, "learning_rate": 9.604382057170512e-07, "loss": 0.0505, "num_tokens": 5894872.0, "reward": 0.732421875, "reward_std": 0.01137983426451683, "rewards//mean": 0.732421875, "rewards//std": 0.036037378013134, "step": 682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1366, "grad_norm": 9.4186372756958, "kl": 1.7058883737772703, "learning_rate": 9.603143994122232e-07, "loss": 0.0682, "num_tokens": 5903480.0, "reward": 0.7294921875, "reward_std": 0.015523059293627739, "rewards//mean": 0.7294921875, "rewards//std": 0.03373950347304344, "step": 683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1368, "grad_norm": 6.104360103607178, "kl": 1.9442303087562323, "learning_rate": 9.601904076928102e-07, "loss": 0.0778, "num_tokens": 5912088.0, "reward": 0.7520751953125, "reward_std": 0.009610344655811787, "rewards//mean": 0.7520751953125, "rewards//std": 0.027347903698682785, "step": 684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.137, "grad_norm": 7.810328006744385, "kl": 1.7898661009967327, "learning_rate": 9.60066230608756e-07, "loss": 0.0716, "num_tokens": 5920760.0, "reward": 0.78533935546875, "reward_std": 0.011764805763959885, "rewards//mean": 0.78533935546875, "rewards//std": 0.020439328625798225, "step": 685 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1372, "grad_norm": 8.834696769714355, "kl": 2.512271039187908, "learning_rate": 9.599418682100792e-07, "loss": 0.1005, "num_tokens": 5929320.0, "reward": 0.7481689453125, "reward_std": 0.009096116758883, "rewards//mean": 0.7481689453125, "rewards//std": 0.02685638517141342, "step": 686 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1374, "grad_norm": 5.598320960998535, "kl": 1.4660762287676334, "learning_rate": 9.598173205468727e-07, "loss": 0.0586, "num_tokens": 5937912.0, "reward": 0.7310791015625, "reward_std": 0.009189204312860966, "rewards//mean": 0.7310791015625, "rewards//std": 0.031467948108911514, "step": 687 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1376, "grad_norm": 7.345466136932373, "kl": 1.1756226010620594, "learning_rate": 9.596925876693047e-07, "loss": 0.047, "num_tokens": 5946624.0, "reward": 0.7484130859375, "reward_std": 0.011727170087397099, "rewards//mean": 0.7484130859375, "rewards//std": 0.027522260323166847, "step": 688 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1378, "grad_norm": 5.2736382484436035, "kl": 2.056324951350689, "learning_rate": 9.595676696276171e-07, "loss": 0.0823, "num_tokens": 5955192.0, "reward": 0.71044921875, "reward_std": 0.011874396353960037, "rewards//mean": 0.71044921875, "rewards//std": 0.032641246914863586, "step": 689 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.138, "grad_norm": 6.256668567657471, "kl": 1.3164225164800882, "learning_rate": 9.594425664721274e-07, "loss": 0.0527, "num_tokens": 5963760.0, "reward": 0.76806640625, "reward_std": 0.011647808365523815, "rewards//mean": 0.76806640625, "rewards//std": 0.023401303216814995, "step": 690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1382, "grad_norm": 6.807124614715576, "kl": 1.285219006240368, "learning_rate": 9.593172782532267e-07, "loss": 0.0514, "num_tokens": 5972352.0, "reward": 0.7537841796875, "reward_std": 0.01029349397867918, "rewards//mean": 0.7537841796875, "rewards//std": 0.029031902551651, "step": 691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1384, "grad_norm": 8.663215637207031, "kl": 2.398256568238139, "learning_rate": 9.591918050213813e-07, "loss": 0.0959, "num_tokens": 5980920.0, "reward": 0.74505615234375, "reward_std": 0.011829331517219543, "rewards//mean": 0.74505615234375, "rewards//std": 0.026759473606944084, "step": 692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1386, "grad_norm": 6.38392448425293, "kl": 1.77195187471807, "learning_rate": 9.590661468271318e-07, "loss": 0.0709, "num_tokens": 5989576.0, "reward": 0.759765625, "reward_std": 0.011866278946399689, "rewards//mean": 0.759765625, "rewards//std": 0.03202323615550995, "step": 693 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1388, "grad_norm": 6.773484706878662, "kl": 1.5403836891055107, "learning_rate": 9.589403037210931e-07, "loss": 0.0616, "num_tokens": 5998160.0, "reward": 0.7611083984375, "reward_std": 0.00890538189560175, "rewards//mean": 0.7611083984375, "rewards//std": 0.023263435810804367, "step": 694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.139, "grad_norm": 8.277812004089355, "kl": 3.2806071285158396, "learning_rate": 9.58814275753955e-07, "loss": 0.1312, "num_tokens": 6006776.0, "reward": 0.7581787109375, "reward_std": 0.013688182458281517, "rewards//mean": 0.7581787109375, "rewards//std": 0.03678056225180626, "step": 695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1392, "grad_norm": 8.247855186462402, "kl": 0.9284660518169403, "learning_rate": 9.586880629764817e-07, "loss": 0.0371, "num_tokens": 6015376.0, "reward": 0.77325439453125, "reward_std": 0.009077081456780434, "rewards//mean": 0.77325439453125, "rewards//std": 0.018577009439468384, "step": 696 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1394, "grad_norm": 6.369467258453369, "kl": 2.8970567900687456, "learning_rate": 9.585616654395112e-07, "loss": 0.1159, "num_tokens": 6023976.0, "reward": 0.75634765625, "reward_std": 0.01232706569135189, "rewards//mean": 0.75634765625, "rewards//std": 0.0297206062823534, "step": 697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1396, "grad_norm": 5.498188018798828, "kl": 2.0883035007864237, "learning_rate": 9.584350831939569e-07, "loss": 0.0835, "num_tokens": 6032584.0, "reward": 0.78399658203125, "reward_std": 0.01031227596104145, "rewards//mean": 0.78399658203125, "rewards//std": 0.023454852402210236, "step": 698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1398, "grad_norm": 5.672773361206055, "kl": 1.3796262983232737, "learning_rate": 9.58308316290806e-07, "loss": 0.0552, "num_tokens": 6041200.0, "reward": 0.73748779296875, "reward_std": 0.007399318739771843, "rewards//mean": 0.73748779296875, "rewards//std": 0.034528397023677826, "step": 699 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.14, "grad_norm": 4.546067237854004, "kl": 0.89860107190907, "learning_rate": 9.581813647811197e-07, "loss": 0.0359, "num_tokens": 6049776.0, "reward": 0.7552490234375, "reward_std": 0.00793510116636753, "rewards//mean": 0.7552490234375, "rewards//std": 0.03077531047165394, "step": 700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1402, "grad_norm": 6.4733734130859375, "kl": 2.410829773172736, "learning_rate": 9.580542287160346e-07, "loss": 0.0964, "num_tokens": 6058352.0, "reward": 0.77392578125, "reward_std": 0.012427553534507751, "rewards//mean": 0.77392578125, "rewards//std": 0.03509419038891792, "step": 701 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1404, "grad_norm": 6.268716335296631, "kl": 1.872730614617467, "learning_rate": 9.579269081467613e-07, "loss": 0.0749, "num_tokens": 6066912.0, "reward": 0.74884033203125, "reward_std": 0.010377561673521996, "rewards//mean": 0.74884033203125, "rewards//std": 0.027583450078964233, "step": 702 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1406, "grad_norm": 6.586522102355957, "kl": 1.6278119310736656, "learning_rate": 9.57799403124584e-07, "loss": 0.0651, "num_tokens": 6075536.0, "reward": 0.76214599609375, "reward_std": 0.012576766312122345, "rewards//mean": 0.76214599609375, "rewards//std": 0.029373183846473694, "step": 703 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1408, "grad_norm": 4.88497257232666, "kl": 1.7092098165303469, "learning_rate": 9.576717137008617e-07, "loss": 0.0684, "num_tokens": 6084152.0, "reward": 0.7347412109375, "reward_std": 0.009226744994521141, "rewards//mean": 0.7347412109375, "rewards//std": 0.03108462318778038, "step": 704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.141, "grad_norm": 4.4319233894348145, "kl": 0.4497374640777707, "learning_rate": 9.575438399270278e-07, "loss": 0.018, "num_tokens": 6092728.0, "reward": 0.7845458984375, "reward_std": 0.0051748850382864475, "rewards//mean": 0.7845458984375, "rewards//std": 0.017977897077798843, "step": 705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1412, "grad_norm": 4.941458702087402, "kl": 1.4851972945034504, "learning_rate": 9.5741578185459e-07, "loss": 0.0594, "num_tokens": 6101480.0, "reward": 0.72454833984375, "reward_std": 0.010255906730890274, "rewards//mean": 0.72454833984375, "rewards//std": 0.03637181594967842, "step": 706 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1414, "grad_norm": 8.730281829833984, "kl": 4.643188184127212, "learning_rate": 9.572875395351301e-07, "loss": 0.1857, "num_tokens": 6110208.0, "reward": 0.74664306640625, "reward_std": 0.019012529402971268, "rewards//mean": 0.74664306640625, "rewards//std": 0.03865654394030571, "step": 707 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1416, "grad_norm": 5.186831951141357, "kl": 2.2443429697304964, "learning_rate": 9.571591130203037e-07, "loss": 0.0898, "num_tokens": 6118832.0, "reward": 0.74261474609375, "reward_std": 0.010612553916871548, "rewards//mean": 0.74261474609375, "rewards//std": 0.02681259624660015, "step": 708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1418, "grad_norm": 6.91388463973999, "kl": 1.2408266803249717, "learning_rate": 9.570305023618415e-07, "loss": 0.0496, "num_tokens": 6127480.0, "reward": 0.749755859375, "reward_std": 0.008738374337553978, "rewards//mean": 0.749755859375, "rewards//std": 0.03242814913392067, "step": 709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.142, "grad_norm": 5.295790672302246, "kl": 2.0118208341300488, "learning_rate": 9.569017076115475e-07, "loss": 0.0805, "num_tokens": 6136016.0, "reward": 0.7340087890625, "reward_std": 0.01210230216383934, "rewards//mean": 0.7340087890625, "rewards//std": 0.028114309534430504, "step": 710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1422, "grad_norm": 5.204315662384033, "kl": 1.6005570348352194, "learning_rate": 9.567727288213004e-07, "loss": 0.064, "num_tokens": 6144688.0, "reward": 0.75543212890625, "reward_std": 0.008108418434858322, "rewards//mean": 0.75543212890625, "rewards//std": 0.02097950503230095, "step": 711 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1424, "grad_norm": 6.038320064544678, "kl": 2.145377606153488, "learning_rate": 9.566435660430527e-07, "loss": 0.0858, "num_tokens": 6153288.0, "reward": 0.74969482421875, "reward_std": 0.005925554782152176, "rewards//mean": 0.74969482421875, "rewards//std": 0.03265463560819626, "step": 712 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1426, "grad_norm": 6.113224983215332, "kl": 1.8366315010935068, "learning_rate": 9.565142193288312e-07, "loss": 0.0735, "num_tokens": 6161936.0, "reward": 0.75714111328125, "reward_std": 0.011059759184718132, "rewards//mean": 0.75714111328125, "rewards//std": 0.03809284418821335, "step": 713 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1428, "grad_norm": 16.415096282958984, "kl": 6.374369312077761, "learning_rate": 9.563846887307368e-07, "loss": 0.255, "num_tokens": 6170696.0, "reward": 0.76641845703125, "reward_std": 0.01262245699763298, "rewards//mean": 0.76641845703125, "rewards//std": 0.029513539746403694, "step": 714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.143, "grad_norm": 3.9718143939971924, "kl": 1.2525438275188208, "learning_rate": 9.562549743009442e-07, "loss": 0.0501, "num_tokens": 6179384.0, "reward": 0.776123046875, "reward_std": 0.006306346971541643, "rewards//mean": 0.776123046875, "rewards//std": 0.025131750851869583, "step": 715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1432, "grad_norm": 5.382948398590088, "kl": 3.109356554225087, "learning_rate": 9.561250760917025e-07, "loss": 0.1244, "num_tokens": 6187896.0, "reward": 0.76251220703125, "reward_std": 0.010493282228708267, "rewards//mean": 0.76251220703125, "rewards//std": 0.025067143142223358, "step": 716 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1434, "grad_norm": 5.200241565704346, "kl": 1.0269698034971952, "learning_rate": 9.55994994155335e-07, "loss": 0.0411, "num_tokens": 6196536.0, "reward": 0.7301025390625, "reward_std": 0.006362422835081816, "rewards//mean": 0.7301025390625, "rewards//std": 0.019817089661955833, "step": 717 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1436, "grad_norm": 5.346518516540527, "kl": 1.3138235844671726, "learning_rate": 9.558647285442381e-07, "loss": 0.0526, "num_tokens": 6205168.0, "reward": 0.7586669921875, "reward_std": 0.007854700088500977, "rewards//mean": 0.7586669921875, "rewards//std": 0.016418185085058212, "step": 718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1438, "grad_norm": 14.749935150146484, "kl": 4.530792691744864, "learning_rate": 9.55734279310883e-07, "loss": 0.1812, "num_tokens": 6213784.0, "reward": 0.7567138671875, "reward_std": 0.008137339726090431, "rewards//mean": 0.7567138671875, "rewards//std": 0.02853119745850563, "step": 719 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.144, "grad_norm": 6.514327526092529, "kl": 2.0024897698313, "learning_rate": 9.55603646507815e-07, "loss": 0.0801, "num_tokens": 6222432.0, "reward": 0.754638671875, "reward_std": 0.008971983566880226, "rewards//mean": 0.754638671875, "rewards//std": 0.026665473356842995, "step": 720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1442, "grad_norm": 4.103392601013184, "kl": 3.0026297997683287, "learning_rate": 9.554728301876524e-07, "loss": 0.1201, "num_tokens": 6231232.0, "reward": 0.75634765625, "reward_std": 0.011162412352859974, "rewards//mean": 0.75634765625, "rewards//std": 0.03155466169118881, "step": 721 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1444, "grad_norm": 4.249013900756836, "kl": 1.2251688614487648, "learning_rate": 9.553418304030885e-07, "loss": 0.049, "num_tokens": 6239984.0, "reward": 0.7470703125, "reward_std": 0.00776326097548008, "rewards//mean": 0.7470703125, "rewards//std": 0.02208612486720085, "step": 722 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1446, "grad_norm": 3.8857667446136475, "kl": 1.779951322823763, "learning_rate": 9.552106472068897e-07, "loss": 0.0712, "num_tokens": 6248576.0, "reward": 0.74786376953125, "reward_std": 0.008643015287816525, "rewards//mean": 0.74786376953125, "rewards//std": 0.02321873977780342, "step": 723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1448, "grad_norm": 9.575364112854004, "kl": 3.3338482324033976, "learning_rate": 9.550792806518967e-07, "loss": 0.1334, "num_tokens": 6257128.0, "reward": 0.7659912109375, "reward_std": 0.007942140102386475, "rewards//mean": 0.7659912109375, "rewards//std": 0.022742237895727158, "step": 724 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.145, "grad_norm": 8.159613609313965, "kl": 1.2658102735877037, "learning_rate": 9.549477307910236e-07, "loss": 0.0506, "num_tokens": 6265768.0, "reward": 0.782470703125, "reward_std": 0.007177484221756458, "rewards//mean": 0.782470703125, "rewards//std": 0.02122350223362446, "step": 725 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1452, "grad_norm": 4.528134346008301, "kl": 1.4327340200543404, "learning_rate": 9.548159976772592e-07, "loss": 0.0573, "num_tokens": 6274448.0, "reward": 0.767333984375, "reward_std": 0.007283302955329418, "rewards//mean": 0.767333984375, "rewards//std": 0.01673099398612976, "step": 726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1454, "grad_norm": 10.091915130615234, "kl": 2.697010073810816, "learning_rate": 9.546840813636652e-07, "loss": 0.1079, "num_tokens": 6283040.0, "reward": 0.74798583984375, "reward_std": 0.007095325272530317, "rewards//mean": 0.74798583984375, "rewards//std": 0.02330269105732441, "step": 727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1456, "grad_norm": 7.179139614105225, "kl": 1.4060106426477432, "learning_rate": 9.545519819033777e-07, "loss": 0.0562, "num_tokens": 6291704.0, "reward": 0.72039794921875, "reward_std": 0.011540853418409824, "rewards//mean": 0.72039794921875, "rewards//std": 0.038297347724437714, "step": 728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1458, "grad_norm": 4.981100082397461, "kl": 1.1590757528319955, "learning_rate": 9.544196993496062e-07, "loss": 0.0464, "num_tokens": 6300344.0, "reward": 0.759521484375, "reward_std": 0.006217144895344973, "rewards//mean": 0.759521484375, "rewards//std": 0.032910048961639404, "step": 729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.146, "grad_norm": 6.2702717781066895, "kl": 2.595333404839039, "learning_rate": 9.54287233755634e-07, "loss": 0.1038, "num_tokens": 6308912.0, "reward": 0.75537109375, "reward_std": 0.014401756227016449, "rewards//mean": 0.75537109375, "rewards//std": 0.029078010469675064, "step": 730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1462, "grad_norm": 3.8329060077667236, "kl": 1.225698129273951, "learning_rate": 9.541545851748185e-07, "loss": 0.049, "num_tokens": 6317680.0, "reward": 0.7335205078125, "reward_std": 0.006505907978862524, "rewards//mean": 0.7335205078125, "rewards//std": 0.029027730226516724, "step": 731 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1464, "grad_norm": 4.659354209899902, "kl": 0.8769095670431852, "learning_rate": 9.540217536605905e-07, "loss": 0.0351, "num_tokens": 6326368.0, "reward": 0.802001953125, "reward_std": 0.00857323594391346, "rewards//mean": 0.802001953125, "rewards//std": 0.016875134781003, "step": 732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1466, "grad_norm": 3.9992916584014893, "kl": 2.298052353784442, "learning_rate": 9.538887392664543e-07, "loss": 0.0919, "num_tokens": 6335016.0, "reward": 0.7589111328125, "reward_std": 0.010746128857135773, "rewards//mean": 0.7589111328125, "rewards//std": 0.035307276993989944, "step": 733 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1468, "grad_norm": 5.744341850280762, "kl": 0.4098111856728792, "learning_rate": 9.537555420459881e-07, "loss": 0.0164, "num_tokens": 6343744.0, "reward": 0.77294921875, "reward_std": 0.006228247657418251, "rewards//mean": 0.77294921875, "rewards//std": 0.02306772582232952, "step": 734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.147, "grad_norm": 3.447507619857788, "kl": 0.36606572940945625, "learning_rate": 9.53622162052844e-07, "loss": 0.0146, "num_tokens": 6352456.0, "reward": 0.78509521484375, "reward_std": 0.005339228548109531, "rewards//mean": 0.78509521484375, "rewards//std": 0.01976536586880684, "step": 735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1472, "grad_norm": 5.920298099517822, "kl": 3.188815062865615, "learning_rate": 9.534885993407474e-07, "loss": 0.1276, "num_tokens": 6361208.0, "reward": 0.765869140625, "reward_std": 0.007528609596192837, "rewards//mean": 0.765869140625, "rewards//std": 0.014346149750053883, "step": 736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1474, "grad_norm": 3.670410633087158, "kl": 1.228419966995716, "learning_rate": 9.53354853963497e-07, "loss": 0.0491, "num_tokens": 6369800.0, "reward": 0.7642822265625, "reward_std": 0.008165386505424976, "rewards//mean": 0.7642822265625, "rewards//std": 0.021476976573467255, "step": 737 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1476, "grad_norm": 5.1945343017578125, "kl": 1.1253601480275393, "learning_rate": 9.532209259749658e-07, "loss": 0.045, "num_tokens": 6378480.0, "reward": 0.75177001953125, "reward_std": 0.010367737151682377, "rewards//mean": 0.75177001953125, "rewards//std": 0.02931385673582554, "step": 738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1478, "grad_norm": 5.994574546813965, "kl": 1.9060461390763521, "learning_rate": 9.530868154290996e-07, "loss": 0.0762, "num_tokens": 6387056.0, "reward": 0.73992919921875, "reward_std": 0.010355432517826557, "rewards//mean": 0.73992919921875, "rewards//std": 0.04093245416879654, "step": 739 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.148, "grad_norm": 4.982955455780029, "kl": 0.44224715419113636, "learning_rate": 9.529525223799184e-07, "loss": 0.0177, "num_tokens": 6395720.0, "reward": 0.78619384765625, "reward_std": 0.006468282546848059, "rewards//mean": 0.78619384765625, "rewards//std": 0.018999995663762093, "step": 740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1482, "grad_norm": 3.5400283336639404, "kl": 0.6346832923591137, "learning_rate": 9.528180468815154e-07, "loss": 0.0254, "num_tokens": 6404408.0, "reward": 0.76141357421875, "reward_std": 0.0055562881752848625, "rewards//mean": 0.76141357421875, "rewards//std": 0.017542673274874687, "step": 741 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1484, "grad_norm": 6.327049255371094, "kl": 2.22835841961205, "learning_rate": 9.526833889880572e-07, "loss": 0.0891, "num_tokens": 6413088.0, "reward": 0.76513671875, "reward_std": 0.008300656452775002, "rewards//mean": 0.76513671875, "rewards//std": 0.027826661244034767, "step": 742 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1486, "grad_norm": 4.479706764221191, "kl": 2.2920671328902245, "learning_rate": 9.525485487537841e-07, "loss": 0.0917, "num_tokens": 6421912.0, "reward": 0.7872314453125, "reward_std": 0.00978466309607029, "rewards//mean": 0.7872314453125, "rewards//std": 0.021743163466453552, "step": 743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1488, "grad_norm": 4.605365753173828, "kl": 0.556102741509676, "learning_rate": 9.524135262330098e-07, "loss": 0.0222, "num_tokens": 6430552.0, "reward": 0.7945556640625, "reward_std": 0.006804032251238823, "rewards//mean": 0.7945556640625, "rewards//std": 0.018567737191915512, "step": 744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.149, "grad_norm": 3.1205813884735107, "kl": 0.6468249522149563, "learning_rate": 9.522783214801211e-07, "loss": 0.0259, "num_tokens": 6439144.0, "reward": 0.77020263671875, "reward_std": 0.00491624278947711, "rewards//mean": 0.77020263671875, "rewards//std": 0.013160609640181065, "step": 745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1492, "grad_norm": 5.286034107208252, "kl": 0.9488318208605051, "learning_rate": 9.521429345495786e-07, "loss": 0.038, "num_tokens": 6447728.0, "reward": 0.779296875, "reward_std": 0.006964164320379496, "rewards//mean": 0.779296875, "rewards//std": 0.014566050842404366, "step": 746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1494, "grad_norm": 7.0882344245910645, "kl": 1.0645874477922916, "learning_rate": 9.520073654959162e-07, "loss": 0.0426, "num_tokens": 6456312.0, "reward": 0.771484375, "reward_std": 0.007842430844902992, "rewards//mean": 0.771484375, "rewards//std": 0.029090501368045807, "step": 747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1496, "grad_norm": 4.138265609741211, "kl": 1.7977023757994175, "learning_rate": 9.518716143737409e-07, "loss": 0.0719, "num_tokens": 6464936.0, "reward": 0.78375244140625, "reward_std": 0.010723061859607697, "rewards//mean": 0.78375244140625, "rewards//std": 0.019077913835644722, "step": 748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1498, "grad_norm": 4.277359485626221, "kl": 2.4151702784001827, "learning_rate": 9.517356812377335e-07, "loss": 0.0966, "num_tokens": 6473664.0, "reward": 0.734619140625, "reward_std": 0.008682307787239552, "rewards//mean": 0.734619140625, "rewards//std": 0.03941971808671951, "step": 749 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.15, "grad_norm": 5.214427471160889, "kl": 2.4279807303100824, "learning_rate": 9.515995661426477e-07, "loss": 0.0971, "num_tokens": 6482200.0, "reward": 0.74639892578125, "reward_std": 0.010630439035594463, "rewards//mean": 0.74639892578125, "rewards//std": 0.02370777539908886, "step": 750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1502, "grad_norm": 4.322005271911621, "kl": 2.101282015442848, "learning_rate": 9.514632691433106e-07, "loss": 0.0841, "num_tokens": 6490712.0, "reward": 0.751953125, "reward_std": 0.007668613456189632, "rewards//mean": 0.751953125, "rewards//std": 0.028262794017791748, "step": 751 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1504, "grad_norm": 7.709218502044678, "kl": 2.939092857763171, "learning_rate": 9.513267902946227e-07, "loss": 0.1176, "num_tokens": 6499392.0, "reward": 0.77880859375, "reward_std": 0.01251686830073595, "rewards//mean": 0.77880859375, "rewards//std": 0.024384593591094017, "step": 752 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1506, "grad_norm": 4.707404613494873, "kl": 2.776157133281231, "learning_rate": 9.511901296515576e-07, "loss": 0.111, "num_tokens": 6508160.0, "reward": 0.76953125, "reward_std": 0.009979372844099998, "rewards//mean": 0.76953125, "rewards//std": 0.028493236750364304, "step": 753 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1508, "grad_norm": 5.177443981170654, "kl": 1.1029942650347948, "learning_rate": 9.510532872691623e-07, "loss": 0.0441, "num_tokens": 6516832.0, "reward": 0.73565673828125, "reward_std": 0.008477877825498581, "rewards//mean": 0.73565673828125, "rewards//std": 0.02576165273785591, "step": 754 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.151, "grad_norm": 5.199323654174805, "kl": 1.3697152510285378, "learning_rate": 9.509162632025569e-07, "loss": 0.0548, "num_tokens": 6525520.0, "reward": 0.7672119140625, "reward_std": 0.007089829538017511, "rewards//mean": 0.7672119140625, "rewards//std": 0.0234345942735672, "step": 755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1512, "grad_norm": 10.65485668182373, "kl": 6.130337009206414, "learning_rate": 9.507790575069345e-07, "loss": 0.2452, "num_tokens": 6534128.0, "reward": 0.75823974609375, "reward_std": 0.015302046202123165, "rewards//mean": 0.75823974609375, "rewards//std": 0.030154384672641754, "step": 756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1514, "grad_norm": 6.569500923156738, "kl": 0.9125564694404602, "learning_rate": 9.506416702375617e-07, "loss": 0.0365, "num_tokens": 6542736.0, "reward": 0.73785400390625, "reward_std": 0.005343678407371044, "rewards//mean": 0.73785400390625, "rewards//std": 0.026983682066202164, "step": 757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1516, "grad_norm": 4.671652317047119, "kl": 1.829518985003233, "learning_rate": 9.505041014497779e-07, "loss": 0.0732, "num_tokens": 6551352.0, "reward": 0.75714111328125, "reward_std": 0.007469784002751112, "rewards//mean": 0.75714111328125, "rewards//std": 0.02302626520395279, "step": 758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1518, "grad_norm": 5.878870487213135, "kl": 4.019355835393071, "learning_rate": 9.503663511989962e-07, "loss": 0.1608, "num_tokens": 6560040.0, "reward": 0.7625732421875, "reward_std": 0.010441871359944344, "rewards//mean": 0.7625732421875, "rewards//std": 0.02873421274125576, "step": 759 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.152, "grad_norm": 4.517607688903809, "kl": 3.985830817371607, "learning_rate": 9.502284195407018e-07, "loss": 0.1594, "num_tokens": 6568704.0, "reward": 0.73724365234375, "reward_std": 0.012059698812663555, "rewards//mean": 0.73724365234375, "rewards//std": 0.03146043419837952, "step": 760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1522, "grad_norm": 10.41141128540039, "kl": 4.386840686202049, "learning_rate": 9.500903065304539e-07, "loss": 0.1755, "num_tokens": 6577336.0, "reward": 0.74603271484375, "reward_std": 0.009094847366213799, "rewards//mean": 0.74603271484375, "rewards//std": 0.025272227823734283, "step": 761 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1524, "grad_norm": 5.01835298538208, "kl": 3.469432517886162, "learning_rate": 9.499520122238845e-07, "loss": 0.1388, "num_tokens": 6585944.0, "reward": 0.7529296875, "reward_std": 0.00730510801076889, "rewards//mean": 0.7529296875, "rewards//std": 0.03261526674032211, "step": 762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1526, "grad_norm": 4.169670104980469, "kl": 2.461470915004611, "learning_rate": 9.498135366766982e-07, "loss": 0.0985, "num_tokens": 6594568.0, "reward": 0.75091552734375, "reward_std": 0.0068618846125900745, "rewards//mean": 0.75091552734375, "rewards//std": 0.02700723148882389, "step": 763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1528, "grad_norm": 4.55186128616333, "kl": 1.9192438144236803, "learning_rate": 9.496748799446732e-07, "loss": 0.0768, "num_tokens": 6603064.0, "reward": 0.74798583984375, "reward_std": 0.010374845936894417, "rewards//mean": 0.74798583984375, "rewards//std": 0.025798644870519638, "step": 764 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.153, "grad_norm": 5.042366981506348, "kl": 2.082818355411291, "learning_rate": 9.495360420836602e-07, "loss": 0.0833, "num_tokens": 6611760.0, "reward": 0.75909423828125, "reward_std": 0.010806964710354805, "rewards//mean": 0.75909423828125, "rewards//std": 0.027081677690148354, "step": 765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1532, "grad_norm": 4.896661758422852, "kl": 3.562807273119688, "learning_rate": 9.493970231495834e-07, "loss": 0.1425, "num_tokens": 6620544.0, "reward": 0.784912109375, "reward_std": 0.010713660158216953, "rewards//mean": 0.784912109375, "rewards//std": 0.034392718225717545, "step": 766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1534, "grad_norm": 6.086737155914307, "kl": 3.450151313096285, "learning_rate": 9.492578231984393e-07, "loss": 0.138, "num_tokens": 6629192.0, "reward": 0.7493896484375, "reward_std": 0.009554892778396606, "rewards//mean": 0.7493896484375, "rewards//std": 0.029250076040625572, "step": 767 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1536, "grad_norm": 9.301119804382324, "kl": 4.979688977822661, "learning_rate": 9.491184422862979e-07, "loss": 0.1992, "num_tokens": 6637832.0, "reward": 0.73736572265625, "reward_std": 0.01189520675688982, "rewards//mean": 0.73736572265625, "rewards//std": 0.030771559104323387, "step": 768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1538, "grad_norm": 6.5520405769348145, "kl": 3.4958593603223562, "learning_rate": 9.489788804693015e-07, "loss": 0.1398, "num_tokens": 6646552.0, "reward": 0.7847900390625, "reward_std": 0.015342256985604763, "rewards//mean": 0.7847900390625, "rewards//std": 0.030415115877985954, "step": 769 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.154, "grad_norm": 4.01801061630249, "kl": 1.6592401564121246, "learning_rate": 9.488391378036659e-07, "loss": 0.0664, "num_tokens": 6655176.0, "reward": 0.78173828125, "reward_std": 0.007254729513078928, "rewards//mean": 0.78173828125, "rewards//std": 0.018394500017166138, "step": 770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1542, "grad_norm": 5.3736371994018555, "kl": 2.364370469003916, "learning_rate": 9.486992143456791e-07, "loss": 0.0946, "num_tokens": 6663840.0, "reward": 0.76715087890625, "reward_std": 0.008144975639879704, "rewards//mean": 0.76715087890625, "rewards//std": 0.030509719625115395, "step": 771 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1544, "grad_norm": 4.031811714172363, "kl": 2.198170654475689, "learning_rate": 9.485591101517026e-07, "loss": 0.0879, "num_tokens": 6672440.0, "reward": 0.7484130859375, "reward_std": 0.008436856791377068, "rewards//mean": 0.7484130859375, "rewards//std": 0.021751515567302704, "step": 772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1546, "grad_norm": 3.4090960025787354, "kl": 0.6820312235504389, "learning_rate": 9.4841882527817e-07, "loss": 0.0273, "num_tokens": 6680960.0, "reward": 0.74945068359375, "reward_std": 0.007616454269737005, "rewards//mean": 0.74945068359375, "rewards//std": 0.023113535717129707, "step": 773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1548, "grad_norm": 4.615508079528809, "kl": 1.7933346442878246, "learning_rate": 9.482783597815882e-07, "loss": 0.0717, "num_tokens": 6689760.0, "reward": 0.77838134765625, "reward_std": 0.009177163243293762, "rewards//mean": 0.77838134765625, "rewards//std": 0.028075717389583588, "step": 774 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.155, "grad_norm": 5.0275068283081055, "kl": 2.915667874738574, "learning_rate": 9.481377137185369e-07, "loss": 0.1166, "num_tokens": 6698392.0, "reward": 0.785400390625, "reward_std": 0.011071623302996159, "rewards//mean": 0.785400390625, "rewards//std": 0.02442800998687744, "step": 775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1552, "grad_norm": 4.190218925476074, "kl": 2.2015086244791746, "learning_rate": 9.479968871456679e-07, "loss": 0.0881, "num_tokens": 6707040.0, "reward": 0.7630615234375, "reward_std": 0.007179578300565481, "rewards//mean": 0.7630615234375, "rewards//std": 0.026376372203230858, "step": 776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1554, "grad_norm": 5.644638538360596, "kl": 1.7162752319127321, "learning_rate": 9.478558801197064e-07, "loss": 0.0687, "num_tokens": 6715672.0, "reward": 0.7744140625, "reward_std": 0.007982452400028706, "rewards//mean": 0.7744140625, "rewards//std": 0.019722526893019676, "step": 777 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1556, "grad_norm": 4.0156941413879395, "kl": 2.7852271664887667, "learning_rate": 9.4771469269745e-07, "loss": 0.1114, "num_tokens": 6724312.0, "reward": 0.7691650390625, "reward_std": 0.010146573185920715, "rewards//mean": 0.7691650390625, "rewards//std": 0.027671460062265396, "step": 778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1558, "grad_norm": 7.933249473571777, "kl": 4.763269757851958, "learning_rate": 9.475733249357688e-07, "loss": 0.1905, "num_tokens": 6732968.0, "reward": 0.77203369140625, "reward_std": 0.013013869524002075, "rewards//mean": 0.77203369140625, "rewards//std": 0.027981745079159737, "step": 779 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.156, "grad_norm": 12.175159454345703, "kl": 5.108350550755858, "learning_rate": 9.474317768916059e-07, "loss": 0.2043, "num_tokens": 6741616.0, "reward": 0.76422119140625, "reward_std": 0.009025199338793755, "rewards//mean": 0.76422119140625, "rewards//std": 0.022524451836943626, "step": 780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1562, "grad_norm": 9.010479927062988, "kl": 2.115995079278946, "learning_rate": 9.472900486219768e-07, "loss": 0.0846, "num_tokens": 6750248.0, "reward": 0.77886962890625, "reward_std": 0.011248920112848282, "rewards//mean": 0.77886962890625, "rewards//std": 0.030533524230122566, "step": 781 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1564, "grad_norm": 5.800589561462402, "kl": 2.6221635565161705, "learning_rate": 9.471481401839696e-07, "loss": 0.1049, "num_tokens": 6758784.0, "reward": 0.7642822265625, "reward_std": 0.011129805818200111, "rewards//mean": 0.7642822265625, "rewards//std": 0.02669582888484001, "step": 782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1566, "grad_norm": 4.211091995239258, "kl": 4.653603246435523, "learning_rate": 9.470060516347449e-07, "loss": 0.1861, "num_tokens": 6767448.0, "reward": 0.78985595703125, "reward_std": 0.014145842753350735, "rewards//mean": 0.78985595703125, "rewards//std": 0.020530959591269493, "step": 783 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1568, "grad_norm": 4.587401866912842, "kl": 3.3550094813108444, "learning_rate": 9.468637830315362e-07, "loss": 0.1342, "num_tokens": 6776224.0, "reward": 0.7330322265625, "reward_std": 0.011062645353376865, "rewards//mean": 0.7330322265625, "rewards//std": 0.04221653565764427, "step": 784 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.157, "grad_norm": 5.721545696258545, "kl": 3.0369136724621058, "learning_rate": 9.467213344316491e-07, "loss": 0.1215, "num_tokens": 6784832.0, "reward": 0.7496337890625, "reward_std": 0.008115509524941444, "rewards//mean": 0.7496337890625, "rewards//std": 0.03495047613978386, "step": 785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1572, "grad_norm": 6.887928009033203, "kl": 3.6168970745056868, "learning_rate": 9.465787058924619e-07, "loss": 0.1447, "num_tokens": 6793464.0, "reward": 0.78460693359375, "reward_std": 0.006885099224746227, "rewards//mean": 0.78460693359375, "rewards//std": 0.02213943377137184, "step": 786 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1574, "grad_norm": 4.287591457366943, "kl": 3.2999197766184807, "learning_rate": 9.464358974714252e-07, "loss": 0.132, "num_tokens": 6801992.0, "reward": 0.77069091796875, "reward_std": 0.013432151637971401, "rewards//mean": 0.77069091796875, "rewards//std": 0.029137752950191498, "step": 787 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1576, "grad_norm": 6.390154838562012, "kl": 2.886853411793709, "learning_rate": 9.462929092260628e-07, "loss": 0.1155, "num_tokens": 6810688.0, "reward": 0.74188232421875, "reward_std": 0.013014253228902817, "rewards//mean": 0.74188232421875, "rewards//std": 0.03236407786607742, "step": 788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1578, "grad_norm": 7.804890155792236, "kl": 5.38600605353713, "learning_rate": 9.461497412139696e-07, "loss": 0.2154, "num_tokens": 6819336.0, "reward": 0.75628662109375, "reward_std": 0.014761541038751602, "rewards//mean": 0.75628662109375, "rewards//std": 0.03839721903204918, "step": 789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.158, "grad_norm": 11.392706871032715, "kl": 5.6631733775138855, "learning_rate": 9.460063934928141e-07, "loss": 0.2265, "num_tokens": 6828088.0, "reward": 0.7474365234375, "reward_std": 0.011669349856674671, "rewards//mean": 0.7474365234375, "rewards//std": 0.028933709487318993, "step": 790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1582, "grad_norm": 6.763251304626465, "kl": 4.461639614775777, "learning_rate": 9.458628661203366e-07, "loss": 0.1785, "num_tokens": 6836808.0, "reward": 0.7615966796875, "reward_std": 0.010639270767569542, "rewards//mean": 0.7615966796875, "rewards//std": 0.014894389547407627, "step": 791 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1584, "grad_norm": 4.676641941070557, "kl": 3.748870886862278, "learning_rate": 9.4571915915435e-07, "loss": 0.15, "num_tokens": 6845400.0, "reward": 0.75372314453125, "reward_std": 0.007258172146975994, "rewards//mean": 0.75372314453125, "rewards//std": 0.017859095707535744, "step": 792 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1586, "grad_norm": 4.426313877105713, "kl": 3.1063752956688404, "learning_rate": 9.455752726527392e-07, "loss": 0.1243, "num_tokens": 6854096.0, "reward": 0.7286376953125, "reward_std": 0.011803104542195797, "rewards//mean": 0.7286376953125, "rewards//std": 0.030851947143673897, "step": 793 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1588, "grad_norm": 3.786665916442871, "kl": 1.685317488387227, "learning_rate": 9.454312066734622e-07, "loss": 0.0674, "num_tokens": 6862656.0, "reward": 0.72662353515625, "reward_std": 0.0077147455886006355, "rewards//mean": 0.72662353515625, "rewards//std": 0.020392615348100662, "step": 794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.159, "grad_norm": 4.603367328643799, "kl": 1.2162437960505486, "learning_rate": 9.452869612745483e-07, "loss": 0.0486, "num_tokens": 6871248.0, "reward": 0.7705078125, "reward_std": 0.008626492694020271, "rewards//mean": 0.7705078125, "rewards//std": 0.021687744185328484, "step": 795 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1592, "grad_norm": 4.271348476409912, "kl": 1.668197050690651, "learning_rate": 9.451425365140994e-07, "loss": 0.0667, "num_tokens": 6879968.0, "reward": 0.74993896484375, "reward_std": 0.006746210157871246, "rewards//mean": 0.74993896484375, "rewards//std": 0.030317597091197968, "step": 796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1594, "grad_norm": 6.6405510902404785, "kl": 3.7507582679390907, "learning_rate": 9.449979324502903e-07, "loss": 0.15, "num_tokens": 6888536.0, "reward": 0.77099609375, "reward_std": 0.01573047786951065, "rewards//mean": 0.77099609375, "rewards//std": 0.024084772914648056, "step": 797 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1596, "grad_norm": 4.867390155792236, "kl": 2.5980178639292717, "learning_rate": 9.448531491413672e-07, "loss": 0.1039, "num_tokens": 6897080.0, "reward": 0.77197265625, "reward_std": 0.01075395755469799, "rewards//mean": 0.77197265625, "rewards//std": 0.02174908109009266, "step": 798 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1598, "grad_norm": 3.9256911277770996, "kl": 0.747510327026248, "learning_rate": 9.447081866456487e-07, "loss": 0.0299, "num_tokens": 6905712.0, "reward": 0.75689697265625, "reward_std": 0.0040665497072041035, "rewards//mean": 0.75689697265625, "rewards//std": 0.026224639266729355, "step": 799 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.16, "grad_norm": 8.939791679382324, "kl": 2.05123102478683, "learning_rate": 9.445630450215259e-07, "loss": 0.082, "num_tokens": 6914304.0, "reward": 0.7764892578125, "reward_std": 0.009935218840837479, "rewards//mean": 0.7764892578125, "rewards//std": 0.023566000163555145, "step": 800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1602, "grad_norm": 7.597315311431885, "kl": 1.4521824270486832, "learning_rate": 9.444177243274617e-07, "loss": 0.0581, "num_tokens": 6922960.0, "reward": 0.7889404296875, "reward_std": 0.007975172251462936, "rewards//mean": 0.7889404296875, "rewards//std": 0.02867092378437519, "step": 801 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1604, "grad_norm": 4.933373928070068, "kl": 1.937702227383852, "learning_rate": 9.442722246219913e-07, "loss": 0.0775, "num_tokens": 6931632.0, "reward": 0.73895263671875, "reward_std": 0.006113705225288868, "rewards//mean": 0.73895263671875, "rewards//std": 0.03296652063727379, "step": 802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1606, "grad_norm": 4.05463981628418, "kl": 2.27868514880538, "learning_rate": 9.441265459637219e-07, "loss": 0.0911, "num_tokens": 6940272.0, "reward": 0.77081298828125, "reward_std": 0.00799493957310915, "rewards//mean": 0.77081298828125, "rewards//std": 0.025553978979587555, "step": 803 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1608, "grad_norm": 5.101175785064697, "kl": 4.256691545248032, "learning_rate": 9.43980688411333e-07, "loss": 0.1703, "num_tokens": 6948856.0, "reward": 0.74481201171875, "reward_std": 0.019318178296089172, "rewards//mean": 0.74481201171875, "rewards//std": 0.03434465453028679, "step": 804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.161, "grad_norm": 5.4669952392578125, "kl": 2.1037301030009985, "learning_rate": 9.438346520235758e-07, "loss": 0.0841, "num_tokens": 6957592.0, "reward": 0.75518798828125, "reward_std": 0.008407503366470337, "rewards//mean": 0.75518798828125, "rewards//std": 0.023105019703507423, "step": 805 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1612, "grad_norm": 5.856110572814941, "kl": 3.3386458940804005, "learning_rate": 9.436884368592739e-07, "loss": 0.1335, "num_tokens": 6966240.0, "reward": 0.775390625, "reward_std": 0.010535828769207, "rewards//mean": 0.775390625, "rewards//std": 0.02530582621693611, "step": 806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1614, "grad_norm": 5.265435218811035, "kl": 2.4264450185000896, "learning_rate": 9.435420429773227e-07, "loss": 0.0971, "num_tokens": 6974904.0, "reward": 0.7279052734375, "reward_std": 0.01252413634210825, "rewards//mean": 0.7279052734375, "rewards//std": 0.03498857095837593, "step": 807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1616, "grad_norm": 5.059149265289307, "kl": 1.7861047461628914, "learning_rate": 9.433954704366896e-07, "loss": 0.0714, "num_tokens": 6983504.0, "reward": 0.77508544921875, "reward_std": 0.007414050400257111, "rewards//mean": 0.77508544921875, "rewards//std": 0.023144295439124107, "step": 808 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1618, "grad_norm": 4.534353733062744, "kl": 3.3108605053275824, "learning_rate": 9.43248719296414e-07, "loss": 0.1324, "num_tokens": 6992216.0, "reward": 0.756591796875, "reward_std": 0.011271260678768158, "rewards//mean": 0.756591796875, "rewards//std": 0.02740016020834446, "step": 809 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.162, "grad_norm": 5.283889293670654, "kl": 3.1531424652785063, "learning_rate": 9.431017896156073e-07, "loss": 0.1261, "num_tokens": 7000816.0, "reward": 0.7689208984375, "reward_std": 0.015889810398221016, "rewards//mean": 0.7689208984375, "rewards//std": 0.03676409646868706, "step": 810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1622, "grad_norm": 5.881231307983398, "kl": 1.6919131707400084, "learning_rate": 9.429546814534528e-07, "loss": 0.0677, "num_tokens": 7009480.0, "reward": 0.7716064453125, "reward_std": 0.005492859985679388, "rewards//mean": 0.7716064453125, "rewards//std": 0.020552054047584534, "step": 811 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1624, "grad_norm": 9.305164337158203, "kl": 4.946976518258452, "learning_rate": 9.428073948692054e-07, "loss": 0.1979, "num_tokens": 7018136.0, "reward": 0.73931884765625, "reward_std": 0.01171068660914898, "rewards//mean": 0.73931884765625, "rewards//std": 0.02789396606385708, "step": 812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1626, "grad_norm": 5.397275447845459, "kl": 4.610825087875128, "learning_rate": 9.426599299221924e-07, "loss": 0.1844, "num_tokens": 7026824.0, "reward": 0.75689697265625, "reward_std": 0.016051501035690308, "rewards//mean": 0.75689697265625, "rewards//std": 0.03400936350226402, "step": 813 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1628, "grad_norm": 6.010089874267578, "kl": 3.0963895712047815, "learning_rate": 9.425122866718127e-07, "loss": 0.1239, "num_tokens": 7035488.0, "reward": 0.74285888671875, "reward_std": 0.009649118408560753, "rewards//mean": 0.74285888671875, "rewards//std": 0.02659890055656433, "step": 814 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.163, "grad_norm": 5.974343776702881, "kl": 2.298018941655755, "learning_rate": 9.423644651775368e-07, "loss": 0.0919, "num_tokens": 7044072.0, "reward": 0.7584228515625, "reward_std": 0.013415565714240074, "rewards//mean": 0.7584228515625, "rewards//std": 0.023501677438616753, "step": 815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1632, "grad_norm": 10.788091659545898, "kl": 4.28102938272059, "learning_rate": 9.422164654989071e-07, "loss": 0.1712, "num_tokens": 7052744.0, "reward": 0.76611328125, "reward_std": 0.011416466906666756, "rewards//mean": 0.76611328125, "rewards//std": 0.027370311319828033, "step": 816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1634, "grad_norm": 5.269587516784668, "kl": 1.0280618537217379, "learning_rate": 9.420682876955381e-07, "loss": 0.0411, "num_tokens": 7061384.0, "reward": 0.7451171875, "reward_std": 0.00575895793735981, "rewards//mean": 0.7451171875, "rewards//std": 0.02525792457163334, "step": 817 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1636, "grad_norm": 3.6746578216552734, "kl": 3.659264462068677, "learning_rate": 9.419199318271156e-07, "loss": 0.1464, "num_tokens": 7070008.0, "reward": 0.747314453125, "reward_std": 0.010371357202529907, "rewards//mean": 0.747314453125, "rewards//std": 0.028897326439619064, "step": 818 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1638, "grad_norm": 6.09009313583374, "kl": 4.993275761604309, "learning_rate": 9.417713979533974e-07, "loss": 0.1997, "num_tokens": 7078680.0, "reward": 0.77337646484375, "reward_std": 0.012450158596038818, "rewards//mean": 0.77337646484375, "rewards//std": 0.023514797911047935, "step": 819 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.164, "grad_norm": 6.171932697296143, "kl": 2.3480502869933844, "learning_rate": 9.41622686134213e-07, "loss": 0.0939, "num_tokens": 7087336.0, "reward": 0.73858642578125, "reward_std": 0.004920503124594688, "rewards//mean": 0.73858642578125, "rewards//std": 0.026958424597978592, "step": 820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1642, "grad_norm": 4.886020660400391, "kl": 4.0925056375563145, "learning_rate": 9.414737964294634e-07, "loss": 0.1637, "num_tokens": 7096016.0, "reward": 0.75732421875, "reward_std": 0.01674531400203705, "rewards//mean": 0.75732421875, "rewards//std": 0.026461461558938026, "step": 821 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1644, "grad_norm": 12.292215347290039, "kl": 5.2495632860809565, "learning_rate": 9.413247288991215e-07, "loss": 0.21, "num_tokens": 7104736.0, "reward": 0.724365234375, "reward_std": 0.00853973813354969, "rewards//mean": 0.724365234375, "rewards//std": 0.031062457710504532, "step": 822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1646, "grad_norm": 6.820827007293701, "kl": 3.6908694114536047, "learning_rate": 9.411754836032314e-07, "loss": 0.1476, "num_tokens": 7113368.0, "reward": 0.757080078125, "reward_std": 0.007114241365343332, "rewards//mean": 0.757080078125, "rewards//std": 0.025457311421632767, "step": 823 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1648, "grad_norm": 5.6415324211120605, "kl": 1.4121773652732372, "learning_rate": 9.410260606019094e-07, "loss": 0.0565, "num_tokens": 7122128.0, "reward": 0.77734375, "reward_std": 0.00641645397990942, "rewards//mean": 0.77734375, "rewards//std": 0.02536318637430668, "step": 824 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.165, "grad_norm": 4.019225120544434, "kl": 1.3440431728959084, "learning_rate": 9.408764599553428e-07, "loss": 0.0538, "num_tokens": 7130776.0, "reward": 0.75555419921875, "reward_std": 0.007369856350123882, "rewards//mean": 0.75555419921875, "rewards//std": 0.01826639473438263, "step": 825 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1652, "grad_norm": 4.040704727172852, "kl": 1.9375916495919228, "learning_rate": 9.40726681723791e-07, "loss": 0.0775, "num_tokens": 7139528.0, "reward": 0.7393798828125, "reward_std": 0.007135326508432627, "rewards//mean": 0.7393798828125, "rewards//std": 0.032818615436553955, "step": 826 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1654, "grad_norm": 4.428711891174316, "kl": 2.506573861464858, "learning_rate": 9.405767259675844e-07, "loss": 0.1003, "num_tokens": 7148128.0, "reward": 0.72418212890625, "reward_std": 0.009128805249929428, "rewards//mean": 0.72418212890625, "rewards//std": 0.046274978667497635, "step": 827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1656, "grad_norm": 6.2188568115234375, "kl": 1.8739633075892925, "learning_rate": 9.404265927471253e-07, "loss": 0.075, "num_tokens": 7156728.0, "reward": 0.7354736328125, "reward_std": 0.008282147347927094, "rewards//mean": 0.7354736328125, "rewards//std": 0.03260389342904091, "step": 828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1658, "grad_norm": 4.146193981170654, "kl": 0.5284234452992678, "learning_rate": 9.402762821228874e-07, "loss": 0.0211, "num_tokens": 7165408.0, "reward": 0.76214599609375, "reward_std": 0.005288444459438324, "rewards//mean": 0.76214599609375, "rewards//std": 0.02323112264275551, "step": 829 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.166, "grad_norm": 3.5680792331695557, "kl": 2.4490291960537434, "learning_rate": 9.401257941554156e-07, "loss": 0.098, "num_tokens": 7174016.0, "reward": 0.76226806640625, "reward_std": 0.009816572070121765, "rewards//mean": 0.76226806640625, "rewards//std": 0.03335590288043022, "step": 830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1662, "grad_norm": 4.464192867279053, "kl": 0.9390498735010624, "learning_rate": 9.399751289053266e-07, "loss": 0.0376, "num_tokens": 7182656.0, "reward": 0.76025390625, "reward_std": 0.0055123730562627316, "rewards//mean": 0.76025390625, "rewards//std": 0.021389752626419067, "step": 831 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1664, "grad_norm": 3.8422210216522217, "kl": 1.9241707921028137, "learning_rate": 9.398242864333083e-07, "loss": 0.077, "num_tokens": 7191360.0, "reward": 0.76580810546875, "reward_std": 0.0072711799293756485, "rewards//mean": 0.76580810546875, "rewards//std": 0.03402137756347656, "step": 832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1666, "grad_norm": 5.312897205352783, "kl": 3.2794720102101564, "learning_rate": 9.396732668001199e-07, "loss": 0.1312, "num_tokens": 7200080.0, "reward": 0.7540283203125, "reward_std": 0.007833273150026798, "rewards//mean": 0.7540283203125, "rewards//std": 0.01580943912267685, "step": 833 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1668, "grad_norm": 3.0219478607177734, "kl": 0.4920041970908642, "learning_rate": 9.395220700665922e-07, "loss": 0.0197, "num_tokens": 7208816.0, "reward": 0.7708740234375, "reward_std": 0.004750157240778208, "rewards//mean": 0.7708740234375, "rewards//std": 0.026108719408512115, "step": 834 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.167, "grad_norm": 6.649507522583008, "kl": 2.959937596693635, "learning_rate": 9.393706962936274e-07, "loss": 0.1184, "num_tokens": 7217488.0, "reward": 0.76373291015625, "reward_std": 0.008829772472381592, "rewards//mean": 0.76373291015625, "rewards//std": 0.029438046738505363, "step": 835 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1672, "grad_norm": 3.517529010772705, "kl": 0.6590344868600368, "learning_rate": 9.392191455421987e-07, "loss": 0.0264, "num_tokens": 7226024.0, "reward": 0.76226806640625, "reward_std": 0.004965063184499741, "rewards//mean": 0.76226806640625, "rewards//std": 0.022465234622359276, "step": 836 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1674, "grad_norm": 4.816415309906006, "kl": 2.957753771916032, "learning_rate": 9.390674178733507e-07, "loss": 0.1183, "num_tokens": 7234672.0, "reward": 0.75238037109375, "reward_std": 0.008637012913823128, "rewards//mean": 0.75238037109375, "rewards//std": 0.030326582491397858, "step": 837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1676, "grad_norm": 7.869420528411865, "kl": 4.866269508376718, "learning_rate": 9.389155133481992e-07, "loss": 0.1947, "num_tokens": 7243248.0, "reward": 0.73876953125, "reward_std": 0.009045137092471123, "rewards//mean": 0.73876953125, "rewards//std": 0.030556384474039078, "step": 838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1678, "grad_norm": 4.110578536987305, "kl": 1.7108943182975054, "learning_rate": 9.387634320279314e-07, "loss": 0.0684, "num_tokens": 7252024.0, "reward": 0.744384765625, "reward_std": 0.006312161218374968, "rewards//mean": 0.744384765625, "rewards//std": 0.026244329288601875, "step": 839 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.168, "grad_norm": 5.340089797973633, "kl": 3.5931700356304646, "learning_rate": 9.386111739738056e-07, "loss": 0.1437, "num_tokens": 7260632.0, "reward": 0.75054931640625, "reward_std": 0.008358034305274487, "rewards//mean": 0.75054931640625, "rewards//std": 0.03334864228963852, "step": 840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1682, "grad_norm": 4.9973931312561035, "kl": 3.812736565247178, "learning_rate": 9.384587392471514e-07, "loss": 0.1525, "num_tokens": 7269232.0, "reward": 0.7659912109375, "reward_std": 0.01163836196064949, "rewards//mean": 0.7659912109375, "rewards//std": 0.022742237895727158, "step": 841 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1684, "grad_norm": 5.224663734436035, "kl": 2.3228061739355326, "learning_rate": 9.383061279093696e-07, "loss": 0.0929, "num_tokens": 7277888.0, "reward": 0.76190185546875, "reward_std": 0.008733944036066532, "rewards//mean": 0.76190185546875, "rewards//std": 0.014934353530406952, "step": 842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1686, "grad_norm": 6.6156439781188965, "kl": 2.879060560837388, "learning_rate": 9.381533400219317e-07, "loss": 0.1152, "num_tokens": 7286528.0, "reward": 0.77838134765625, "reward_std": 0.008324716240167618, "rewards//mean": 0.77838134765625, "rewards//std": 0.02941541187465191, "step": 843 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1688, "grad_norm": 4.613117694854736, "kl": 2.078267527744174, "learning_rate": 9.38000375646381e-07, "loss": 0.0831, "num_tokens": 7295168.0, "reward": 0.7542724609375, "reward_std": 0.008011859841644764, "rewards//mean": 0.7542724609375, "rewards//std": 0.020312009379267693, "step": 844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.169, "grad_norm": 5.4162797927856445, "kl": 2.765382260084152, "learning_rate": 9.378472348443314e-07, "loss": 0.1106, "num_tokens": 7303800.0, "reward": 0.74188232421875, "reward_std": 0.006411589216440916, "rewards//mean": 0.74188232421875, "rewards//std": 0.021545499563217163, "step": 845 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1692, "grad_norm": 4.39764928817749, "kl": 4.487566510215402, "learning_rate": 9.376939176774677e-07, "loss": 0.1795, "num_tokens": 7312504.0, "reward": 0.77227783203125, "reward_std": 0.013200690969824791, "rewards//mean": 0.77227783203125, "rewards//std": 0.027863016352057457, "step": 846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1694, "grad_norm": 12.082962989807129, "kl": 4.153339693322778, "learning_rate": 9.375404242075466e-07, "loss": 0.1661, "num_tokens": 7321176.0, "reward": 0.74102783203125, "reward_std": 0.005838182754814625, "rewards//mean": 0.74102783203125, "rewards//std": 0.02251705899834633, "step": 847 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1696, "grad_norm": 4.346368312835693, "kl": 2.3379539754241705, "learning_rate": 9.373867544963948e-07, "loss": 0.0935, "num_tokens": 7329792.0, "reward": 0.776123046875, "reward_std": 0.009924739599227905, "rewards//mean": 0.776123046875, "rewards//std": 0.024348560720682144, "step": 848 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1698, "grad_norm": 4.499940872192383, "kl": 2.722176406532526, "learning_rate": 9.372329086059107e-07, "loss": 0.1089, "num_tokens": 7338440.0, "reward": 0.77099609375, "reward_std": 0.007846756838262081, "rewards//mean": 0.77099609375, "rewards//std": 0.021882306784391403, "step": 849 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.17, "grad_norm": 4.038485527038574, "kl": 1.8677042871713638, "learning_rate": 9.370788865980632e-07, "loss": 0.0747, "num_tokens": 7347016.0, "reward": 0.76861572265625, "reward_std": 0.008194370195269585, "rewards//mean": 0.76861572265625, "rewards//std": 0.025379817932844162, "step": 850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1702, "grad_norm": 5.125847339630127, "kl": 1.8140365444123745, "learning_rate": 9.369246885348925e-07, "loss": 0.0726, "num_tokens": 7355664.0, "reward": 0.7032470703125, "reward_std": 0.006922622211277485, "rewards//mean": 0.7032470703125, "rewards//std": 0.03520766645669937, "step": 851 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1704, "grad_norm": 4.963730335235596, "kl": 3.619083819910884, "learning_rate": 9.367703144785095e-07, "loss": 0.1448, "num_tokens": 7364392.0, "reward": 0.76422119140625, "reward_std": 0.010131246410310268, "rewards//mean": 0.76422119140625, "rewards//std": 0.0317067876458168, "step": 852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1706, "grad_norm": 4.636470794677734, "kl": 3.3408317267894745, "learning_rate": 9.366157644910959e-07, "loss": 0.1336, "num_tokens": 7373032.0, "reward": 0.76007080078125, "reward_std": 0.010227615013718605, "rewards//mean": 0.76007080078125, "rewards//std": 0.030487382784485817, "step": 853 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1708, "grad_norm": 4.1317219734191895, "kl": 0.7655187211930752, "learning_rate": 9.364610386349047e-07, "loss": 0.0306, "num_tokens": 7381688.0, "reward": 0.76336669921875, "reward_std": 0.005188517272472382, "rewards//mean": 0.76336669921875, "rewards//std": 0.017850618809461594, "step": 854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.171, "grad_norm": 6.155958652496338, "kl": 4.210917366668582, "learning_rate": 9.363061369722594e-07, "loss": 0.1684, "num_tokens": 7390344.0, "reward": 0.7872314453125, "reward_std": 0.015684831887483597, "rewards//mean": 0.7872314453125, "rewards//std": 0.027795907109975815, "step": 855 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1712, "grad_norm": 3.3234474658966064, "kl": 0.5247700996696949, "learning_rate": 9.361510595655544e-07, "loss": 0.021, "num_tokens": 7398960.0, "reward": 0.7786865234375, "reward_std": 0.004236909560859203, "rewards//mean": 0.7786865234375, "rewards//std": 0.02105558104813099, "step": 856 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1714, "grad_norm": 7.604945182800293, "kl": 4.309569004923105, "learning_rate": 9.359958064772546e-07, "loss": 0.1724, "num_tokens": 7407632.0, "reward": 0.73480224609375, "reward_std": 0.007704409770667553, "rewards//mean": 0.73480224609375, "rewards//std": 0.027944933623075485, "step": 857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1716, "grad_norm": 9.44655990600586, "kl": 2.2118360735476017, "learning_rate": 9.35840377769896e-07, "loss": 0.0885, "num_tokens": 7416224.0, "reward": 0.7835693359375, "reward_std": 0.010356249287724495, "rewards//mean": 0.7835693359375, "rewards//std": 0.020383426919579506, "step": 858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1718, "grad_norm": 4.0900163650512695, "kl": 3.979547144845128, "learning_rate": 9.356847735060856e-07, "loss": 0.1592, "num_tokens": 7424872.0, "reward": 0.78057861328125, "reward_std": 0.007768874987959862, "rewards//mean": 0.78057861328125, "rewards//std": 0.021693937480449677, "step": 859 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.172, "grad_norm": 5.1128830909729, "kl": 2.8700168412178755, "learning_rate": 9.355289937485004e-07, "loss": 0.1148, "num_tokens": 7433544.0, "reward": 0.77447509765625, "reward_std": 0.008934141136705875, "rewards//mean": 0.77447509765625, "rewards//std": 0.028673233464360237, "step": 860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1722, "grad_norm": 4.9949870109558105, "kl": 3.751866629347205, "learning_rate": 9.353730385598886e-07, "loss": 0.1501, "num_tokens": 7442144.0, "reward": 0.76397705078125, "reward_std": 0.010951785370707512, "rewards//mean": 0.76397705078125, "rewards//std": 0.02761087566614151, "step": 861 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1724, "grad_norm": 5.703150272369385, "kl": 3.635123133659363, "learning_rate": 9.35216908003069e-07, "loss": 0.1454, "num_tokens": 7450776.0, "reward": 0.7626953125, "reward_std": 0.010348414070904255, "rewards//mean": 0.7626953125, "rewards//std": 0.02896534651517868, "step": 862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1726, "grad_norm": 4.669232368469238, "kl": 4.9712036326527596, "learning_rate": 9.350606021409308e-07, "loss": 0.1988, "num_tokens": 7459440.0, "reward": 0.78179931640625, "reward_std": 0.012088283896446228, "rewards//mean": 0.78179931640625, "rewards//std": 0.021719740703701973, "step": 863 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1728, "grad_norm": 3.92879319190979, "kl": 2.666841896250844, "learning_rate": 9.349041210364341e-07, "loss": 0.1067, "num_tokens": 7468160.0, "reward": 0.7828369140625, "reward_std": 0.009088682942092419, "rewards//mean": 0.7828369140625, "rewards//std": 0.021312827244400978, "step": 864 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.173, "grad_norm": 4.29932975769043, "kl": 2.829136600717902, "learning_rate": 9.347474647526095e-07, "loss": 0.1132, "num_tokens": 7476752.0, "reward": 0.77197265625, "reward_std": 0.007671776227653027, "rewards//mean": 0.77197265625, "rewards//std": 0.015032478608191013, "step": 865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1732, "grad_norm": 5.589138984680176, "kl": 2.3940916135907173, "learning_rate": 9.34590633352558e-07, "loss": 0.0958, "num_tokens": 7485448.0, "reward": 0.77569580078125, "reward_std": 0.01034168154001236, "rewards//mean": 0.77569580078125, "rewards//std": 0.02812366373836994, "step": 866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1734, "grad_norm": 11.367834091186523, "kl": 4.466622106730938, "learning_rate": 9.344336268994515e-07, "loss": 0.1787, "num_tokens": 7494104.0, "reward": 0.7293701171875, "reward_std": 0.00977625884115696, "rewards//mean": 0.7293701171875, "rewards//std": 0.023882482200860977, "step": 867 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1736, "grad_norm": 7.060291290283203, "kl": 3.5269526708871126, "learning_rate": 9.342764454565319e-07, "loss": 0.1411, "num_tokens": 7502752.0, "reward": 0.7628173828125, "reward_std": 0.008809993974864483, "rewards//mean": 0.7628173828125, "rewards//std": 0.0270316731184721, "step": 868 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1738, "grad_norm": 4.928085803985596, "kl": 2.0577304176986217, "learning_rate": 9.341190890871121e-07, "loss": 0.0823, "num_tokens": 7511400.0, "reward": 0.73443603515625, "reward_std": 0.008873803541064262, "rewards//mean": 0.73443603515625, "rewards//std": 0.028051983565092087, "step": 869 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.174, "grad_norm": 4.21465539932251, "kl": 2.301447683945298, "learning_rate": 9.339615578545752e-07, "loss": 0.0921, "num_tokens": 7520040.0, "reward": 0.7689208984375, "reward_std": 0.01025544386357069, "rewards//mean": 0.7689208984375, "rewards//std": 0.02152203768491745, "step": 870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1742, "grad_norm": 5.656636714935303, "kl": 4.707288537174463, "learning_rate": 9.338038518223745e-07, "loss": 0.1883, "num_tokens": 7528672.0, "reward": 0.77557373046875, "reward_std": 0.011117503046989441, "rewards//mean": 0.77557373046875, "rewards//std": 0.019021494314074516, "step": 871 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1744, "grad_norm": 6.952838897705078, "kl": 3.672704292461276, "learning_rate": 9.336459710540343e-07, "loss": 0.1469, "num_tokens": 7537480.0, "reward": 0.7584228515625, "reward_std": 0.01173834316432476, "rewards//mean": 0.7584228515625, "rewards//std": 0.027789371088147163, "step": 872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1746, "grad_norm": 3.4130585193634033, "kl": 0.8612559027969837, "learning_rate": 9.334879156131488e-07, "loss": 0.0345, "num_tokens": 7546136.0, "reward": 0.78509521484375, "reward_std": 0.0066023776307702065, "rewards//mean": 0.78509521484375, "rewards//std": 0.014854062348604202, "step": 873 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1748, "grad_norm": 3.6809604167938232, "kl": 2.227695981040597, "learning_rate": 9.333296855633827e-07, "loss": 0.0891, "num_tokens": 7554760.0, "reward": 0.77325439453125, "reward_std": 0.006009181495755911, "rewards//mean": 0.77325439453125, "rewards//std": 0.01577049307525158, "step": 874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.175, "grad_norm": 6.711003303527832, "kl": 0.94081418402493, "learning_rate": 9.331712809684711e-07, "loss": 0.0376, "num_tokens": 7563360.0, "reward": 0.78240966796875, "reward_std": 0.006355063058435917, "rewards//mean": 0.78240966796875, "rewards//std": 0.02656359225511551, "step": 875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1752, "grad_norm": 4.049739360809326, "kl": 2.161626299843192, "learning_rate": 9.330127018922193e-07, "loss": 0.0865, "num_tokens": 7572032.0, "reward": 0.75213623046875, "reward_std": 0.008111530914902687, "rewards//mean": 0.75213623046875, "rewards//std": 0.024724205955863, "step": 876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1754, "grad_norm": 4.6504011154174805, "kl": 2.31799946911633, "learning_rate": 9.32853948398503e-07, "loss": 0.0927, "num_tokens": 7580640.0, "reward": 0.73931884765625, "reward_std": 0.008125009015202522, "rewards//mean": 0.73931884765625, "rewards//std": 0.031251877546310425, "step": 877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1756, "grad_norm": 4.644308567047119, "kl": 3.264233808964491, "learning_rate": 9.32695020551268e-07, "loss": 0.1306, "num_tokens": 7589144.0, "reward": 0.72784423828125, "reward_std": 0.007564897648990154, "rewards//mean": 0.72784423828125, "rewards//std": 0.029049299657344818, "step": 878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1758, "grad_norm": 8.128149032592773, "kl": 2.8776080328971148, "learning_rate": 9.325359184145305e-07, "loss": 0.1151, "num_tokens": 7597784.0, "reward": 0.74053955078125, "reward_std": 0.005500072613358498, "rewards//mean": 0.74053955078125, "rewards//std": 0.022518403828144073, "step": 879 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.176, "grad_norm": 6.089498519897461, "kl": 3.266717841848731, "learning_rate": 9.323766420523767e-07, "loss": 0.1307, "num_tokens": 7606448.0, "reward": 0.75592041015625, "reward_std": 0.008798999711871147, "rewards//mean": 0.75592041015625, "rewards//std": 0.022249925881624222, "step": 880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1762, "grad_norm": 7.295170307159424, "kl": 4.568239964544773, "learning_rate": 9.322171915289633e-07, "loss": 0.1827, "num_tokens": 7615096.0, "reward": 0.74615478515625, "reward_std": 0.010985281318426132, "rewards//mean": 0.74615478515625, "rewards//std": 0.023318275809288025, "step": 881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1764, "grad_norm": 12.643634796142578, "kl": 8.211217939853668, "learning_rate": 9.320575669085169e-07, "loss": 0.3284, "num_tokens": 7623656.0, "reward": 0.7593994140625, "reward_std": 0.01920841634273529, "rewards//mean": 0.7593994140625, "rewards//std": 0.0324493832886219, "step": 882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1766, "grad_norm": 8.465526580810547, "kl": 4.972064973786473, "learning_rate": 9.31897768255334e-07, "loss": 0.1989, "num_tokens": 7632288.0, "reward": 0.74188232421875, "reward_std": 0.007918823510408401, "rewards//mean": 0.74188232421875, "rewards//std": 0.033590611070394516, "step": 883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1768, "grad_norm": 6.798526763916016, "kl": 3.7168193999677896, "learning_rate": 9.317377956337818e-07, "loss": 0.1487, "num_tokens": 7640848.0, "reward": 0.72265625, "reward_std": 0.009313051588833332, "rewards//mean": 0.72265625, "rewards//std": 0.02570466510951519, "step": 884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.177, "grad_norm": 4.845640182495117, "kl": 3.51280864700675, "learning_rate": 9.315776491082972e-07, "loss": 0.1405, "num_tokens": 7649536.0, "reward": 0.7335205078125, "reward_std": 0.013687143102288246, "rewards//mean": 0.7335205078125, "rewards//std": 0.02856513299047947, "step": 885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1772, "grad_norm": 6.412348747253418, "kl": 3.11839210242033, "learning_rate": 9.314173287433872e-07, "loss": 0.1247, "num_tokens": 7658176.0, "reward": 0.7615966796875, "reward_std": 0.011410508304834366, "rewards//mean": 0.7615966796875, "rewards//std": 0.022755546495318413, "step": 886 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1774, "grad_norm": 6.040501594543457, "kl": 4.250001834705472, "learning_rate": 9.312568346036287e-07, "loss": 0.17, "num_tokens": 7666712.0, "reward": 0.78076171875, "reward_std": 0.017303170636296272, "rewards//mean": 0.78076171875, "rewards//std": 0.030333632603287697, "step": 887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1776, "grad_norm": 6.827248573303223, "kl": 3.988034488633275, "learning_rate": 9.310961667536688e-07, "loss": 0.1595, "num_tokens": 7675416.0, "reward": 0.75567626953125, "reward_std": 0.01062617264688015, "rewards//mean": 0.75567626953125, "rewards//std": 0.0382918119430542, "step": 888 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1778, "grad_norm": 6.079413414001465, "kl": 1.765782754868269, "learning_rate": 9.309353252582245e-07, "loss": 0.0706, "num_tokens": 7684048.0, "reward": 0.7513427734375, "reward_std": 0.008468233980238438, "rewards//mean": 0.7513427734375, "rewards//std": 0.03319104388356209, "step": 889 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.178, "grad_norm": 5.774899959564209, "kl": 3.7291163839399815, "learning_rate": 9.307743101820827e-07, "loss": 0.1492, "num_tokens": 7692680.0, "reward": 0.7694091796875, "reward_std": 0.012057274580001831, "rewards//mean": 0.7694091796875, "rewards//std": 0.02732132002711296, "step": 890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1782, "grad_norm": 4.276165962219238, "kl": 2.535430883988738, "learning_rate": 9.306131215901003e-07, "loss": 0.1014, "num_tokens": 7701288.0, "reward": 0.75970458984375, "reward_std": 0.010588009841740131, "rewards//mean": 0.75970458984375, "rewards//std": 0.03684157878160477, "step": 891 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1784, "grad_norm": 8.516181945800781, "kl": 3.2093895692378283, "learning_rate": 9.304517595472039e-07, "loss": 0.1284, "num_tokens": 7709928.0, "reward": 0.77935791015625, "reward_std": 0.01215230394154787, "rewards//mean": 0.77935791015625, "rewards//std": 0.027515863999724388, "step": 892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1786, "grad_norm": 4.359760284423828, "kl": 3.3796353228390217, "learning_rate": 9.302902241183903e-07, "loss": 0.1352, "num_tokens": 7718504.0, "reward": 0.76055908203125, "reward_std": 0.01085734460502863, "rewards//mean": 0.76055908203125, "rewards//std": 0.019657080993056297, "step": 893 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1788, "grad_norm": 3.341512680053711, "kl": 2.6856060586869717, "learning_rate": 9.301285153687259e-07, "loss": 0.1074, "num_tokens": 7727272.0, "reward": 0.76300048828125, "reward_std": 0.009410968981683254, "rewards//mean": 0.76300048828125, "rewards//std": 0.027410585433244705, "step": 894 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.179, "grad_norm": 7.049658298492432, "kl": 4.035892087966204, "learning_rate": 9.29966633363347e-07, "loss": 0.1614, "num_tokens": 7736008.0, "reward": 0.73785400390625, "reward_std": 0.007529266644269228, "rewards//mean": 0.73785400390625, "rewards//std": 0.036212481558322906, "step": 895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1792, "grad_norm": 9.364948272705078, "kl": 3.315326800569892, "learning_rate": 9.298045781674595e-07, "loss": 0.1326, "num_tokens": 7744680.0, "reward": 0.74951171875, "reward_std": 0.008564095944166183, "rewards//mean": 0.74951171875, "rewards//std": 0.030667154118418694, "step": 896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1794, "grad_norm": 4.9357123374938965, "kl": 3.4647033978253603, "learning_rate": 9.296423498463395e-07, "loss": 0.1386, "num_tokens": 7753328.0, "reward": 0.78826904296875, "reward_std": 0.009156836196780205, "rewards//mean": 0.78826904296875, "rewards//std": 0.019822722300887108, "step": 897 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1796, "grad_norm": 5.250432968139648, "kl": 3.9305388052016497, "learning_rate": 9.294799484653322e-07, "loss": 0.1572, "num_tokens": 7761912.0, "reward": 0.77301025390625, "reward_std": 0.011185125447809696, "rewards//mean": 0.77301025390625, "rewards//std": 0.020352492108941078, "step": 898 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1798, "grad_norm": 4.539963722229004, "kl": 2.059523830190301, "learning_rate": 9.29317374089853e-07, "loss": 0.0824, "num_tokens": 7770584.0, "reward": 0.7725830078125, "reward_std": 0.010079247877001762, "rewards//mean": 0.7725830078125, "rewards//std": 0.02816380187869072, "step": 899 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.18, "grad_norm": 5.043696880340576, "kl": 4.320511316880584, "learning_rate": 9.291546267853869e-07, "loss": 0.1728, "num_tokens": 7779176.0, "reward": 0.765869140625, "reward_std": 0.01119816955178976, "rewards//mean": 0.765869140625, "rewards//std": 0.03090611845254898, "step": 900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1802, "grad_norm": 3.6633341312408447, "kl": 2.414677979424596, "learning_rate": 9.289917066174885e-07, "loss": 0.0966, "num_tokens": 7787928.0, "reward": 0.75933837890625, "reward_std": 0.010336682200431824, "rewards//mean": 0.75933837890625, "rewards//std": 0.0272560715675354, "step": 901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1804, "grad_norm": 4.289097309112549, "kl": 3.825998166576028, "learning_rate": 9.288286136517819e-07, "loss": 0.153, "num_tokens": 7796560.0, "reward": 0.7718505859375, "reward_std": 0.010769478045403957, "rewards//mean": 0.7718505859375, "rewards//std": 0.03365122526884079, "step": 902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1806, "grad_norm": 6.650477409362793, "kl": 3.9704781882464886, "learning_rate": 9.28665347953961e-07, "loss": 0.1588, "num_tokens": 7805264.0, "reward": 0.734375, "reward_std": 0.012096371501684189, "rewards//mean": 0.734375, "rewards//std": 0.03403255715966225, "step": 903 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1808, "grad_norm": 7.73297119140625, "kl": 4.305436102673411, "learning_rate": 9.285019095897893e-07, "loss": 0.1722, "num_tokens": 7813920.0, "reward": 0.7784423828125, "reward_std": 0.009727489203214645, "rewards//mean": 0.7784423828125, "rewards//std": 0.018208811059594154, "step": 904 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.181, "grad_norm": 5.065776824951172, "kl": 4.7629342544823885, "learning_rate": 9.283382986250996e-07, "loss": 0.1905, "num_tokens": 7822416.0, "reward": 0.715576171875, "reward_std": 0.012758323922753334, "rewards//mean": 0.715576171875, "rewards//std": 0.0380057655274868, "step": 905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1812, "grad_norm": 3.914884090423584, "kl": 3.7148711811751127, "learning_rate": 9.281745151257945e-07, "loss": 0.1486, "num_tokens": 7831000.0, "reward": 0.7301025390625, "reward_std": 0.010050761513411999, "rewards//mean": 0.7301025390625, "rewards//std": 0.033412866294384, "step": 906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1814, "grad_norm": 3.7434592247009277, "kl": 1.0109682343900204, "learning_rate": 9.280105591578458e-07, "loss": 0.0404, "num_tokens": 7839616.0, "reward": 0.77264404296875, "reward_std": 0.004304264672100544, "rewards//mean": 0.77264404296875, "rewards//std": 0.024021128192543983, "step": 907 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1816, "grad_norm": 4.644937515258789, "kl": 2.340283691883087, "learning_rate": 9.278464307872951e-07, "loss": 0.0936, "num_tokens": 7848272.0, "reward": 0.78802490234375, "reward_std": 0.00775968236848712, "rewards//mean": 0.78802490234375, "rewards//std": 0.0348847433924675, "step": 908 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1818, "grad_norm": 4.306415557861328, "kl": 1.1228253487497568, "learning_rate": 9.276821300802533e-07, "loss": 0.0449, "num_tokens": 7856824.0, "reward": 0.77325439453125, "reward_std": 0.009302186779677868, "rewards//mean": 0.77325439453125, "rewards//std": 0.018797343596816063, "step": 909 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.182, "grad_norm": 10.667795181274414, "kl": 5.042736670002341, "learning_rate": 9.275176571029006e-07, "loss": 0.2017, "num_tokens": 7865384.0, "reward": 0.7467041015625, "reward_std": 0.0096443435177207, "rewards//mean": 0.7467041015625, "rewards//std": 0.03261689096689224, "step": 910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1822, "grad_norm": 3.2729358673095703, "kl": 3.073277685791254, "learning_rate": 9.273530119214867e-07, "loss": 0.1229, "num_tokens": 7874016.0, "reward": 0.7843017578125, "reward_std": 0.008409148082137108, "rewards//mean": 0.7843017578125, "rewards//std": 0.020954687148332596, "step": 911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1824, "grad_norm": 4.72935676574707, "kl": 3.132097691297531, "learning_rate": 9.271881946023308e-07, "loss": 0.1253, "num_tokens": 7882688.0, "reward": 0.78680419921875, "reward_std": 0.01133975014090538, "rewards//mean": 0.78680419921875, "rewards//std": 0.025956004858016968, "step": 912 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1826, "grad_norm": 6.119086265563965, "kl": 3.040343400090933, "learning_rate": 9.270232052118212e-07, "loss": 0.1216, "num_tokens": 7891408.0, "reward": 0.766845703125, "reward_std": 0.011883992701768875, "rewards//mean": 0.766845703125, "rewards//std": 0.024918828159570694, "step": 913 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1828, "grad_norm": 9.933708190917969, "kl": 2.188705800101161, "learning_rate": 9.268580438164155e-07, "loss": 0.0875, "num_tokens": 7900024.0, "reward": 0.7637939453125, "reward_std": 0.007796712219715118, "rewards//mean": 0.7637939453125, "rewards//std": 0.029773274436593056, "step": 914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.183, "grad_norm": 6.544854164123535, "kl": 2.9327501095831394, "learning_rate": 9.266927104826408e-07, "loss": 0.1173, "num_tokens": 7908696.0, "reward": 0.77923583984375, "reward_std": 0.012672207318246365, "rewards//mean": 0.77923583984375, "rewards//std": 0.030726268887519836, "step": 915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1832, "grad_norm": 9.247836112976074, "kl": 3.3677007276564837, "learning_rate": 9.265272052770935e-07, "loss": 0.1347, "num_tokens": 7917392.0, "reward": 0.79522705078125, "reward_std": 0.011345906183123589, "rewards//mean": 0.79522705078125, "rewards//std": 0.019485371187329292, "step": 916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1834, "grad_norm": 6.582352161407471, "kl": 2.0685936380177736, "learning_rate": 9.263615282664388e-07, "loss": 0.0827, "num_tokens": 7926048.0, "reward": 0.748046875, "reward_std": 0.006280519068241119, "rewards//mean": 0.748046875, "rewards//std": 0.04136914759874344, "step": 917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1836, "grad_norm": 16.554576873779297, "kl": 7.73712888173759, "learning_rate": 9.261956795174115e-07, "loss": 0.3095, "num_tokens": 7935016.0, "reward": 0.7740478515625, "reward_std": 0.011473117396235466, "rewards//mean": 0.7740478515625, "rewards//std": 0.04325369372963905, "step": 918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1838, "grad_norm": 5.420589923858643, "kl": 2.9603129997849464, "learning_rate": 9.260296590968156e-07, "loss": 0.1184, "num_tokens": 7943656.0, "reward": 0.7490234375, "reward_std": 0.0086138267070055, "rewards//mean": 0.7490234375, "rewards//std": 0.028510231524705887, "step": 919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.184, "grad_norm": 3.921104669570923, "kl": 1.8178562615066767, "learning_rate": 9.258634670715237e-07, "loss": 0.0727, "num_tokens": 7952288.0, "reward": 0.743408203125, "reward_std": 0.01061804685741663, "rewards//mean": 0.743408203125, "rewards//std": 0.028526155278086662, "step": 920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1842, "grad_norm": 9.220185279846191, "kl": 3.6572155449539423, "learning_rate": 9.256971035084784e-07, "loss": 0.1463, "num_tokens": 7960904.0, "reward": 0.74310302734375, "reward_std": 0.0070690978318452835, "rewards//mean": 0.74310302734375, "rewards//std": 0.037472233176231384, "step": 921 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1844, "grad_norm": 6.308659553527832, "kl": 6.336701916530728, "learning_rate": 9.255305684746907e-07, "loss": 0.2535, "num_tokens": 7969416.0, "reward": 0.7818603515625, "reward_std": 0.015000980347394943, "rewards//mean": 0.7818603515625, "rewards//std": 0.022186988964676857, "step": 922 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1846, "grad_norm": 10.97612190246582, "kl": 3.909780852496624, "learning_rate": 9.253638620372408e-07, "loss": 0.1564, "num_tokens": 7978048.0, "reward": 0.77264404296875, "reward_std": 0.009893986396491528, "rewards//mean": 0.77264404296875, "rewards//std": 0.028040649369359016, "step": 923 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1848, "grad_norm": 7.256546974182129, "kl": 5.387146627530456, "learning_rate": 9.251969842632783e-07, "loss": 0.2155, "num_tokens": 7986648.0, "reward": 0.765380859375, "reward_std": 0.010658979415893555, "rewards//mean": 0.765380859375, "rewards//std": 0.020266123116016388, "step": 924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.185, "grad_norm": 6.968551158905029, "kl": 5.536201927810907, "learning_rate": 9.250299352200212e-07, "loss": 0.2214, "num_tokens": 7995320.0, "reward": 0.7738037109375, "reward_std": 0.014981362968683243, "rewards//mean": 0.7738037109375, "rewards//std": 0.025607606396079063, "step": 925 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1852, "grad_norm": 9.227314949035645, "kl": 2.338663136586547, "learning_rate": 9.248627149747572e-07, "loss": 0.0935, "num_tokens": 8003952.0, "reward": 0.77825927734375, "reward_std": 0.010928423143923283, "rewards//mean": 0.77825927734375, "rewards//std": 0.03368736431002617, "step": 926 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1854, "grad_norm": 10.275461196899414, "kl": 5.7213707230985165, "learning_rate": 9.246953235948422e-07, "loss": 0.2289, "num_tokens": 8012560.0, "reward": 0.7593994140625, "reward_std": 0.012199217453598976, "rewards//mean": 0.7593994140625, "rewards//std": 0.031053440645337105, "step": 927 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1856, "grad_norm": 5.597161293029785, "kl": 4.975529843941331, "learning_rate": 9.245277611477018e-07, "loss": 0.199, "num_tokens": 8021216.0, "reward": 0.77362060546875, "reward_std": 0.009361838921904564, "rewards//mean": 0.77362060546875, "rewards//std": 0.024144329130649567, "step": 928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1858, "grad_norm": 14.230475425720215, "kl": 6.894909713417292, "learning_rate": 9.2436002770083e-07, "loss": 0.2758, "num_tokens": 8029968.0, "reward": 0.7471923828125, "reward_std": 0.01625603809952736, "rewards//mean": 0.7471923828125, "rewards//std": 0.03861016035079956, "step": 929 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.186, "grad_norm": 10.567397117614746, "kl": 4.905621299520135, "learning_rate": 9.241921233217897e-07, "loss": 0.1962, "num_tokens": 8038632.0, "reward": 0.75860595703125, "reward_std": 0.010758135467767715, "rewards//mean": 0.75860595703125, "rewards//std": 0.028630439192056656, "step": 930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1862, "grad_norm": 12.504945755004883, "kl": 6.215815335512161, "learning_rate": 9.240240480782129e-07, "loss": 0.2486, "num_tokens": 8047248.0, "reward": 0.7353515625, "reward_std": 0.01204628311097622, "rewards//mean": 0.7353515625, "rewards//std": 0.024842167273163795, "step": 931 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1864, "grad_norm": 9.700063705444336, "kl": 2.4597761891782284, "learning_rate": 9.238558020378003e-07, "loss": 0.0984, "num_tokens": 8055896.0, "reward": 0.73919677734375, "reward_std": 0.010735702700912952, "rewards//mean": 0.73919677734375, "rewards//std": 0.0354938767850399, "step": 932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1866, "grad_norm": 11.815288543701172, "kl": 2.485420987010002, "learning_rate": 9.236873852683212e-07, "loss": 0.0994, "num_tokens": 8064552.0, "reward": 0.783447265625, "reward_std": 0.009777087718248367, "rewards//mean": 0.783447265625, "rewards//std": 0.024704068899154663, "step": 933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1868, "grad_norm": 9.662408828735352, "kl": 6.564951363950968, "learning_rate": 9.235187978376141e-07, "loss": 0.2626, "num_tokens": 8073232.0, "reward": 0.75762939453125, "reward_std": 0.011026167310774326, "rewards//mean": 0.75762939453125, "rewards//std": 0.03184446692466736, "step": 934 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.187, "grad_norm": 8.485936164855957, "kl": 4.204109560698271, "learning_rate": 9.233500398135858e-07, "loss": 0.1682, "num_tokens": 8081840.0, "reward": 0.75640869140625, "reward_std": 0.018468102440238, "rewards//mean": 0.75640869140625, "rewards//std": 0.031229589134454727, "step": 935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1872, "grad_norm": 6.324738025665283, "kl": 5.124746752902865, "learning_rate": 9.23181111264212e-07, "loss": 0.205, "num_tokens": 8090544.0, "reward": 0.750244140625, "reward_std": 0.014276128262281418, "rewards//mean": 0.750244140625, "rewards//std": 0.034024547785520554, "step": 936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1874, "grad_norm": 4.18878173828125, "kl": 2.7625075317919254, "learning_rate": 9.230120122575375e-07, "loss": 0.1105, "num_tokens": 8099208.0, "reward": 0.7501220703125, "reward_std": 0.009567821398377419, "rewards//mean": 0.7501220703125, "rewards//std": 0.024079428985714912, "step": 937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1876, "grad_norm": 11.401058197021484, "kl": 4.1268694922327995, "learning_rate": 9.228427428616748e-07, "loss": 0.1651, "num_tokens": 8107856.0, "reward": 0.75201416015625, "reward_std": 0.012402264401316643, "rewards//mean": 0.75201416015625, "rewards//std": 0.026383472606539726, "step": 938 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1878, "grad_norm": 10.560399055480957, "kl": 2.1033124066889286, "learning_rate": 9.22673303144806e-07, "loss": 0.0841, "num_tokens": 8116456.0, "reward": 0.7196044921875, "reward_std": 0.010810447856783867, "rewards//mean": 0.7196044921875, "rewards//std": 0.04457870125770569, "step": 939 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.188, "grad_norm": 5.703797817230225, "kl": 4.017137309536338, "learning_rate": 9.22503693175181e-07, "loss": 0.1607, "num_tokens": 8125016.0, "reward": 0.7542724609375, "reward_std": 0.011905958876013756, "rewards//mean": 0.7542724609375, "rewards//std": 0.037134453654289246, "step": 940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1882, "grad_norm": 4.964807033538818, "kl": 4.473344206809998, "learning_rate": 9.223339130211192e-07, "loss": 0.1789, "num_tokens": 8133600.0, "reward": 0.76043701171875, "reward_std": 0.012273428961634636, "rewards//mean": 0.76043701171875, "rewards//std": 0.0338330902159214, "step": 941 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1884, "grad_norm": 7.53067684173584, "kl": 5.051701173186302, "learning_rate": 9.221639627510075e-07, "loss": 0.2021, "num_tokens": 8142232.0, "reward": 0.76171875, "reward_std": 0.017527125775814056, "rewards//mean": 0.76171875, "rewards//std": 0.023261159658432007, "step": 942 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1886, "grad_norm": 10.17851448059082, "kl": 3.9920184295624495, "learning_rate": 9.219938424333023e-07, "loss": 0.1597, "num_tokens": 8150976.0, "reward": 0.74755859375, "reward_std": 0.016106873750686646, "rewards//mean": 0.74755859375, "rewards//std": 0.02955717034637928, "step": 943 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1888, "grad_norm": 6.212186813354492, "kl": 3.6984230782836676, "learning_rate": 9.218235521365276e-07, "loss": 0.1479, "num_tokens": 8159640.0, "reward": 0.76153564453125, "reward_std": 0.011410975828766823, "rewards//mean": 0.76153564453125, "rewards//std": 0.045327410101890564, "step": 944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.189, "grad_norm": 5.5202131271362305, "kl": 2.3476297073066235, "learning_rate": 9.216530919292767e-07, "loss": 0.0939, "num_tokens": 8168288.0, "reward": 0.7337646484375, "reward_std": 0.006907983683049679, "rewards//mean": 0.7337646484375, "rewards//std": 0.026604946702718735, "step": 945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1892, "grad_norm": 5.84950590133667, "kl": 4.926447322592139, "learning_rate": 9.214824618802107e-07, "loss": 0.1971, "num_tokens": 8176928.0, "reward": 0.765869140625, "reward_std": 0.011790703982114792, "rewards//mean": 0.765869140625, "rewards//std": 0.024831196293234825, "step": 946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1894, "grad_norm": 6.732231616973877, "kl": 4.739807171747088, "learning_rate": 9.213116620580596e-07, "loss": 0.1896, "num_tokens": 8185472.0, "reward": 0.77294921875, "reward_std": 0.019628707319498062, "rewards//mean": 0.77294921875, "rewards//std": 0.02937633916735649, "step": 947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1896, "grad_norm": 6.770096778869629, "kl": 5.924624875187874, "learning_rate": 9.211406925316212e-07, "loss": 0.237, "num_tokens": 8194096.0, "reward": 0.7716064453125, "reward_std": 0.016851382330060005, "rewards//mean": 0.7716064453125, "rewards//std": 0.03264472633600235, "step": 948 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1898, "grad_norm": 5.986906051635742, "kl": 4.227919224649668, "learning_rate": 9.209695533697623e-07, "loss": 0.1691, "num_tokens": 8202632.0, "reward": 0.75616455078125, "reward_std": 0.01510709524154663, "rewards//mean": 0.75616455078125, "rewards//std": 0.01998850516974926, "step": 949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.19, "grad_norm": 5.989424228668213, "kl": 3.7134490106254816, "learning_rate": 9.207982446414177e-07, "loss": 0.1485, "num_tokens": 8211200.0, "reward": 0.77642822265625, "reward_std": 0.015160230919718742, "rewards//mean": 0.77642822265625, "rewards//std": 0.02674589306116104, "step": 950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1902, "grad_norm": 7.805545806884766, "kl": 3.576969100162387, "learning_rate": 9.206267664155906e-07, "loss": 0.1431, "num_tokens": 8219872.0, "reward": 0.77569580078125, "reward_std": 0.014061015099287033, "rewards//mean": 0.77569580078125, "rewards//std": 0.03300506994128227, "step": 951 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1904, "grad_norm": 20.467113494873047, "kl": 7.740051489323378, "learning_rate": 9.20455118761352e-07, "loss": 0.3096, "num_tokens": 8228504.0, "reward": 0.75762939453125, "reward_std": 0.015062106773257256, "rewards//mean": 0.75762939453125, "rewards//std": 0.02359641157090664, "step": 952 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1906, "grad_norm": 7.589361190795898, "kl": 4.330275880172849, "learning_rate": 9.202833017478421e-07, "loss": 0.1732, "num_tokens": 8237192.0, "reward": 0.74267578125, "reward_std": 0.017783362418413162, "rewards//mean": 0.74267578125, "rewards//std": 0.031570009887218475, "step": 953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1908, "grad_norm": 10.500250816345215, "kl": 4.117554211989045, "learning_rate": 9.201113154442683e-07, "loss": 0.1647, "num_tokens": 8245880.0, "reward": 0.736328125, "reward_std": 0.00937635637819767, "rewards//mean": 0.736328125, "rewards//std": 0.0351114422082901, "step": 954 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.191, "grad_norm": 8.20156478881836, "kl": 3.607573974877596, "learning_rate": 9.199391599199071e-07, "loss": 0.1443, "num_tokens": 8254472.0, "reward": 0.786376953125, "reward_std": 0.012617506086826324, "rewards//mean": 0.786376953125, "rewards//std": 0.021450528874993324, "step": 955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1912, "grad_norm": 9.058380126953125, "kl": 6.485325139015913, "learning_rate": 9.197668352441023e-07, "loss": 0.2594, "num_tokens": 8263168.0, "reward": 0.76019287109375, "reward_std": 0.015972504392266273, "rewards//mean": 0.76019287109375, "rewards//std": 0.026581251993775368, "step": 956 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1914, "grad_norm": 13.800029754638672, "kl": 5.103701438754797, "learning_rate": 9.195943414862665e-07, "loss": 0.2041, "num_tokens": 8271784.0, "reward": 0.716552734375, "reward_std": 0.009655911475419998, "rewards//mean": 0.716552734375, "rewards//std": 0.041520360857248306, "step": 957 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1916, "grad_norm": 14.575108528137207, "kl": 5.376349829137325, "learning_rate": 9.194216787158804e-07, "loss": 0.2151, "num_tokens": 8280496.0, "reward": 0.74310302734375, "reward_std": 0.008960509672760963, "rewards//mean": 0.74310302734375, "rewards//std": 0.03096770867705345, "step": 958 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1918, "grad_norm": 13.319855690002441, "kl": 2.192217580974102, "learning_rate": 9.192488470024919e-07, "loss": 0.0877, "num_tokens": 8289160.0, "reward": 0.75482177734375, "reward_std": 0.005320992320775986, "rewards//mean": 0.75482177734375, "rewards//std": 0.024868888780474663, "step": 959 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.192, "grad_norm": 4.39548921585083, "kl": 5.349567232653499, "learning_rate": 9.190758464157182e-07, "loss": 0.214, "num_tokens": 8297768.0, "reward": 0.732421875, "reward_std": 0.01328906137496233, "rewards//mean": 0.732421875, "rewards//std": 0.032429084181785583, "step": 960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1922, "grad_norm": 6.398663520812988, "kl": 2.967155009508133, "learning_rate": 9.189026770252436e-07, "loss": 0.1187, "num_tokens": 8306440.0, "reward": 0.74798583984375, "reward_std": 0.006764195393770933, "rewards//mean": 0.74798583984375, "rewards//std": 0.02624541148543358, "step": 961 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1924, "grad_norm": 11.789690017700195, "kl": 2.2300401646643877, "learning_rate": 9.187293389008208e-07, "loss": 0.0892, "num_tokens": 8315072.0, "reward": 0.75286865234375, "reward_std": 0.00717154610902071, "rewards//mean": 0.75286865234375, "rewards//std": 0.021261893212795258, "step": 962 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1926, "grad_norm": 7.450940132141113, "kl": 1.6545574720948935, "learning_rate": 9.185558321122704e-07, "loss": 0.0662, "num_tokens": 8323776.0, "reward": 0.76458740234375, "reward_std": 0.007642059121280909, "rewards//mean": 0.76458740234375, "rewards//std": 0.02798931673169136, "step": 963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1928, "grad_norm": 3.8259990215301514, "kl": 2.7195714991539717, "learning_rate": 9.183821567294808e-07, "loss": 0.1088, "num_tokens": 8332408.0, "reward": 0.754638671875, "reward_std": 0.007108988240361214, "rewards//mean": 0.754638671875, "rewards//std": 0.021864313632249832, "step": 964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.193, "grad_norm": 10.221814155578613, "kl": 6.793953355401754, "learning_rate": 9.182083128224086e-07, "loss": 0.2718, "num_tokens": 8341096.0, "reward": 0.75445556640625, "reward_std": 0.013266519643366337, "rewards//mean": 0.75445556640625, "rewards//std": 0.02583499811589718, "step": 965 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1932, "grad_norm": 7.6526875495910645, "kl": 2.6198825035244226, "learning_rate": 9.180343004610779e-07, "loss": 0.1048, "num_tokens": 8349760.0, "reward": 0.77655029296875, "reward_std": 0.015037826262414455, "rewards//mean": 0.77655029296875, "rewards//std": 0.026265589520335197, "step": 966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1934, "grad_norm": 4.734843730926514, "kl": 2.7961494512856007, "learning_rate": 9.178601197155811e-07, "loss": 0.1118, "num_tokens": 8358440.0, "reward": 0.754638671875, "reward_std": 0.007869785651564598, "rewards//mean": 0.754638671875, "rewards//std": 0.02464517392218113, "step": 967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1936, "grad_norm": 8.340118408203125, "kl": 2.958549842238426, "learning_rate": 9.176857706560779e-07, "loss": 0.1183, "num_tokens": 8367064.0, "reward": 0.7425537109375, "reward_std": 0.007053266745060682, "rewards//mean": 0.7425537109375, "rewards//std": 0.028019385412335396, "step": 968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1938, "grad_norm": 11.365989685058594, "kl": 2.293816465884447, "learning_rate": 9.175112533527963e-07, "loss": 0.0918, "num_tokens": 8375712.0, "reward": 0.72906494140625, "reward_std": 0.013245773501694202, "rewards//mean": 0.72906494140625, "rewards//std": 0.029432903975248337, "step": 969 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.194, "grad_norm": 6.58137845993042, "kl": 5.306589435786009, "learning_rate": 9.173365678760317e-07, "loss": 0.2123, "num_tokens": 8384464.0, "reward": 0.75830078125, "reward_std": 0.015954727306962013, "rewards//mean": 0.75830078125, "rewards//std": 0.035512689501047134, "step": 970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1942, "grad_norm": 7.5659074783325195, "kl": 4.431315923109651, "learning_rate": 9.171617142961476e-07, "loss": 0.1773, "num_tokens": 8393096.0, "reward": 0.76904296875, "reward_std": 0.015092447400093079, "rewards//mean": 0.76904296875, "rewards//std": 0.02833554334938526, "step": 971 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1944, "grad_norm": 7.9966936111450195, "kl": 4.54252371750772, "learning_rate": 9.169866926835747e-07, "loss": 0.1817, "num_tokens": 8401712.0, "reward": 0.746337890625, "reward_std": 0.012072852812707424, "rewards//mean": 0.746337890625, "rewards//std": 0.021143468096852303, "step": 972 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1946, "grad_norm": 6.257865905761719, "kl": 3.8888294734060764, "learning_rate": 9.16811503108812e-07, "loss": 0.1556, "num_tokens": 8410496.0, "reward": 0.74932861328125, "reward_std": 0.010210035368800163, "rewards//mean": 0.74932861328125, "rewards//std": 0.02372054196894169, "step": 973 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1948, "grad_norm": 12.508591651916504, "kl": 6.170948276296258, "learning_rate": 9.166361456424257e-07, "loss": 0.2468, "num_tokens": 8419064.0, "reward": 0.75628662109375, "reward_std": 0.012421725317835808, "rewards//mean": 0.75628662109375, "rewards//std": 0.03375828638672829, "step": 974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.195, "grad_norm": 9.373515129089355, "kl": 6.465175721794367, "learning_rate": 9.164606203550497e-07, "loss": 0.2586, "num_tokens": 8427840.0, "reward": 0.76715087890625, "reward_std": 0.010892903432250023, "rewards//mean": 0.76715087890625, "rewards//std": 0.02707776427268982, "step": 975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1952, "grad_norm": 16.16870880126953, "kl": 8.718793712556362, "learning_rate": 9.162849273173856e-07, "loss": 0.3488, "num_tokens": 8436616.0, "reward": 0.772216796875, "reward_std": 0.018080944195389748, "rewards//mean": 0.772216796875, "rewards//std": 0.03033662587404251, "step": 976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1954, "grad_norm": 5.9699835777282715, "kl": 4.7948022577911615, "learning_rate": 9.161090666002027e-07, "loss": 0.1918, "num_tokens": 8445216.0, "reward": 0.78094482421875, "reward_std": 0.013379577547311783, "rewards//mean": 0.78094482421875, "rewards//std": 0.03297937288880348, "step": 977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1956, "grad_norm": 5.532942295074463, "kl": 5.0951950047165155, "learning_rate": 9.159330382743373e-07, "loss": 0.2038, "num_tokens": 8453896.0, "reward": 0.7803955078125, "reward_std": 0.01595272868871689, "rewards//mean": 0.7803955078125, "rewards//std": 0.03219272568821907, "step": 978 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1958, "grad_norm": 9.143082618713379, "kl": 5.502403872087598, "learning_rate": 9.157568424106941e-07, "loss": 0.2201, "num_tokens": 8462552.0, "reward": 0.77056884765625, "reward_std": 0.013268399983644485, "rewards//mean": 0.77056884765625, "rewards//std": 0.02603461965918541, "step": 979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.196, "grad_norm": 21.102041244506836, "kl": 8.055116983130574, "learning_rate": 9.155804790802443e-07, "loss": 0.3222, "num_tokens": 8471232.0, "reward": 0.75640869140625, "reward_std": 0.021582897752523422, "rewards//mean": 0.75640869140625, "rewards//std": 0.036334339529275894, "step": 980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1962, "grad_norm": 8.947332382202148, "kl": 6.066417574882507, "learning_rate": 9.154039483540272e-07, "loss": 0.2427, "num_tokens": 8479856.0, "reward": 0.76123046875, "reward_std": 0.014370684511959553, "rewards//mean": 0.76123046875, "rewards//std": 0.026213163509964943, "step": 981 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1964, "grad_norm": 5.394735336303711, "kl": 6.074656603857875, "learning_rate": 9.152272503031495e-07, "loss": 0.243, "num_tokens": 8488528.0, "reward": 0.74566650390625, "reward_std": 0.017375901341438293, "rewards//mean": 0.74566650390625, "rewards//std": 0.03097064234316349, "step": 982 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1966, "grad_norm": 7.742849826812744, "kl": 4.014471486210823, "learning_rate": 9.150503849987851e-07, "loss": 0.1606, "num_tokens": 8497112.0, "reward": 0.74835205078125, "reward_std": 0.009470732882618904, "rewards//mean": 0.74835205078125, "rewards//std": 0.02913879044353962, "step": 983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1968, "grad_norm": 5.740931034088135, "kl": 2.7029154244810343, "learning_rate": 9.14873352512175e-07, "loss": 0.1081, "num_tokens": 8505752.0, "reward": 0.770263671875, "reward_std": 0.010124212130904198, "rewards//mean": 0.770263671875, "rewards//std": 0.030890440568327904, "step": 984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.197, "grad_norm": 22.27727508544922, "kl": 2.468091754242778, "learning_rate": 9.146961529146284e-07, "loss": 0.0987, "num_tokens": 8514376.0, "reward": 0.7520751953125, "reward_std": 0.008476316928863525, "rewards//mean": 0.7520751953125, "rewards//std": 0.03124200738966465, "step": 985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1972, "grad_norm": 5.4475297927856445, "kl": 4.27043379843235, "learning_rate": 9.145187862775208e-07, "loss": 0.1708, "num_tokens": 8522960.0, "reward": 0.730712890625, "reward_std": 0.009307174943387508, "rewards//mean": 0.730712890625, "rewards//std": 0.02970735915005207, "step": 986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1974, "grad_norm": 5.541208744049072, "kl": 2.2829291969537735, "learning_rate": 9.143412526722958e-07, "loss": 0.0913, "num_tokens": 8531496.0, "reward": 0.7393798828125, "reward_std": 0.010197298601269722, "rewards//mean": 0.7393798828125, "rewards//std": 0.03410697728395462, "step": 987 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1976, "grad_norm": 6.083037376403809, "kl": 5.546741595491767, "learning_rate": 9.141635521704636e-07, "loss": 0.2219, "num_tokens": 8540232.0, "reward": 0.75115966796875, "reward_std": 0.013699512928724289, "rewards//mean": 0.75115966796875, "rewards//std": 0.03273149952292442, "step": 988 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1978, "grad_norm": 6.53829288482666, "kl": 2.6669446397572756, "learning_rate": 9.139856848436023e-07, "loss": 0.1067, "num_tokens": 8548920.0, "reward": 0.7674560546875, "reward_std": 0.010561926290392876, "rewards//mean": 0.7674560546875, "rewards//std": 0.030694536864757538, "step": 989 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.198, "grad_norm": 6.94627571105957, "kl": 5.959777383133769, "learning_rate": 9.138076507633565e-07, "loss": 0.2384, "num_tokens": 8557688.0, "reward": 0.77606201171875, "reward_std": 0.016999781131744385, "rewards//mean": 0.77606201171875, "rewards//std": 0.03706483542919159, "step": 990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1982, "grad_norm": 4.428110122680664, "kl": 1.7214825954288244, "learning_rate": 9.136294500014385e-07, "loss": 0.0689, "num_tokens": 8566352.0, "reward": 0.76226806640625, "reward_std": 0.007398197427392006, "rewards//mean": 0.76226806640625, "rewards//std": 0.023050576448440552, "step": 991 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1984, "grad_norm": 7.641226291656494, "kl": 4.1124687902629375, "learning_rate": 9.134510826296276e-07, "loss": 0.1645, "num_tokens": 8575064.0, "reward": 0.7491455078125, "reward_std": 0.008870312944054604, "rewards//mean": 0.7491455078125, "rewards//std": 0.025424884632229805, "step": 992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1986, "grad_norm": 4.991622447967529, "kl": 4.2028822265565395, "learning_rate": 9.1327254871977e-07, "loss": 0.1681, "num_tokens": 8583664.0, "reward": 0.74114990234375, "reward_std": 0.008776310831308365, "rewards//mean": 0.74114990234375, "rewards//std": 0.024421734735369682, "step": 993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1988, "grad_norm": 6.3080644607543945, "kl": 4.167820289731026, "learning_rate": 9.130938483437791e-07, "loss": 0.1667, "num_tokens": 8592344.0, "reward": 0.75457763671875, "reward_std": 0.007406790740787983, "rewards//mean": 0.75457763671875, "rewards//std": 0.02382243052124977, "step": 994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.199, "grad_norm": 7.225706100463867, "kl": 2.1177551988512278, "learning_rate": 9.129149815736357e-07, "loss": 0.0847, "num_tokens": 8601024.0, "reward": 0.73858642578125, "reward_std": 0.0062733604572713375, "rewards//mean": 0.73858642578125, "rewards//std": 0.035408902913331985, "step": 995 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1992, "grad_norm": 5.533758640289307, "kl": 3.727728743106127, "learning_rate": 9.12735948481387e-07, "loss": 0.1491, "num_tokens": 8609688.0, "reward": 0.76995849609375, "reward_std": 0.009360658004879951, "rewards//mean": 0.76995849609375, "rewards//std": 0.03498094156384468, "step": 996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1994, "grad_norm": 6.253836154937744, "kl": 1.8540999237447977, "learning_rate": 9.125567491391475e-07, "loss": 0.0742, "num_tokens": 8618400.0, "reward": 0.77691650390625, "reward_std": 0.00872066244482994, "rewards//mean": 0.77691650390625, "rewards//std": 0.028246654197573662, "step": 997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1996, "grad_norm": 5.4483771324157715, "kl": 4.82985270768404, "learning_rate": 9.123773836190989e-07, "loss": 0.1932, "num_tokens": 8627216.0, "reward": 0.7860107421875, "reward_std": 0.016828035935759544, "rewards//mean": 0.7860107421875, "rewards//std": 0.029769206419587135, "step": 998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.1998, "grad_norm": 6.1263957023620605, "kl": 5.5002775229513645, "learning_rate": 9.121978519934895e-07, "loss": 0.22, "num_tokens": 8635816.0, "reward": 0.7213134765625, "reward_std": 0.01754913106560707, "rewards//mean": 0.7213134765625, "rewards//std": 0.03597830981016159, "step": 999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2, "grad_norm": 5.050541877746582, "kl": 4.969757461920381, "learning_rate": 9.120181543346346e-07, "loss": 0.1988, "num_tokens": 8644496.0, "reward": 0.7615966796875, "reward_std": 0.01377120241522789, "rewards//mean": 0.7615966796875, "rewards//std": 0.028236806392669678, "step": 1000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2002, "grad_norm": 5.757530212402344, "kl": 3.2096720319241285, "learning_rate": 9.118382907149163e-07, "loss": 0.1284, "num_tokens": 8653088.0, "reward": 0.77130126953125, "reward_std": 0.010810548439621925, "rewards//mean": 0.77130126953125, "rewards//std": 0.0304406750947237, "step": 1001 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2004, "grad_norm": 11.434347152709961, "kl": 5.517957452684641, "learning_rate": 9.116582612067838e-07, "loss": 0.2207, "num_tokens": 8661776.0, "reward": 0.751708984375, "reward_std": 0.010216728784143925, "rewards//mean": 0.751708984375, "rewards//std": 0.0311636570841074, "step": 1002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2006, "grad_norm": 5.935197830200195, "kl": 1.692651392892003, "learning_rate": 9.11478065882753e-07, "loss": 0.0677, "num_tokens": 8670552.0, "reward": 0.777587890625, "reward_std": 0.006503267213702202, "rewards//mean": 0.777587890625, "rewards//std": 0.0248604416847229, "step": 1003 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2008, "grad_norm": 9.93894100189209, "kl": 1.6166835632175207, "learning_rate": 9.112977048154064e-07, "loss": 0.0647, "num_tokens": 8679200.0, "reward": 0.77734375, "reward_std": 0.00589034054428339, "rewards//mean": 0.77734375, "rewards//std": 0.022162755951285362, "step": 1004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.201, "grad_norm": 3.701009750366211, "kl": 3.6509775668382645, "learning_rate": 9.111171780773936e-07, "loss": 0.146, "num_tokens": 8687768.0, "reward": 0.762451171875, "reward_std": 0.01004645973443985, "rewards//mean": 0.762451171875, "rewards//std": 0.03575991094112396, "step": 1005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2012, "grad_norm": 5.172314643859863, "kl": 4.7731046769768, "learning_rate": 9.109364857414305e-07, "loss": 0.1909, "num_tokens": 8696448.0, "reward": 0.715576171875, "reward_std": 0.011936701834201813, "rewards//mean": 0.715576171875, "rewards//std": 0.033341433852910995, "step": 1006 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2014, "grad_norm": 5.025206565856934, "kl": 2.409999590367079, "learning_rate": 9.107556278803002e-07, "loss": 0.0964, "num_tokens": 8705088.0, "reward": 0.772216796875, "reward_std": 0.012615786865353584, "rewards//mean": 0.772216796875, "rewards//std": 0.027142604812979698, "step": 1007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2016, "grad_norm": 4.429189682006836, "kl": 3.5807060841470957, "learning_rate": 9.10574604566852e-07, "loss": 0.1432, "num_tokens": 8713744.0, "reward": 0.7252197265625, "reward_std": 0.010518459603190422, "rewards//mean": 0.7252197265625, "rewards//std": 0.029008951038122177, "step": 1008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2018, "grad_norm": 6.667728424072266, "kl": 4.246804840862751, "learning_rate": 9.103934158740022e-07, "loss": 0.1699, "num_tokens": 8722456.0, "reward": 0.7818603515625, "reward_std": 0.010562303476035595, "rewards//mean": 0.7818603515625, "rewards//std": 0.022672908380627632, "step": 1009 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.202, "grad_norm": 3.641935348510742, "kl": 2.2933216989040375, "learning_rate": 9.102120618747336e-07, "loss": 0.0917, "num_tokens": 8731136.0, "reward": 0.75347900390625, "reward_std": 0.005826765671372414, "rewards//mean": 0.75347900390625, "rewards//std": 0.025220664218068123, "step": 1010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2022, "grad_norm": 3.6189522743225098, "kl": 2.471486421301961, "learning_rate": 9.100305426420956e-07, "loss": 0.0989, "num_tokens": 8739872.0, "reward": 0.7725830078125, "reward_std": 0.00782520417124033, "rewards//mean": 0.7725830078125, "rewards//std": 0.01757252775132656, "step": 1011 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2024, "grad_norm": 5.1977128982543945, "kl": 1.7518767956644297, "learning_rate": 9.098488582492039e-07, "loss": 0.0701, "num_tokens": 8748488.0, "reward": 0.7718505859375, "reward_std": 0.005266233813017607, "rewards//mean": 0.7718505859375, "rewards//std": 0.020596198737621307, "step": 1012 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2026, "grad_norm": 6.584754467010498, "kl": 4.939415320754051, "learning_rate": 9.096670087692411e-07, "loss": 0.1976, "num_tokens": 8757128.0, "reward": 0.761962890625, "reward_std": 0.01434204913675785, "rewards//mean": 0.761962890625, "rewards//std": 0.027355927973985672, "step": 1013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2028, "grad_norm": 4.593523979187012, "kl": 3.1541009787470102, "learning_rate": 9.094849942754563e-07, "loss": 0.1262, "num_tokens": 8765896.0, "reward": 0.732666015625, "reward_std": 0.009667712263762951, "rewards//mean": 0.732666015625, "rewards//std": 0.0408380888402462, "step": 1014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.203, "grad_norm": 3.591948986053467, "kl": 2.6722115837037563, "learning_rate": 9.093028148411648e-07, "loss": 0.1069, "num_tokens": 8774560.0, "reward": 0.76611328125, "reward_std": 0.008937373757362366, "rewards//mean": 0.76611328125, "rewards//std": 0.024886002764105797, "step": 1015 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2032, "grad_norm": 4.636039733886719, "kl": 4.22676838748157, "learning_rate": 9.091204705397483e-07, "loss": 0.1691, "num_tokens": 8783240.0, "reward": 0.76513671875, "reward_std": 0.010582895949482918, "rewards//mean": 0.76513671875, "rewards//std": 0.02228807657957077, "step": 1016 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2034, "grad_norm": 5.955030918121338, "kl": 2.6091385539621115, "learning_rate": 9.089379614446553e-07, "loss": 0.1044, "num_tokens": 8791904.0, "reward": 0.77581787109375, "reward_std": 0.010004453361034393, "rewards//mean": 0.77581787109375, "rewards//std": 0.02522546611726284, "step": 1017 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2036, "grad_norm": 6.787014484405518, "kl": 1.3281155284494162, "learning_rate": 9.087552876294002e-07, "loss": 0.0531, "num_tokens": 8800440.0, "reward": 0.76031494140625, "reward_std": 0.007894234731793404, "rewards//mean": 0.76031494140625, "rewards//std": 0.025328470394015312, "step": 1018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2038, "grad_norm": 4.883639335632324, "kl": 4.105878921225667, "learning_rate": 9.085724491675642e-07, "loss": 0.1642, "num_tokens": 8809040.0, "reward": 0.75872802734375, "reward_std": 0.01019999384880066, "rewards//mean": 0.75872802734375, "rewards//std": 0.03147197887301445, "step": 1019 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.204, "grad_norm": 9.381871223449707, "kl": 5.848114220425487, "learning_rate": 9.083894461327945e-07, "loss": 0.2339, "num_tokens": 8817712.0, "reward": 0.7694091796875, "reward_std": 0.011212250217795372, "rewards//mean": 0.7694091796875, "rewards//std": 0.035338133573532104, "step": 1020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2042, "grad_norm": 11.179381370544434, "kl": 4.110636135563254, "learning_rate": 9.082062785988048e-07, "loss": 0.1644, "num_tokens": 8826296.0, "reward": 0.76470947265625, "reward_std": 0.011189059354364872, "rewards//mean": 0.76470947265625, "rewards//std": 0.02923784777522087, "step": 1021 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2044, "grad_norm": 8.341014862060547, "kl": 4.192100854590535, "learning_rate": 9.080229466393749e-07, "loss": 0.1677, "num_tokens": 8835024.0, "reward": 0.76715087890625, "reward_std": 0.007700664922595024, "rewards//mean": 0.76715087890625, "rewards//std": 0.02517741173505783, "step": 1022 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2046, "grad_norm": 10.33596420288086, "kl": 6.088332811370492, "learning_rate": 9.078394503283508e-07, "loss": 0.2435, "num_tokens": 8843712.0, "reward": 0.7244873046875, "reward_std": 0.010851889848709106, "rewards//mean": 0.7244873046875, "rewards//std": 0.024832110852003098, "step": 1023 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2048, "grad_norm": 13.496170043945312, "kl": 2.899243412539363, "learning_rate": 9.076557897396451e-07, "loss": 0.116, "num_tokens": 8852368.0, "reward": 0.7635498046875, "reward_std": 0.012002428993582726, "rewards//mean": 0.7635498046875, "rewards//std": 0.022487880662083626, "step": 1024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.205, "grad_norm": 16.10167694091797, "kl": 6.5625031385570765, "learning_rate": 9.074719649472357e-07, "loss": 0.2625, "num_tokens": 8860928.0, "reward": 0.7440185546875, "reward_std": 0.014959938824176788, "rewards//mean": 0.7440185546875, "rewards//std": 0.03191693499684334, "step": 1025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2052, "grad_norm": 8.42684268951416, "kl": 3.2908788062632084, "learning_rate": 9.072879760251679e-07, "loss": 0.1316, "num_tokens": 8869624.0, "reward": 0.75390625, "reward_std": 0.009384368546307087, "rewards//mean": 0.75390625, "rewards//std": 0.028237072750926018, "step": 1026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2054, "grad_norm": 5.730315208435059, "kl": 5.489510273560882, "learning_rate": 9.071038230475519e-07, "loss": 0.2196, "num_tokens": 8878224.0, "reward": 0.79742431640625, "reward_std": 0.018564797937870026, "rewards//mean": 0.79742431640625, "rewards//std": 0.03276246786117554, "step": 1027 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2056, "grad_norm": 4.618313789367676, "kl": 6.042632514610887, "learning_rate": 9.069195060885646e-07, "loss": 0.2417, "num_tokens": 8886832.0, "reward": 0.7794189453125, "reward_std": 0.016224637627601624, "rewards//mean": 0.7794189453125, "rewards//std": 0.027339044958353043, "step": 1028 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2058, "grad_norm": 13.91735553741455, "kl": 2.8623254615813494, "learning_rate": 9.067350252224489e-07, "loss": 0.1145, "num_tokens": 8895536.0, "reward": 0.76702880859375, "reward_std": 0.011839576065540314, "rewards//mean": 0.76702880859375, "rewards//std": 0.022567424923181534, "step": 1029 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.206, "grad_norm": 14.280287742614746, "kl": 9.39994865655899, "learning_rate": 9.065503805235137e-07, "loss": 0.376, "num_tokens": 8904104.0, "reward": 0.74761962890625, "reward_std": 0.015248171985149384, "rewards//mean": 0.74761962890625, "rewards//std": 0.034334514290094376, "step": 1030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2062, "grad_norm": 14.15838623046875, "kl": 7.228221353143454, "learning_rate": 9.06365572066134e-07, "loss": 0.2891, "num_tokens": 8912784.0, "reward": 0.7474365234375, "reward_std": 0.010489135049283504, "rewards//mean": 0.7474365234375, "rewards//std": 0.023737527430057526, "step": 1031 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2064, "grad_norm": 10.072168350219727, "kl": 7.966044461354613, "learning_rate": 9.061805999247503e-07, "loss": 0.3186, "num_tokens": 8921536.0, "reward": 0.7689208984375, "reward_std": 0.017396284267306328, "rewards//mean": 0.7689208984375, "rewards//std": 0.03338748589158058, "step": 1032 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2066, "grad_norm": 13.820401191711426, "kl": 9.57500672340393, "learning_rate": 9.059954641738697e-07, "loss": 0.383, "num_tokens": 8930136.0, "reward": 0.72674560546875, "reward_std": 0.01634276658296585, "rewards//mean": 0.72674560546875, "rewards//std": 0.031121307983994484, "step": 1033 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2068, "grad_norm": 14.01988410949707, "kl": 7.840656926855445, "learning_rate": 9.058101648880645e-07, "loss": 0.3136, "num_tokens": 8938864.0, "reward": 0.7684326171875, "reward_std": 0.012227555736899376, "rewards//mean": 0.7684326171875, "rewards//std": 0.0253867506980896, "step": 1034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.207, "grad_norm": 10.134082794189453, "kl": 5.084686573594809, "learning_rate": 9.056247021419734e-07, "loss": 0.2034, "num_tokens": 8947568.0, "reward": 0.75775146484375, "reward_std": 0.017827101051807404, "rewards//mean": 0.75775146484375, "rewards//std": 0.031400710344314575, "step": 1035 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2072, "grad_norm": 11.32555103302002, "kl": 4.593394014984369, "learning_rate": 9.054390760103009e-07, "loss": 0.1837, "num_tokens": 8956272.0, "reward": 0.7672119140625, "reward_std": 0.017249010503292084, "rewards//mean": 0.7672119140625, "rewards//std": 0.02907150238752365, "step": 1036 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2074, "grad_norm": 14.796903610229492, "kl": 7.180980386212468, "learning_rate": 9.052532865678171e-07, "loss": 0.2872, "num_tokens": 8964968.0, "reward": 0.757080078125, "reward_std": 0.018639879301190376, "rewards//mean": 0.757080078125, "rewards//std": 0.036210861057043076, "step": 1037 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2076, "grad_norm": 13.861976623535156, "kl": 7.610343635082245, "learning_rate": 9.050673338893577e-07, "loss": 0.3044, "num_tokens": 8973680.0, "reward": 0.7294921875, "reward_std": 0.024282868951559067, "rewards//mean": 0.7294921875, "rewards//std": 0.04542652890086174, "step": 1038 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2078, "grad_norm": 8.83830451965332, "kl": 6.84739294461906, "learning_rate": 9.04881218049825e-07, "loss": 0.2739, "num_tokens": 8982352.0, "reward": 0.7283935546875, "reward_std": 0.018906250596046448, "rewards//mean": 0.7283935546875, "rewards//std": 0.03308689221739769, "step": 1039 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.208, "grad_norm": 12.931081771850586, "kl": 3.712457677349448, "learning_rate": 9.046949391241858e-07, "loss": 0.1485, "num_tokens": 8991008.0, "reward": 0.72955322265625, "reward_std": 0.01683615706861019, "rewards//mean": 0.72955322265625, "rewards//std": 0.03612961992621422, "step": 1040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2082, "grad_norm": 9.187896728515625, "kl": 7.100620772689581, "learning_rate": 9.045084971874737e-07, "loss": 0.284, "num_tokens": 8999736.0, "reward": 0.7667236328125, "reward_std": 0.016009004786610603, "rewards//mean": 0.7667236328125, "rewards//std": 0.03237278759479523, "step": 1041 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2084, "grad_norm": 5.4558892250061035, "kl": 3.412343100644648, "learning_rate": 9.043218923147873e-07, "loss": 0.1365, "num_tokens": 9008384.0, "reward": 0.73309326171875, "reward_std": 0.013455754145979881, "rewards//mean": 0.73309326171875, "rewards//std": 0.0391681045293808, "step": 1042 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2086, "grad_norm": 7.426380157470703, "kl": 4.572388235479593, "learning_rate": 9.04135124581291e-07, "loss": 0.1829, "num_tokens": 9017096.0, "reward": 0.74603271484375, "reward_std": 0.01465071365237236, "rewards//mean": 0.74603271484375, "rewards//std": 0.030461056157946587, "step": 1043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2088, "grad_norm": 8.59954833984375, "kl": 4.7474064864218235, "learning_rate": 9.039481940622146e-07, "loss": 0.1899, "num_tokens": 9025688.0, "reward": 0.74676513671875, "reward_std": 0.01007864996790886, "rewards//mean": 0.74676513671875, "rewards//std": 0.020723234862089157, "step": 1044 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.209, "grad_norm": 14.09008502960205, "kl": 3.974072055891156, "learning_rate": 9.037611008328543e-07, "loss": 0.159, "num_tokens": 9034344.0, "reward": 0.75958251953125, "reward_std": 0.017579447478055954, "rewards//mean": 0.75958251953125, "rewards//std": 0.040504928678274155, "step": 1045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2092, "grad_norm": 12.411527633666992, "kl": 4.509534182026982, "learning_rate": 9.035738449685706e-07, "loss": 0.1804, "num_tokens": 9043032.0, "reward": 0.745361328125, "reward_std": 0.020337410271167755, "rewards//mean": 0.745361328125, "rewards//std": 0.043262965977191925, "step": 1046 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2094, "grad_norm": 13.162151336669922, "kl": 3.1250076573342085, "learning_rate": 9.033864265447906e-07, "loss": 0.125, "num_tokens": 9051672.0, "reward": 0.7392578125, "reward_std": 0.019312258809804916, "rewards//mean": 0.7392578125, "rewards//std": 0.02981417626142502, "step": 1047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2096, "grad_norm": 10.181493759155273, "kl": 1.9324559811502695, "learning_rate": 9.031988456370061e-07, "loss": 0.0773, "num_tokens": 9060280.0, "reward": 0.71868896484375, "reward_std": 0.012040507048368454, "rewards//mean": 0.71868896484375, "rewards//std": 0.028852174058556557, "step": 1048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2098, "grad_norm": 13.57272720336914, "kl": 2.8750363830477, "learning_rate": 9.030111023207749e-07, "loss": 0.115, "num_tokens": 9068992.0, "reward": 0.732177734375, "reward_std": 0.013641199097037315, "rewards//mean": 0.732177734375, "rewards//std": 0.029122747480869293, "step": 1049 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.21, "grad_norm": 11.630790710449219, "kl": 2.7353729140013456, "learning_rate": 9.028231966717198e-07, "loss": 0.1094, "num_tokens": 9077648.0, "reward": 0.76727294921875, "reward_std": 0.01553562842309475, "rewards//mean": 0.76727294921875, "rewards//std": 0.03606965392827988, "step": 1050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2102, "grad_norm": 6.212285041809082, "kl": 4.770133439451456, "learning_rate": 9.026351287655293e-07, "loss": 0.1908, "num_tokens": 9086304.0, "reward": 0.76593017578125, "reward_std": 0.01928180642426014, "rewards//mean": 0.76593017578125, "rewards//std": 0.027439288794994354, "step": 1051 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2104, "grad_norm": 14.253013610839844, "kl": 2.6883560456335545, "learning_rate": 9.02446898677957e-07, "loss": 0.1075, "num_tokens": 9094952.0, "reward": 0.77740478515625, "reward_std": 0.007559204939752817, "rewards//mean": 0.77740478515625, "rewards//std": 0.01712612248957157, "step": 1052 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2106, "grad_norm": 7.245607376098633, "kl": 2.7295682933181524, "learning_rate": 9.02258506484822e-07, "loss": 0.1092, "num_tokens": 9103560.0, "reward": 0.7664794921875, "reward_std": 0.009524633176624775, "rewards//mean": 0.7664794921875, "rewards//std": 0.02959785796701908, "step": 1053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2108, "grad_norm": 7.491701126098633, "kl": 3.3947287648916245, "learning_rate": 9.02069952262009e-07, "loss": 0.1358, "num_tokens": 9112192.0, "reward": 0.7398681640625, "reward_std": 0.012579179368913174, "rewards//mean": 0.7398681640625, "rewards//std": 0.0433906652033329, "step": 1054 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.211, "grad_norm": 11.61394214630127, "kl": 7.261725638061762, "learning_rate": 9.018812360854671e-07, "loss": 0.2905, "num_tokens": 9120840.0, "reward": 0.7535400390625, "reward_std": 0.021491069346666336, "rewards//mean": 0.7535400390625, "rewards//std": 0.03429994359612465, "step": 1055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2112, "grad_norm": 7.258157730102539, "kl": 5.469349566847086, "learning_rate": 9.016923580312113e-07, "loss": 0.2188, "num_tokens": 9129488.0, "reward": 0.7584228515625, "reward_std": 0.011703206226229668, "rewards//mean": 0.7584228515625, "rewards//std": 0.03139088675379753, "step": 1056 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2114, "grad_norm": 7.042768955230713, "kl": 6.2929673083126545, "learning_rate": 9.015033181753218e-07, "loss": 0.2517, "num_tokens": 9138040.0, "reward": 0.75927734375, "reward_std": 0.017466634511947632, "rewards//mean": 0.75927734375, "rewards//std": 0.033592019230127335, "step": 1057 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2116, "grad_norm": 4.279386520385742, "kl": 4.2685501947999, "learning_rate": 9.013141165939438e-07, "loss": 0.1707, "num_tokens": 9146648.0, "reward": 0.77880859375, "reward_std": 0.013631336390972137, "rewards//mean": 0.77880859375, "rewards//std": 0.022846169769763947, "step": 1058 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2118, "grad_norm": 6.588010311126709, "kl": 6.117425901815295, "learning_rate": 9.011247533632875e-07, "loss": 0.2447, "num_tokens": 9155216.0, "reward": 0.73260498046875, "reward_std": 0.01622149348258972, "rewards//mean": 0.73260498046875, "rewards//std": 0.03554288670420647, "step": 1059 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.212, "grad_norm": 12.971135139465332, "kl": 4.657304907217622, "learning_rate": 9.009352285596285e-07, "loss": 0.1863, "num_tokens": 9163848.0, "reward": 0.7452392578125, "reward_std": 0.009879851713776588, "rewards//mean": 0.7452392578125, "rewards//std": 0.026249809190630913, "step": 1060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2122, "grad_norm": 16.559602737426758, "kl": 9.081853482872248, "learning_rate": 9.007455422593075e-07, "loss": 0.3633, "num_tokens": 9172520.0, "reward": 0.77069091796875, "reward_std": 0.013468243181705475, "rewards//mean": 0.77069091796875, "rewards//std": 0.03465792536735535, "step": 1061 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2124, "grad_norm": 13.66545295715332, "kl": 9.093273393809795, "learning_rate": 9.0055569453873e-07, "loss": 0.3637, "num_tokens": 9181096.0, "reward": 0.75872802734375, "reward_std": 0.02095349133014679, "rewards//mean": 0.75872802734375, "rewards//std": 0.029730726033449173, "step": 1062 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2126, "grad_norm": 14.912736892700195, "kl": 8.069859202951193, "learning_rate": 9.003656854743666e-07, "loss": 0.3228, "num_tokens": 9189752.0, "reward": 0.72222900390625, "reward_std": 0.013820171356201172, "rewards//mean": 0.72222900390625, "rewards//std": 0.04395302012562752, "step": 1063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2128, "grad_norm": 4.15281343460083, "kl": 5.782654831185937, "learning_rate": 9.00175515142753e-07, "loss": 0.2313, "num_tokens": 9198392.0, "reward": 0.735595703125, "reward_std": 0.01619836874306202, "rewards//mean": 0.735595703125, "rewards//std": 0.0386064350605011, "step": 1064 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.213, "grad_norm": 5.644056797027588, "kl": 4.852701520547271, "learning_rate": 8.9998518362049e-07, "loss": 0.1941, "num_tokens": 9207120.0, "reward": 0.75128173828125, "reward_std": 0.01600177399814129, "rewards//mean": 0.75128173828125, "rewards//std": 0.02943856082856655, "step": 1065 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2132, "grad_norm": 13.419784545898438, "kl": 3.319156812503934, "learning_rate": 8.997946909842424e-07, "loss": 0.1328, "num_tokens": 9215808.0, "reward": 0.7642822265625, "reward_std": 0.01321438979357481, "rewards//mean": 0.7642822265625, "rewards//std": 0.025038518011569977, "step": 1066 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2134, "grad_norm": 7.289127826690674, "kl": 5.853924106806517, "learning_rate": 8.996040373107414e-07, "loss": 0.2342, "num_tokens": 9224600.0, "reward": 0.756591796875, "reward_std": 0.011727886274456978, "rewards//mean": 0.756591796875, "rewards//std": 0.033392246812582016, "step": 1067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2136, "grad_norm": 6.986499786376953, "kl": 4.16066362336278, "learning_rate": 8.994132226767819e-07, "loss": 0.1664, "num_tokens": 9233272.0, "reward": 0.76837158203125, "reward_std": 0.015098395757377148, "rewards//mean": 0.76837158203125, "rewards//std": 0.04439473897218704, "step": 1068 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2138, "grad_norm": 15.410289764404297, "kl": 6.831299565732479, "learning_rate": 8.992222471592239e-07, "loss": 0.2733, "num_tokens": 9241864.0, "reward": 0.76251220703125, "reward_std": 0.013419141992926598, "rewards//mean": 0.76251220703125, "rewards//std": 0.03551732376217842, "step": 1069 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.214, "grad_norm": 6.315592288970947, "kl": 3.5881161894649267, "learning_rate": 8.990311108349926e-07, "loss": 0.1435, "num_tokens": 9250568.0, "reward": 0.764892578125, "reward_std": 0.012547735124826431, "rewards//mean": 0.764892578125, "rewards//std": 0.03994462266564369, "step": 1070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2142, "grad_norm": 5.48075008392334, "kl": 6.089798431843519, "learning_rate": 8.988398137810776e-07, "loss": 0.2436, "num_tokens": 9259280.0, "reward": 0.750732421875, "reward_std": 0.012852808460593224, "rewards//mean": 0.750732421875, "rewards//std": 0.018200911581516266, "step": 1071 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2144, "grad_norm": 6.696509838104248, "kl": 5.232636630535126, "learning_rate": 8.986483560745333e-07, "loss": 0.2093, "num_tokens": 9267928.0, "reward": 0.73919677734375, "reward_std": 0.01306705642491579, "rewards//mean": 0.73919677734375, "rewards//std": 0.02292281948029995, "step": 1072 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2146, "grad_norm": 11.351648330688477, "kl": 3.9859825875610113, "learning_rate": 8.984567377924789e-07, "loss": 0.1594, "num_tokens": 9276600.0, "reward": 0.7408447265625, "reward_std": 0.010947344824671745, "rewards//mean": 0.7408447265625, "rewards//std": 0.03646643087267876, "step": 1073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2148, "grad_norm": 4.902538776397705, "kl": 4.923521373420954, "learning_rate": 8.982649590120981e-07, "loss": 0.1969, "num_tokens": 9285168.0, "reward": 0.7689208984375, "reward_std": 0.02040485292673111, "rewards//mean": 0.7689208984375, "rewards//std": 0.039611831307411194, "step": 1074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.215, "grad_norm": 9.982236862182617, "kl": 4.026648405939341, "learning_rate": 8.980730198106394e-07, "loss": 0.1611, "num_tokens": 9293880.0, "reward": 0.7222900390625, "reward_std": 0.01029210351407528, "rewards//mean": 0.7222900390625, "rewards//std": 0.033063095062971115, "step": 1075 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2152, "grad_norm": 5.773685455322266, "kl": 5.7916702423244715, "learning_rate": 8.97880920265416e-07, "loss": 0.2317, "num_tokens": 9302696.0, "reward": 0.75469970703125, "reward_std": 0.014176889322698116, "rewards//mean": 0.75469970703125, "rewards//std": 0.032497573643922806, "step": 1076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2154, "grad_norm": 7.009709358215332, "kl": 2.574088655412197, "learning_rate": 8.976886604538055e-07, "loss": 0.103, "num_tokens": 9311360.0, "reward": 0.77276611328125, "reward_std": 0.01645975187420845, "rewards//mean": 0.77276611328125, "rewards//std": 0.034481894224882126, "step": 1077 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2156, "grad_norm": 7.832144260406494, "kl": 3.6851198878139257, "learning_rate": 8.974962404532501e-07, "loss": 0.1474, "num_tokens": 9320040.0, "reward": 0.77398681640625, "reward_std": 0.01394729409366846, "rewards//mean": 0.77398681640625, "rewards//std": 0.02839844487607479, "step": 1078 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2158, "grad_norm": 5.432318687438965, "kl": 4.033744588494301, "learning_rate": 8.973036603412566e-07, "loss": 0.1613, "num_tokens": 9328624.0, "reward": 0.77685546875, "reward_std": 0.010864097625017166, "rewards//mean": 0.77685546875, "rewards//std": 0.02566222846508026, "step": 1079 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.216, "grad_norm": 7.712913990020752, "kl": 5.05657709389925, "learning_rate": 8.971109201953962e-07, "loss": 0.2023, "num_tokens": 9337216.0, "reward": 0.7747802734375, "reward_std": 0.018777944147586823, "rewards//mean": 0.7747802734375, "rewards//std": 0.028647683560848236, "step": 1080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2162, "grad_norm": 7.410307884216309, "kl": 4.115938827395439, "learning_rate": 8.969180200933047e-07, "loss": 0.1646, "num_tokens": 9345800.0, "reward": 0.76898193359375, "reward_std": 0.020733792334794998, "rewards//mean": 0.76898193359375, "rewards//std": 0.03307882696390152, "step": 1081 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2164, "grad_norm": 4.52678918838501, "kl": 3.3587297461926937, "learning_rate": 8.967249601126821e-07, "loss": 0.1343, "num_tokens": 9354368.0, "reward": 0.76116943359375, "reward_std": 0.014119971543550491, "rewards//mean": 0.76116943359375, "rewards//std": 0.04334123060107231, "step": 1082 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2166, "grad_norm": 4.825457572937012, "kl": 5.383301317691803, "learning_rate": 8.96531740331293e-07, "loss": 0.2153, "num_tokens": 9363000.0, "reward": 0.76666259765625, "reward_std": 0.014202171936631203, "rewards//mean": 0.76666259765625, "rewards//std": 0.02980801835656166, "step": 1083 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2168, "grad_norm": 8.467222213745117, "kl": 6.032531779259443, "learning_rate": 8.963383608269663e-07, "loss": 0.2413, "num_tokens": 9371672.0, "reward": 0.753173828125, "reward_std": 0.012710340321063995, "rewards//mean": 0.753173828125, "rewards//std": 0.021919630467891693, "step": 1084 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.217, "grad_norm": 4.363509654998779, "kl": 5.882176134735346, "learning_rate": 8.961448216775953e-07, "loss": 0.2353, "num_tokens": 9380344.0, "reward": 0.74407958984375, "reward_std": 0.0107672568410635, "rewards//mean": 0.74407958984375, "rewards//std": 0.024567583575844765, "step": 1085 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2172, "grad_norm": 19.103225708007812, "kl": 10.323764387518167, "learning_rate": 8.959511229611375e-07, "loss": 0.413, "num_tokens": 9389040.0, "reward": 0.76043701171875, "reward_std": 0.019833790138363838, "rewards//mean": 0.76043701171875, "rewards//std": 0.027805369347333908, "step": 1086 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2174, "grad_norm": 11.866991996765137, "kl": 6.114652732387185, "learning_rate": 8.957572647556147e-07, "loss": 0.2446, "num_tokens": 9397592.0, "reward": 0.7386474609375, "reward_std": 0.013309924863278866, "rewards//mean": 0.7386474609375, "rewards//std": 0.029476908966898918, "step": 1087 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2176, "grad_norm": 5.4716410636901855, "kl": 7.496361957862973, "learning_rate": 8.95563247139113e-07, "loss": 0.2999, "num_tokens": 9406272.0, "reward": 0.75762939453125, "reward_std": 0.0194938275963068, "rewards//mean": 0.75762939453125, "rewards//std": 0.039723023772239685, "step": 1088 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2178, "grad_norm": 11.618474960327148, "kl": 4.0657080095261335, "learning_rate": 8.953690701897827e-07, "loss": 0.1626, "num_tokens": 9414848.0, "reward": 0.7457275390625, "reward_std": 0.021586135029792786, "rewards//mean": 0.7457275390625, "rewards//std": 0.029965855181217194, "step": 1089 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.218, "grad_norm": 12.74976634979248, "kl": 7.561018029227853, "learning_rate": 8.951747339858382e-07, "loss": 0.3024, "num_tokens": 9423448.0, "reward": 0.728271484375, "reward_std": 0.009536255151033401, "rewards//mean": 0.728271484375, "rewards//std": 0.02986186370253563, "step": 1090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2182, "grad_norm": 4.4736480712890625, "kl": 4.955842969939113, "learning_rate": 8.94980238605558e-07, "loss": 0.1982, "num_tokens": 9432040.0, "reward": 0.75628662109375, "reward_std": 0.012968906201422215, "rewards//mean": 0.75628662109375, "rewards//std": 0.02879440411925316, "step": 1091 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2184, "grad_norm": 6.511483669281006, "kl": 4.5022532027214766, "learning_rate": 8.947855841272851e-07, "loss": 0.1801, "num_tokens": 9440632.0, "reward": 0.76416015625, "reward_std": 0.007533428259193897, "rewards//mean": 0.76416015625, "rewards//std": 0.023025689646601677, "step": 1092 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2186, "grad_norm": 15.795879364013672, "kl": 9.1130366679281, "learning_rate": 8.94590770629426e-07, "loss": 0.3645, "num_tokens": 9449312.0, "reward": 0.69879150390625, "reward_std": 0.015679676085710526, "rewards//mean": 0.69879150390625, "rewards//std": 0.0388331413269043, "step": 1093 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2188, "grad_norm": 4.858828067779541, "kl": 3.203987028449774, "learning_rate": 8.943957981904517e-07, "loss": 0.1282, "num_tokens": 9458032.0, "reward": 0.763671875, "reward_std": 0.006650151684880257, "rewards//mean": 0.763671875, "rewards//std": 0.026097411289811134, "step": 1094 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.219, "grad_norm": 5.840320587158203, "kl": 3.567983377724886, "learning_rate": 8.942006668888971e-07, "loss": 0.1427, "num_tokens": 9466552.0, "reward": 0.75341796875, "reward_std": 0.00962599366903305, "rewards//mean": 0.75341796875, "rewards//std": 0.025887753814458847, "step": 1095 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2192, "grad_norm": 9.678954124450684, "kl": 6.329554399475455, "learning_rate": 8.940053768033608e-07, "loss": 0.2532, "num_tokens": 9475248.0, "reward": 0.744384765625, "reward_std": 0.01467149518430233, "rewards//mean": 0.744384765625, "rewards//std": 0.036906491965055466, "step": 1096 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2194, "grad_norm": 5.616601943969727, "kl": 6.981848675757647, "learning_rate": 8.938099280125062e-07, "loss": 0.2793, "num_tokens": 9483984.0, "reward": 0.77020263671875, "reward_std": 0.023348022252321243, "rewards//mean": 0.77020263671875, "rewards//std": 0.03965055197477341, "step": 1097 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2196, "grad_norm": 8.251206398010254, "kl": 5.501217184588313, "learning_rate": 8.936143205950595e-07, "loss": 0.22, "num_tokens": 9492560.0, "reward": 0.7432861328125, "reward_std": 0.014951873570680618, "rewards//mean": 0.7432861328125, "rewards//std": 0.03821925073862076, "step": 1098 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2198, "grad_norm": 5.047215938568115, "kl": 4.080555723980069, "learning_rate": 8.934185546298115e-07, "loss": 0.1632, "num_tokens": 9501136.0, "reward": 0.70928955078125, "reward_std": 0.01419641263782978, "rewards//mean": 0.70928955078125, "rewards//std": 0.04334646835923195, "step": 1099 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.22, "grad_norm": 8.520955085754395, "kl": 4.37362390384078, "learning_rate": 8.932226301956169e-07, "loss": 0.1749, "num_tokens": 9509816.0, "reward": 0.74566650390625, "reward_std": 0.015053149312734604, "rewards//mean": 0.74566650390625, "rewards//std": 0.02922438271343708, "step": 1100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2202, "grad_norm": 15.754254341125488, "kl": 6.733986398205161, "learning_rate": 8.930265473713937e-07, "loss": 0.2694, "num_tokens": 9518472.0, "reward": 0.75933837890625, "reward_std": 0.017398901283740997, "rewards//mean": 0.75933837890625, "rewards//std": 0.04254456236958504, "step": 1101 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2204, "grad_norm": 6.832894802093506, "kl": 3.5568563863635063, "learning_rate": 8.928303062361243e-07, "loss": 0.1423, "num_tokens": 9527136.0, "reward": 0.7445068359375, "reward_std": 0.009682252071797848, "rewards//mean": 0.7445068359375, "rewards//std": 0.03515086695551872, "step": 1102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2206, "grad_norm": 6.307701110839844, "kl": 3.7589014023542404, "learning_rate": 8.926339068688545e-07, "loss": 0.1504, "num_tokens": 9535776.0, "reward": 0.75201416015625, "reward_std": 0.01643678918480873, "rewards//mean": 0.75201416015625, "rewards//std": 0.039432723075151443, "step": 1103 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2208, "grad_norm": 9.606477737426758, "kl": 3.097434274852276, "learning_rate": 8.924373493486941e-07, "loss": 0.1239, "num_tokens": 9544424.0, "reward": 0.7449951171875, "reward_std": 0.008526843972504139, "rewards//mean": 0.7449951171875, "rewards//std": 0.03196622058749199, "step": 1104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.221, "grad_norm": 9.570741653442383, "kl": 4.895392829552293, "learning_rate": 8.922406337548161e-07, "loss": 0.1958, "num_tokens": 9553032.0, "reward": 0.71478271484375, "reward_std": 0.011852547526359558, "rewards//mean": 0.71478271484375, "rewards//std": 0.024934541434049606, "step": 1105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2212, "grad_norm": 13.413549423217773, "kl": 7.705706216394901, "learning_rate": 8.920437601664579e-07, "loss": 0.3082, "num_tokens": 9561712.0, "reward": 0.7745361328125, "reward_std": 0.011381777003407478, "rewards//mean": 0.7745361328125, "rewards//std": 0.03425578027963638, "step": 1106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2214, "grad_norm": 6.399602890014648, "kl": 2.971532279625535, "learning_rate": 8.918467286629198e-07, "loss": 0.1189, "num_tokens": 9570320.0, "reward": 0.732421875, "reward_std": 0.008850199170410633, "rewards//mean": 0.732421875, "rewards//std": 0.03154314309358597, "step": 1107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2216, "grad_norm": 10.602251052856445, "kl": 3.5051215067505836, "learning_rate": 8.916495393235665e-07, "loss": 0.1402, "num_tokens": 9578848.0, "reward": 0.748779296875, "reward_std": 0.014799104072153568, "rewards//mean": 0.748779296875, "rewards//std": 0.037011343985795975, "step": 1108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2218, "grad_norm": 4.765389442443848, "kl": 5.119537103921175, "learning_rate": 8.914521922278255e-07, "loss": 0.2048, "num_tokens": 9587480.0, "reward": 0.73248291015625, "reward_std": 0.010286146774888039, "rewards//mean": 0.73248291015625, "rewards//std": 0.030241606757044792, "step": 1109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.222, "grad_norm": 11.750473022460938, "kl": 3.9613872449845076, "learning_rate": 8.912546874551882e-07, "loss": 0.1585, "num_tokens": 9596152.0, "reward": 0.74652099609375, "reward_std": 0.006439933553338051, "rewards//mean": 0.74652099609375, "rewards//std": 0.03288974612951279, "step": 1110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2222, "grad_norm": 14.945819854736328, "kl": 7.06269839219749, "learning_rate": 8.910570250852096e-07, "loss": 0.2825, "num_tokens": 9604800.0, "reward": 0.73834228515625, "reward_std": 0.01386193186044693, "rewards//mean": 0.73834228515625, "rewards//std": 0.043678704649209976, "step": 1111 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2224, "grad_norm": 5.244224548339844, "kl": 4.3932148311287165, "learning_rate": 8.908592051975081e-07, "loss": 0.1757, "num_tokens": 9613480.0, "reward": 0.7559814453125, "reward_std": 0.013672607019543648, "rewards//mean": 0.7559814453125, "rewards//std": 0.02797829546034336, "step": 1112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2226, "grad_norm": 8.444457054138184, "kl": 3.298034707084298, "learning_rate": 8.906612278717655e-07, "loss": 0.1319, "num_tokens": 9622088.0, "reward": 0.73394775390625, "reward_std": 0.014404860325157642, "rewards//mean": 0.73394775390625, "rewards//std": 0.038185324519872665, "step": 1113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2228, "grad_norm": 12.326435089111328, "kl": 5.131952002644539, "learning_rate": 8.90463093187727e-07, "loss": 0.2053, "num_tokens": 9630704.0, "reward": 0.762451171875, "reward_std": 0.01745082065463066, "rewards//mean": 0.762451171875, "rewards//std": 0.025881905108690262, "step": 1114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.223, "grad_norm": 7.397994041442871, "kl": 2.4171334765851498, "learning_rate": 8.902648012252012e-07, "loss": 0.0967, "num_tokens": 9639360.0, "reward": 0.7742919921875, "reward_std": 0.00961617287248373, "rewards//mean": 0.7742919921875, "rewards//std": 0.024601832032203674, "step": 1115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2232, "grad_norm": 6.574716091156006, "kl": 4.173199244774878, "learning_rate": 8.900663520640603e-07, "loss": 0.1669, "num_tokens": 9647976.0, "reward": 0.73150634765625, "reward_std": 0.014405167661607265, "rewards//mean": 0.73150634765625, "rewards//std": 0.03448672220110893, "step": 1116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2234, "grad_norm": 7.505921840667725, "kl": 2.7678875029087067, "learning_rate": 8.898677457842394e-07, "loss": 0.1107, "num_tokens": 9656608.0, "reward": 0.774169921875, "reward_std": 0.015123208984732628, "rewards//mean": 0.774169921875, "rewards//std": 0.029796907678246498, "step": 1117 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2236, "grad_norm": 6.814366817474365, "kl": 5.7939563523977995, "learning_rate": 8.896689824657371e-07, "loss": 0.2318, "num_tokens": 9665184.0, "reward": 0.75482177734375, "reward_std": 0.01692846789956093, "rewards//mean": 0.75482177734375, "rewards//std": 0.03000643104314804, "step": 1118 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2238, "grad_norm": 9.690752029418945, "kl": 6.7204610873013735, "learning_rate": 8.894700621886152e-07, "loss": 0.2688, "num_tokens": 9673856.0, "reward": 0.7537841796875, "reward_std": 0.01766577549278736, "rewards//mean": 0.7537841796875, "rewards//std": 0.039173755794763565, "step": 1119 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.224, "grad_norm": 16.34100914001465, "kl": 3.3960287645459175, "learning_rate": 8.892709850329989e-07, "loss": 0.1358, "num_tokens": 9682448.0, "reward": 0.7314453125, "reward_std": 0.011177017353475094, "rewards//mean": 0.7314453125, "rewards//std": 0.03624512627720833, "step": 1120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2242, "grad_norm": 8.458902359008789, "kl": 4.556045098230243, "learning_rate": 8.890717510790762e-07, "loss": 0.1822, "num_tokens": 9691064.0, "reward": 0.74200439453125, "reward_std": 0.00906654354184866, "rewards//mean": 0.74200439453125, "rewards//std": 0.026697741821408272, "step": 1121 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2244, "grad_norm": 12.766468048095703, "kl": 5.043446036055684, "learning_rate": 8.888723604070989e-07, "loss": 0.2017, "num_tokens": 9699704.0, "reward": 0.76190185546875, "reward_std": 0.009677472524344921, "rewards//mean": 0.76190185546875, "rewards//std": 0.03258131071925163, "step": 1122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2246, "grad_norm": 8.831002235412598, "kl": 3.949399497359991, "learning_rate": 8.886728130973813e-07, "loss": 0.158, "num_tokens": 9708296.0, "reward": 0.77288818359375, "reward_std": 0.012819640338420868, "rewards//mean": 0.77288818359375, "rewards//std": 0.028813323006033897, "step": 1123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2248, "grad_norm": 4.6322197914123535, "kl": 4.173149090260267, "learning_rate": 8.884731092303011e-07, "loss": 0.1669, "num_tokens": 9717040.0, "reward": 0.760498046875, "reward_std": 0.0113409124314785, "rewards//mean": 0.760498046875, "rewards//std": 0.028964301571249962, "step": 1124 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.225, "grad_norm": 10.083248138427734, "kl": 2.7000562883913517, "learning_rate": 8.882732488862987e-07, "loss": 0.108, "num_tokens": 9725680.0, "reward": 0.73944091796875, "reward_std": 0.008040301501750946, "rewards//mean": 0.73944091796875, "rewards//std": 0.03297478333115578, "step": 1125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2252, "grad_norm": 8.444313049316406, "kl": 5.095565229654312, "learning_rate": 8.880732321458784e-07, "loss": 0.2038, "num_tokens": 9734240.0, "reward": 0.77410888671875, "reward_std": 0.012255186215043068, "rewards//mean": 0.77410888671875, "rewards//std": 0.01863071136176586, "step": 1126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2254, "grad_norm": 10.66525650024414, "kl": 2.797660192474723, "learning_rate": 8.878730590896065e-07, "loss": 0.1119, "num_tokens": 9742928.0, "reward": 0.712890625, "reward_std": 0.007319233845919371, "rewards//mean": 0.712890625, "rewards//std": 0.04367043077945709, "step": 1127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2256, "grad_norm": 5.685501575469971, "kl": 2.1741212755441666, "learning_rate": 8.876727297981127e-07, "loss": 0.087, "num_tokens": 9751496.0, "reward": 0.78173828125, "reward_std": 0.009359104558825493, "rewards//mean": 0.78173828125, "rewards//std": 0.02638813480734825, "step": 1128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2258, "grad_norm": 4.537426948547363, "kl": 1.442899452522397, "learning_rate": 8.874722443520898e-07, "loss": 0.0577, "num_tokens": 9760136.0, "reward": 0.77166748046875, "reward_std": 0.011438120156526566, "rewards//mean": 0.77166748046875, "rewards//std": 0.028797032311558723, "step": 1129 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.226, "grad_norm": 6.674938678741455, "kl": 2.905785081908107, "learning_rate": 8.872716028322931e-07, "loss": 0.1162, "num_tokens": 9768864.0, "reward": 0.7750244140625, "reward_std": 0.014743040315806866, "rewards//mean": 0.7750244140625, "rewards//std": 0.03611605241894722, "step": 1130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2262, "grad_norm": 4.763418674468994, "kl": 2.849360417574644, "learning_rate": 8.870708053195413e-07, "loss": 0.114, "num_tokens": 9777504.0, "reward": 0.73541259765625, "reward_std": 0.012042814865708351, "rewards//mean": 0.73541259765625, "rewards//std": 0.041405677795410156, "step": 1131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2264, "grad_norm": 5.136250019073486, "kl": 3.0846480950713158, "learning_rate": 8.868698518947151e-07, "loss": 0.1234, "num_tokens": 9786088.0, "reward": 0.73541259765625, "reward_std": 0.009864496067166328, "rewards//mean": 0.73541259765625, "rewards//std": 0.024873757734894753, "step": 1132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2266, "grad_norm": 8.897073745727539, "kl": 0.7811648137867451, "learning_rate": 8.866687426387591e-07, "loss": 0.0312, "num_tokens": 9794704.0, "reward": 0.76324462890625, "reward_std": 0.00455887708812952, "rewards//mean": 0.76324462890625, "rewards//std": 0.030118217691779137, "step": 1133 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2268, "grad_norm": 11.335514068603516, "kl": 4.045972425490618, "learning_rate": 8.864674776326797e-07, "loss": 0.1618, "num_tokens": 9803360.0, "reward": 0.800537109375, "reward_std": 0.006927927955985069, "rewards//mean": 0.800537109375, "rewards//std": 0.02517027035355568, "step": 1134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.227, "grad_norm": 4.795708656311035, "kl": 0.9940475765615702, "learning_rate": 8.862660569575464e-07, "loss": 0.0398, "num_tokens": 9812040.0, "reward": 0.7625732421875, "reward_std": 0.0066719925962388515, "rewards//mean": 0.7625732421875, "rewards//std": 0.02362758480012417, "step": 1135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2272, "grad_norm": 4.9466633796691895, "kl": 4.480608073994517, "learning_rate": 8.860644806944917e-07, "loss": 0.1792, "num_tokens": 9820664.0, "reward": 0.72784423828125, "reward_std": 0.009332980960607529, "rewards//mean": 0.72784423828125, "rewards//std": 0.032309774309396744, "step": 1136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2274, "grad_norm": 4.079766273498535, "kl": 1.916475109755993, "learning_rate": 8.858627489247104e-07, "loss": 0.0767, "num_tokens": 9829264.0, "reward": 0.76568603515625, "reward_std": 0.007418432272970676, "rewards//mean": 0.76568603515625, "rewards//std": 0.024294957518577576, "step": 1137 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2276, "grad_norm": 3.6407711505889893, "kl": 2.09932229667902, "learning_rate": 8.856608617294599e-07, "loss": 0.084, "num_tokens": 9837976.0, "reward": 0.75091552734375, "reward_std": 0.0055526187643408775, "rewards//mean": 0.75091552734375, "rewards//std": 0.030223581939935684, "step": 1138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2278, "grad_norm": 3.612649917602539, "kl": 2.2236559856683016, "learning_rate": 8.854588191900604e-07, "loss": 0.0889, "num_tokens": 9846624.0, "reward": 0.75433349609375, "reward_std": 0.007318149320781231, "rewards//mean": 0.75433349609375, "rewards//std": 0.025412004441022873, "step": 1139 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.228, "grad_norm": 5.206478595733643, "kl": 4.678205693140626, "learning_rate": 8.852566213878946e-07, "loss": 0.1871, "num_tokens": 9855208.0, "reward": 0.75811767578125, "reward_std": 0.012443074025213718, "rewards//mean": 0.75811767578125, "rewards//std": 0.03624047711491585, "step": 1140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2282, "grad_norm": 4.288930892944336, "kl": 2.0811544209718704, "learning_rate": 8.850542684044078e-07, "loss": 0.0832, "num_tokens": 9863816.0, "reward": 0.8023681640625, "reward_std": 0.006106534507125616, "rewards//mean": 0.8023681640625, "rewards//std": 0.0222033578902483, "step": 1141 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2284, "grad_norm": 4.588595867156982, "kl": 3.811111245304346, "learning_rate": 8.848517603211078e-07, "loss": 0.1524, "num_tokens": 9872392.0, "reward": 0.75927734375, "reward_std": 0.010999436490237713, "rewards//mean": 0.75927734375, "rewards//std": 0.02885223925113678, "step": 1142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2286, "grad_norm": 12.965023040771484, "kl": 5.332149356603622, "learning_rate": 8.846490972195646e-07, "loss": 0.2133, "num_tokens": 9881024.0, "reward": 0.73846435546875, "reward_std": 0.008660933934152126, "rewards//mean": 0.73846435546875, "rewards//std": 0.030555827543139458, "step": 1143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2288, "grad_norm": 9.358354568481445, "kl": 4.364532317966223, "learning_rate": 8.844462791814112e-07, "loss": 0.1746, "num_tokens": 9889648.0, "reward": 0.74139404296875, "reward_std": 0.00542416563257575, "rewards//mean": 0.74139404296875, "rewards//std": 0.02432173490524292, "step": 1144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.229, "grad_norm": 5.2452521324157715, "kl": 3.6066288389265537, "learning_rate": 8.842433062883425e-07, "loss": 0.1443, "num_tokens": 9898224.0, "reward": 0.77191162109375, "reward_std": 0.015570230782032013, "rewards//mean": 0.77191162109375, "rewards//std": 0.02311091497540474, "step": 1145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2292, "grad_norm": 10.84929370880127, "kl": 4.584597546607256, "learning_rate": 8.840401786221159e-07, "loss": 0.1834, "num_tokens": 9906824.0, "reward": 0.73260498046875, "reward_std": 0.007782891392707825, "rewards//mean": 0.73260498046875, "rewards//std": 0.027995266020298004, "step": 1146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2294, "grad_norm": 7.70128870010376, "kl": 2.4459972754120827, "learning_rate": 8.838368962645513e-07, "loss": 0.0978, "num_tokens": 9915560.0, "reward": 0.74884033203125, "reward_std": 0.008964098989963531, "rewards//mean": 0.74884033203125, "rewards//std": 0.025767529383301735, "step": 1147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2296, "grad_norm": 3.8446500301361084, "kl": 4.139189610257745, "learning_rate": 8.836334592975308e-07, "loss": 0.1656, "num_tokens": 9924232.0, "reward": 0.71258544921875, "reward_std": 0.010579858906567097, "rewards//mean": 0.71258544921875, "rewards//std": 0.03684691712260246, "step": 1148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2298, "grad_norm": 4.000614643096924, "kl": 4.734907872974873, "learning_rate": 8.834298678029988e-07, "loss": 0.1894, "num_tokens": 9932832.0, "reward": 0.7659912109375, "reward_std": 0.01792335882782936, "rewards//mean": 0.7659912109375, "rewards//std": 0.037889882922172546, "step": 1149 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.23, "grad_norm": 5.773425579071045, "kl": 3.126021906733513, "learning_rate": 8.83226121862962e-07, "loss": 0.125, "num_tokens": 9941384.0, "reward": 0.76690673828125, "reward_std": 0.01594298519194126, "rewards//mean": 0.76690673828125, "rewards//std": 0.030860472470521927, "step": 1150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2302, "grad_norm": 6.255007743835449, "kl": 3.4385843258351088, "learning_rate": 8.83022221559489e-07, "loss": 0.1375, "num_tokens": 9949968.0, "reward": 0.751708984375, "reward_std": 0.011574260890483856, "rewards//mean": 0.751708984375, "rewards//std": 0.024947969242930412, "step": 1151 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2304, "grad_norm": 6.391628265380859, "kl": 3.548680679872632, "learning_rate": 8.82818166974711e-07, "loss": 0.1419, "num_tokens": 9958640.0, "reward": 0.77655029296875, "reward_std": 0.012465916574001312, "rewards//mean": 0.77655029296875, "rewards//std": 0.026126904413104057, "step": 1152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2306, "grad_norm": 5.595583438873291, "kl": 7.071510713547468, "learning_rate": 8.826139581908211e-07, "loss": 0.2829, "num_tokens": 9967248.0, "reward": 0.77203369140625, "reward_std": 0.0170577522367239, "rewards//mean": 0.77203369140625, "rewards//std": 0.03440322354435921, "step": 1153 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2308, "grad_norm": 3.3383522033691406, "kl": 2.09860529191792, "learning_rate": 8.824095952900746e-07, "loss": 0.0839, "num_tokens": 9975864.0, "reward": 0.7640380859375, "reward_std": 0.007309587672352791, "rewards//mean": 0.7640380859375, "rewards//std": 0.026944171637296677, "step": 1154 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.231, "grad_norm": 6.839156627655029, "kl": 3.384455820545554, "learning_rate": 8.822050783547889e-07, "loss": 0.1354, "num_tokens": 9984496.0, "reward": 0.78863525390625, "reward_std": 0.016305115073919296, "rewards//mean": 0.78863525390625, "rewards//std": 0.030065901577472687, "step": 1155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2312, "grad_norm": 4.410799503326416, "kl": 1.7257406003773212, "learning_rate": 8.820004074673433e-07, "loss": 0.069, "num_tokens": 9993096.0, "reward": 0.78155517578125, "reward_std": 0.004620138555765152, "rewards//mean": 0.78155517578125, "rewards//std": 0.020591329783201218, "step": 1156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2314, "grad_norm": 13.29317569732666, "kl": 6.0627273842692375, "learning_rate": 8.817955827101792e-07, "loss": 0.2425, "num_tokens": 10001696.0, "reward": 0.7540283203125, "reward_std": 0.008711063303053379, "rewards//mean": 0.7540283203125, "rewards//std": 0.0346529558300972, "step": 1157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2316, "grad_norm": 7.899120330810547, "kl": 2.4596549551934004, "learning_rate": 8.815906041658001e-07, "loss": 0.0984, "num_tokens": 10010312.0, "reward": 0.73455810546875, "reward_std": 0.005697751417756081, "rewards//mean": 0.73455810546875, "rewards//std": 0.023973187431693077, "step": 1158 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2318, "grad_norm": 7.7236785888671875, "kl": 3.558136461302638, "learning_rate": 8.813854719167712e-07, "loss": 0.1423, "num_tokens": 10018944.0, "reward": 0.7491455078125, "reward_std": 0.005457497201859951, "rewards//mean": 0.7491455078125, "rewards//std": 0.02287762239575386, "step": 1159 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.232, "grad_norm": 4.86129903793335, "kl": 3.5358568504452705, "learning_rate": 8.8118018604572e-07, "loss": 0.1414, "num_tokens": 10027520.0, "reward": 0.7706298828125, "reward_std": 0.013745088130235672, "rewards//mean": 0.7706298828125, "rewards//std": 0.032327864319086075, "step": 1160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2322, "grad_norm": 4.532315254211426, "kl": 2.457659238949418, "learning_rate": 8.809747466353355e-07, "loss": 0.0983, "num_tokens": 10036120.0, "reward": 0.77154541015625, "reward_std": 0.01017143577337265, "rewards//mean": 0.77154541015625, "rewards//std": 0.027987154200673103, "step": 1161 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2324, "grad_norm": 13.446959495544434, "kl": 6.195372574031353, "learning_rate": 8.807691537683684e-07, "loss": 0.2478, "num_tokens": 10044704.0, "reward": 0.7376708984375, "reward_std": 0.013385182246565819, "rewards//mean": 0.7376708984375, "rewards//std": 0.03089313581585884, "step": 1162 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2326, "grad_norm": 11.518983840942383, "kl": 1.5299587156623602, "learning_rate": 8.805634075276317e-07, "loss": 0.0612, "num_tokens": 10053336.0, "reward": 0.7630615234375, "reward_std": 0.006102503277361393, "rewards//mean": 0.7630615234375, "rewards//std": 0.02801506221294403, "step": 1163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2328, "grad_norm": 3.117992401123047, "kl": 2.9440271724015474, "learning_rate": 8.80357507996e-07, "loss": 0.1178, "num_tokens": 10061936.0, "reward": 0.797119140625, "reward_std": 0.008773138746619225, "rewards//mean": 0.797119140625, "rewards//std": 0.019610146060585976, "step": 1164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.233, "grad_norm": 3.4370851516723633, "kl": 2.1740373596549034, "learning_rate": 8.801514552564095e-07, "loss": 0.087, "num_tokens": 10070680.0, "reward": 0.78009033203125, "reward_std": 0.007442031055688858, "rewards//mean": 0.78009033203125, "rewards//std": 0.024653080850839615, "step": 1165 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2332, "grad_norm": 27.170595169067383, "kl": 10.168828338384628, "learning_rate": 8.799452493918585e-07, "loss": 0.4068, "num_tokens": 10079240.0, "reward": 0.74139404296875, "reward_std": 0.014152072370052338, "rewards//mean": 0.74139404296875, "rewards//std": 0.03934779018163681, "step": 1166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2334, "grad_norm": 3.7726423740386963, "kl": 1.264550181105733, "learning_rate": 8.797388904854063e-07, "loss": 0.0506, "num_tokens": 10087896.0, "reward": 0.75830078125, "reward_std": 0.0036158605944365263, "rewards//mean": 0.75830078125, "rewards//std": 0.025614995509386063, "step": 1167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2336, "grad_norm": 2.8917274475097656, "kl": 2.9061081781983376, "learning_rate": 8.795323786201745e-07, "loss": 0.1162, "num_tokens": 10096480.0, "reward": 0.7794189453125, "reward_std": 0.009156651794910431, "rewards//mean": 0.7794189453125, "rewards//std": 0.03096948191523552, "step": 1168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2338, "grad_norm": 2.762669086456299, "kl": 2.3426098842173815, "learning_rate": 8.79325713879346e-07, "loss": 0.0937, "num_tokens": 10105048.0, "reward": 0.7904052734375, "reward_std": 0.005530244670808315, "rewards//mean": 0.7904052734375, "rewards//std": 0.01978651061654091, "step": 1169 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.234, "grad_norm": 5.916254997253418, "kl": 6.016498297452927, "learning_rate": 8.791188963461652e-07, "loss": 0.2407, "num_tokens": 10113640.0, "reward": 0.73980712890625, "reward_std": 0.01162217278033495, "rewards//mean": 0.73980712890625, "rewards//std": 0.03229149803519249, "step": 1170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2342, "grad_norm": 6.179854869842529, "kl": 2.633426643908024, "learning_rate": 8.789119261039384e-07, "loss": 0.1053, "num_tokens": 10122320.0, "reward": 0.781005859375, "reward_std": 0.009721913374960423, "rewards//mean": 0.781005859375, "rewards//std": 0.027347072958946228, "step": 1171 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2344, "grad_norm": 11.493363380432129, "kl": 4.576944898813963, "learning_rate": 8.78704803236033e-07, "loss": 0.1831, "num_tokens": 10131064.0, "reward": 0.72662353515625, "reward_std": 0.013162564486265182, "rewards//mean": 0.72662353515625, "rewards//std": 0.022803641855716705, "step": 1172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2346, "grad_norm": 3.578338146209717, "kl": 1.9992002733051777, "learning_rate": 8.784975278258782e-07, "loss": 0.08, "num_tokens": 10139696.0, "reward": 0.73089599609375, "reward_std": 0.004772099666297436, "rewards//mean": 0.73089599609375, "rewards//std": 0.03094717301428318, "step": 1173 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2348, "grad_norm": 2.799098491668701, "kl": 0.847711768001318, "learning_rate": 8.782900999569645e-07, "loss": 0.0339, "num_tokens": 10148320.0, "reward": 0.75628662109375, "reward_std": 0.0023677253630012274, "rewards//mean": 0.75628662109375, "rewards//std": 0.022365286946296692, "step": 1174 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.235, "grad_norm": 3.236978769302368, "kl": 4.5400149412453175, "learning_rate": 8.780825197128437e-07, "loss": 0.1816, "num_tokens": 10156992.0, "reward": 0.787353515625, "reward_std": 0.013031869195401669, "rewards//mean": 0.787353515625, "rewards//std": 0.0330202579498291, "step": 1175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2352, "grad_norm": 6.243363380432129, "kl": 4.9968556091189384, "learning_rate": 8.778747871771291e-07, "loss": 0.1999, "num_tokens": 10165576.0, "reward": 0.7786865234375, "reward_std": 0.014137041755020618, "rewards//mean": 0.7786865234375, "rewards//std": 0.021135948598384857, "step": 1176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2354, "grad_norm": 5.1518988609313965, "kl": 6.310009470209479, "learning_rate": 8.776669024334955e-07, "loss": 0.2524, "num_tokens": 10174256.0, "reward": 0.76739501953125, "reward_std": 0.01620473712682724, "rewards//mean": 0.76739501953125, "rewards//std": 0.026647798717021942, "step": 1177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2356, "grad_norm": 7.432278633117676, "kl": 1.9024493284523487, "learning_rate": 8.774588655656787e-07, "loss": 0.0761, "num_tokens": 10182920.0, "reward": 0.72821044921875, "reward_std": 0.005631529726088047, "rewards//mean": 0.72821044921875, "rewards//std": 0.0313568115234375, "step": 1178 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2358, "grad_norm": 3.60760498046875, "kl": 2.6997303441166878, "learning_rate": 8.772506766574761e-07, "loss": 0.108, "num_tokens": 10191600.0, "reward": 0.77288818359375, "reward_std": 0.010570807382464409, "rewards//mean": 0.77288818359375, "rewards//std": 0.025025440379977226, "step": 1179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.236, "grad_norm": 2.924609422683716, "kl": 2.924817770719528, "learning_rate": 8.770423357927462e-07, "loss": 0.117, "num_tokens": 10200344.0, "reward": 0.73712158203125, "reward_std": 0.00714841578155756, "rewards//mean": 0.73712158203125, "rewards//std": 0.024416156113147736, "step": 1180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2362, "grad_norm": 4.570566654205322, "kl": 3.667961521074176, "learning_rate": 8.768338430554082e-07, "loss": 0.1467, "num_tokens": 10208960.0, "reward": 0.7537841796875, "reward_std": 0.00938231311738491, "rewards//mean": 0.7537841796875, "rewards//std": 0.039622530341148376, "step": 1181 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2364, "grad_norm": 13.069175720214844, "kl": 5.0878641698509455, "learning_rate": 8.766251985294434e-07, "loss": 0.2035, "num_tokens": 10217648.0, "reward": 0.74945068359375, "reward_std": 0.005997729487717152, "rewards//mean": 0.74945068359375, "rewards//std": 0.025102747604250908, "step": 1182 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2366, "grad_norm": 7.391311168670654, "kl": 2.7894755378365517, "learning_rate": 8.764164022988937e-07, "loss": 0.1116, "num_tokens": 10226272.0, "reward": 0.7518310546875, "reward_std": 0.0049908217042684555, "rewards//mean": 0.7518310546875, "rewards//std": 0.024929456412792206, "step": 1183 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2368, "grad_norm": 12.16387939453125, "kl": 6.9465586617589, "learning_rate": 8.762074544478621e-07, "loss": 0.2779, "num_tokens": 10234856.0, "reward": 0.750244140625, "reward_std": 0.013612458482384682, "rewards//mean": 0.750244140625, "rewards//std": 0.031062457710504532, "step": 1184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.237, "grad_norm": 5.594395160675049, "kl": 5.4297779724001884, "learning_rate": 8.75998355060513e-07, "loss": 0.2172, "num_tokens": 10243440.0, "reward": 0.73248291015625, "reward_std": 0.010458076372742653, "rewards//mean": 0.73248291015625, "rewards//std": 0.030765656381845474, "step": 1185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2372, "grad_norm": 3.4841830730438232, "kl": 0.9060949124395847, "learning_rate": 8.757891042210712e-07, "loss": 0.0362, "num_tokens": 10252096.0, "reward": 0.78179931640625, "reward_std": 0.0035348234232515097, "rewards//mean": 0.78179931640625, "rewards//std": 0.016005825251340866, "step": 1186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2374, "grad_norm": 6.8736252784729, "kl": 4.493726696819067, "learning_rate": 8.755797020138234e-07, "loss": 0.1797, "num_tokens": 10260736.0, "reward": 0.74163818359375, "reward_std": 0.014008762314915657, "rewards//mean": 0.74163818359375, "rewards//std": 0.0290810689330101, "step": 1187 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2376, "grad_norm": 2.9372668266296387, "kl": 0.9298593588173389, "learning_rate": 8.753701485231164e-07, "loss": 0.0372, "num_tokens": 10269320.0, "reward": 0.76458740234375, "reward_std": 0.003342084353789687, "rewards//mean": 0.76458740234375, "rewards//std": 0.03158337250351906, "step": 1188 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2378, "grad_norm": 4.1192498207092285, "kl": 4.80846232175827, "learning_rate": 8.751604438333586e-07, "loss": 0.1923, "num_tokens": 10278008.0, "reward": 0.769775390625, "reward_std": 0.011125459335744381, "rewards//mean": 0.769775390625, "rewards//std": 0.026936586946249008, "step": 1189 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.238, "grad_norm": 5.081353187561035, "kl": 0.7407982870936394, "learning_rate": 8.749505880290188e-07, "loss": 0.0296, "num_tokens": 10286632.0, "reward": 0.7650146484375, "reward_std": 0.002178550697863102, "rewards//mean": 0.7650146484375, "rewards//std": 0.026145800948143005, "step": 1190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2382, "grad_norm": 5.152669906616211, "kl": 1.9680843688547611, "learning_rate": 8.74740581194627e-07, "loss": 0.0787, "num_tokens": 10295192.0, "reward": 0.75360107421875, "reward_std": 0.008662248030304909, "rewards//mean": 0.75360107421875, "rewards//std": 0.021556738764047623, "step": 1191 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2384, "grad_norm": 4.382171630859375, "kl": 2.6179375406354666, "learning_rate": 8.745304234147739e-07, "loss": 0.1047, "num_tokens": 10303872.0, "reward": 0.7445068359375, "reward_std": 0.01046273298561573, "rewards//mean": 0.7445068359375, "rewards//std": 0.03477679565548897, "step": 1192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2386, "grad_norm": 6.248373508453369, "kl": 2.0443174801766872, "learning_rate": 8.743201147741111e-07, "loss": 0.0818, "num_tokens": 10312528.0, "reward": 0.75799560546875, "reward_std": 0.009483539499342442, "rewards//mean": 0.75799560546875, "rewards//std": 0.02377026528120041, "step": 1193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2388, "grad_norm": 3.7448031902313232, "kl": 2.289671115577221, "learning_rate": 8.741096553573506e-07, "loss": 0.0916, "num_tokens": 10321160.0, "reward": 0.75982666015625, "reward_std": 0.005153092555701733, "rewards//mean": 0.75982666015625, "rewards//std": 0.03834277763962746, "step": 1194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.239, "grad_norm": 6.080252170562744, "kl": 3.648873306810856, "learning_rate": 8.73899045249266e-07, "loss": 0.146, "num_tokens": 10329808.0, "reward": 0.7757568359375, "reward_std": 0.015079613775014877, "rewards//mean": 0.7757568359375, "rewards//std": 0.019409576430916786, "step": 1195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2392, "grad_norm": 8.778388023376465, "kl": 5.553971689194441, "learning_rate": 8.736882845346905e-07, "loss": 0.2222, "num_tokens": 10338520.0, "reward": 0.72332763671875, "reward_std": 0.010533872991800308, "rewards//mean": 0.72332763671875, "rewards//std": 0.025454264134168625, "step": 1196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2394, "grad_norm": 4.520699977874756, "kl": 4.532348919659853, "learning_rate": 8.734773732985185e-07, "loss": 0.1813, "num_tokens": 10347160.0, "reward": 0.736328125, "reward_std": 0.010841740295290947, "rewards//mean": 0.736328125, "rewards//std": 0.03413914144039154, "step": 1197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2396, "grad_norm": 6.137914180755615, "kl": 2.723930537700653, "learning_rate": 8.732663116257055e-07, "loss": 0.109, "num_tokens": 10355760.0, "reward": 0.788818359375, "reward_std": 0.009902745485305786, "rewards//mean": 0.788818359375, "rewards//std": 0.030606873333454132, "step": 1198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2398, "grad_norm": 5.281278610229492, "kl": 3.864069325849414, "learning_rate": 8.730550996012667e-07, "loss": 0.1546, "num_tokens": 10364384.0, "reward": 0.759521484375, "reward_std": 0.014747949317097664, "rewards//mean": 0.759521484375, "rewards//std": 0.024772604927420616, "step": 1199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.24, "grad_norm": 3.3508472442626953, "kl": 1.315210809931159, "learning_rate": 8.728437373102784e-07, "loss": 0.0526, "num_tokens": 10372912.0, "reward": 0.7799072265625, "reward_std": 0.005359763279557228, "rewards//mean": 0.7799072265625, "rewards//std": 0.015472638420760632, "step": 1200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2402, "grad_norm": 19.24559211730957, "kl": 2.4809737019240856, "learning_rate": 8.726322248378774e-07, "loss": 0.0992, "num_tokens": 10381504.0, "reward": 0.7843017578125, "reward_std": 0.010098189115524292, "rewards//mean": 0.7843017578125, "rewards//std": 0.023760473355650902, "step": 1201 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2404, "grad_norm": 7.235874652862549, "kl": 3.858848439529538, "learning_rate": 8.724205622692606e-07, "loss": 0.1544, "num_tokens": 10390168.0, "reward": 0.7529296875, "reward_std": 0.009310659021139145, "rewards//mean": 0.7529296875, "rewards//std": 0.02783101238310337, "step": 1202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2406, "grad_norm": 10.270509719848633, "kl": 5.628273021429777, "learning_rate": 8.72208749689686e-07, "loss": 0.2251, "num_tokens": 10398736.0, "reward": 0.7620849609375, "reward_std": 0.011618580669164658, "rewards//mean": 0.7620849609375, "rewards//std": 0.021773774176836014, "step": 1203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2408, "grad_norm": 5.030480861663818, "kl": 3.306698674336076, "learning_rate": 8.719967871844715e-07, "loss": 0.1323, "num_tokens": 10407320.0, "reward": 0.7060546875, "reward_std": 0.010522833094000816, "rewards//mean": 0.7060546875, "rewards//std": 0.034555185586214066, "step": 1204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.241, "grad_norm": 10.812642097473145, "kl": 3.850353153422475, "learning_rate": 8.717846748389955e-07, "loss": 0.154, "num_tokens": 10415912.0, "reward": 0.7244873046875, "reward_std": 0.008431274443864822, "rewards//mean": 0.7244873046875, "rewards//std": 0.030906852334737778, "step": 1205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2412, "grad_norm": 27.891611099243164, "kl": 3.9897553957998753, "learning_rate": 8.71572412738697e-07, "loss": 0.1596, "num_tokens": 10424592.0, "reward": 0.7591552734375, "reward_std": 0.011899648234248161, "rewards//mean": 0.7591552734375, "rewards//std": 0.02948923222720623, "step": 1206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2414, "grad_norm": 10.925292015075684, "kl": 4.316429942846298, "learning_rate": 8.713600009690751e-07, "loss": 0.1727, "num_tokens": 10433208.0, "reward": 0.751953125, "reward_std": 0.005849565379321575, "rewards//mean": 0.751953125, "rewards//std": 0.03435836732387543, "step": 1207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2416, "grad_norm": 4.999733924865723, "kl": 5.057210177183151, "learning_rate": 8.711474396156892e-07, "loss": 0.2023, "num_tokens": 10441728.0, "reward": 0.74945068359375, "reward_std": 0.010444171726703644, "rewards//mean": 0.74945068359375, "rewards//std": 0.029230598360300064, "step": 1208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2418, "grad_norm": 9.316264152526855, "kl": 2.889298852533102, "learning_rate": 8.709347287641592e-07, "loss": 0.1156, "num_tokens": 10450336.0, "reward": 0.778076171875, "reward_std": 0.012760389596223831, "rewards//mean": 0.778076171875, "rewards//std": 0.025769364088773727, "step": 1209 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.242, "grad_norm": 12.889250755310059, "kl": 2.478827353566885, "learning_rate": 8.707218685001646e-07, "loss": 0.0992, "num_tokens": 10458856.0, "reward": 0.74981689453125, "reward_std": 0.007895766757428646, "rewards//mean": 0.74981689453125, "rewards//std": 0.025338629260659218, "step": 1210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2422, "grad_norm": 8.895816802978516, "kl": 4.8706330712884665, "learning_rate": 8.705088589094458e-07, "loss": 0.1948, "num_tokens": 10467424.0, "reward": 0.72357177734375, "reward_std": 0.011275717057287693, "rewards//mean": 0.72357177734375, "rewards//std": 0.037478696554899216, "step": 1211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2424, "grad_norm": 13.54712200164795, "kl": 8.199546912685037, "learning_rate": 8.702957000778029e-07, "loss": 0.328, "num_tokens": 10476160.0, "reward": 0.7725830078125, "reward_std": 0.019813334569334984, "rewards//mean": 0.7725830078125, "rewards//std": 0.04365084320306778, "step": 1212 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2426, "grad_norm": 4.642307758331299, "kl": 4.662842875346541, "learning_rate": 8.700823920910963e-07, "loss": 0.1865, "num_tokens": 10484800.0, "reward": 0.744384765625, "reward_std": 0.01050968375056982, "rewards//mean": 0.744384765625, "rewards//std": 0.026207387447357178, "step": 1213 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2428, "grad_norm": 7.426133155822754, "kl": 5.004080157727003, "learning_rate": 8.698689350352464e-07, "loss": 0.2002, "num_tokens": 10493448.0, "reward": 0.79730224609375, "reward_std": 0.01092614233493805, "rewards//mean": 0.79730224609375, "rewards//std": 0.021878069266676903, "step": 1214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.243, "grad_norm": 8.481030464172363, "kl": 3.897951491177082, "learning_rate": 8.696553289962337e-07, "loss": 0.1559, "num_tokens": 10502088.0, "reward": 0.78448486328125, "reward_std": 0.008908973075449467, "rewards//mean": 0.78448486328125, "rewards//std": 0.016856174916028976, "step": 1215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2432, "grad_norm": 14.525918960571289, "kl": 4.0093574449419975, "learning_rate": 8.694415740600988e-07, "loss": 0.1604, "num_tokens": 10510608.0, "reward": 0.76397705078125, "reward_std": 0.011123032309114933, "rewards//mean": 0.76397705078125, "rewards//std": 0.028158091008663177, "step": 1216 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2434, "grad_norm": 10.636209487915039, "kl": 4.186413774266839, "learning_rate": 8.69227670312942e-07, "loss": 0.1675, "num_tokens": 10519232.0, "reward": 0.767333984375, "reward_std": 0.012185238301753998, "rewards//mean": 0.767333984375, "rewards//std": 0.021051626652479172, "step": 1217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2436, "grad_norm": 40.315582275390625, "kl": 6.443615116178989, "learning_rate": 8.690136178409235e-07, "loss": 0.2577, "num_tokens": 10527872.0, "reward": 0.74310302734375, "reward_std": 0.015162860043346882, "rewards//mean": 0.74310302734375, "rewards//std": 0.029179802164435387, "step": 1218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2438, "grad_norm": 10.690977096557617, "kl": 4.274382846429944, "learning_rate": 8.687994167302641e-07, "loss": 0.171, "num_tokens": 10536560.0, "reward": 0.71697998046875, "reward_std": 0.010151637718081474, "rewards//mean": 0.71697998046875, "rewards//std": 0.024635881185531616, "step": 1219 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.244, "grad_norm": 34.73554992675781, "kl": 7.326903743669391, "learning_rate": 8.685850670672438e-07, "loss": 0.2931, "num_tokens": 10545168.0, "reward": 0.7501220703125, "reward_std": 0.010396946221590042, "rewards//mean": 0.7501220703125, "rewards//std": 0.028805769979953766, "step": 1220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2442, "grad_norm": 34.42318344116211, "kl": 6.078345267102122, "learning_rate": 8.683705689382024e-07, "loss": 0.2431, "num_tokens": 10553912.0, "reward": 0.72015380859375, "reward_std": 0.013924827799201012, "rewards//mean": 0.72015380859375, "rewards//std": 0.02029588632285595, "step": 1221 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2444, "grad_norm": 8.692242622375488, "kl": 3.378894504159689, "learning_rate": 8.6815592242954e-07, "loss": 0.1352, "num_tokens": 10562520.0, "reward": 0.72772216796875, "reward_std": 0.011134006083011627, "rewards//mean": 0.72772216796875, "rewards//std": 0.037645939737558365, "step": 1222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2446, "grad_norm": 13.238306999206543, "kl": 4.720803817734122, "learning_rate": 8.67941127627716e-07, "loss": 0.1888, "num_tokens": 10571288.0, "reward": 0.77630615234375, "reward_std": 0.01717049442231655, "rewards//mean": 0.77630615234375, "rewards//std": 0.03105752356350422, "step": 1223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2448, "grad_norm": 25.972370147705078, "kl": 3.125759171321988, "learning_rate": 8.677261846192499e-07, "loss": 0.125, "num_tokens": 10579888.0, "reward": 0.75506591796875, "reward_std": 0.011132306419312954, "rewards//mean": 0.75506591796875, "rewards//std": 0.03039887361228466, "step": 1224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.245, "grad_norm": 9.226985931396484, "kl": 6.003644030541182, "learning_rate": 8.675110934907204e-07, "loss": 0.2401, "num_tokens": 10588520.0, "reward": 0.791748046875, "reward_std": 0.01744174212217331, "rewards//mean": 0.791748046875, "rewards//std": 0.02488965168595314, "step": 1225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2452, "grad_norm": 8.872241973876953, "kl": 2.977636018767953, "learning_rate": 8.672958543287666e-07, "loss": 0.1191, "num_tokens": 10597080.0, "reward": 0.77490234375, "reward_std": 0.009975229389965534, "rewards//mean": 0.77490234375, "rewards//std": 0.021793579682707787, "step": 1226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2454, "grad_norm": 11.018606185913086, "kl": 1.4360513500869274, "learning_rate": 8.670804672200865e-07, "loss": 0.0574, "num_tokens": 10605704.0, "reward": 0.74029541015625, "reward_std": 0.009111039340496063, "rewards//mean": 0.74029541015625, "rewards//std": 0.024190053343772888, "step": 1227 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2456, "grad_norm": 7.508191108703613, "kl": 2.0552911534905434, "learning_rate": 8.668649322514381e-07, "loss": 0.0822, "num_tokens": 10614352.0, "reward": 0.74603271484375, "reward_std": 0.011121081188321114, "rewards//mean": 0.74603271484375, "rewards//std": 0.03067498840391636, "step": 1228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2458, "grad_norm": 5.576145648956299, "kl": 2.9354402031749487, "learning_rate": 8.666492495096389e-07, "loss": 0.1174, "num_tokens": 10622968.0, "reward": 0.7451171875, "reward_std": 0.013886923901736736, "rewards//mean": 0.7451171875, "rewards//std": 0.04022333025932312, "step": 1229 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.246, "grad_norm": 8.805665969848633, "kl": 2.536726539954543, "learning_rate": 8.664334190815659e-07, "loss": 0.1015, "num_tokens": 10631624.0, "reward": 0.74835205078125, "reward_std": 0.0117610152810812, "rewards//mean": 0.74835205078125, "rewards//std": 0.028054682537913322, "step": 1230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2462, "grad_norm": 7.829744815826416, "kl": 3.846081856638193, "learning_rate": 8.662174410541554e-07, "loss": 0.1538, "num_tokens": 10640224.0, "reward": 0.737548828125, "reward_std": 0.011144118383526802, "rewards//mean": 0.737548828125, "rewards//std": 0.027942834421992302, "step": 1231 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2464, "grad_norm": 6.520112991333008, "kl": 2.071901887655258, "learning_rate": 8.660013155144035e-07, "loss": 0.0829, "num_tokens": 10648752.0, "reward": 0.73602294921875, "reward_std": 0.008494130335748196, "rewards//mean": 0.73602294921875, "rewards//std": 0.023976976051926613, "step": 1232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2466, "grad_norm": 7.9967875480651855, "kl": 6.293617935851216, "learning_rate": 8.657850425493654e-07, "loss": 0.2517, "num_tokens": 10657368.0, "reward": 0.73828125, "reward_std": 0.02175438404083252, "rewards//mean": 0.73828125, "rewards//std": 0.04595661163330078, "step": 1233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2468, "grad_norm": 3.7229974269866943, "kl": 3.049281246960163, "learning_rate": 8.65568622246156e-07, "loss": 0.122, "num_tokens": 10666032.0, "reward": 0.77191162109375, "reward_std": 0.008106382563710213, "rewards//mean": 0.77191162109375, "rewards//std": 0.023944754153490067, "step": 1234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.247, "grad_norm": 14.904300689697266, "kl": 2.1938663106411695, "learning_rate": 8.653520546919493e-07, "loss": 0.0878, "num_tokens": 10674592.0, "reward": 0.79669189453125, "reward_std": 0.011995483189821243, "rewards//mean": 0.79669189453125, "rewards//std": 0.033562660217285156, "step": 1235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2472, "grad_norm": 9.752826690673828, "kl": 5.046719258651137, "learning_rate": 8.651353399739787e-07, "loss": 0.2019, "num_tokens": 10683232.0, "reward": 0.73748779296875, "reward_std": 0.014928562566637993, "rewards//mean": 0.73748779296875, "rewards//std": 0.0304267480969429, "step": 1236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2474, "grad_norm": 9.397666931152344, "kl": 5.7591456193476915, "learning_rate": 8.649184781795367e-07, "loss": 0.2304, "num_tokens": 10691880.0, "reward": 0.7884521484375, "reward_std": 0.021809879690408707, "rewards//mean": 0.7884521484375, "rewards//std": 0.03189605847001076, "step": 1237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2476, "grad_norm": 15.128095626831055, "kl": 6.171701226383448, "learning_rate": 8.647014693959753e-07, "loss": 0.2469, "num_tokens": 10700504.0, "reward": 0.721435546875, "reward_std": 0.010850808583199978, "rewards//mean": 0.721435546875, "rewards//std": 0.033810317516326904, "step": 1238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2478, "grad_norm": 6.98085880279541, "kl": 2.490553168579936, "learning_rate": 8.644843137107057e-07, "loss": 0.0996, "num_tokens": 10709176.0, "reward": 0.77630615234375, "reward_std": 0.013876741752028465, "rewards//mean": 0.77630615234375, "rewards//std": 0.02504841610789299, "step": 1239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.248, "grad_norm": 23.153583526611328, "kl": 1.9768635164946318, "learning_rate": 8.642670112111981e-07, "loss": 0.0791, "num_tokens": 10717912.0, "reward": 0.77349853515625, "reward_std": 0.007938895374536514, "rewards//mean": 0.77349853515625, "rewards//std": 0.030109168961644173, "step": 1240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2482, "grad_norm": 9.379088401794434, "kl": 6.8707516342401505, "learning_rate": 8.64049561984982e-07, "loss": 0.2748, "num_tokens": 10726616.0, "reward": 0.75604248046875, "reward_std": 0.015093580819666386, "rewards//mean": 0.75604248046875, "rewards//std": 0.02899818681180477, "step": 1241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2484, "grad_norm": 8.29955005645752, "kl": 3.944977167993784, "learning_rate": 8.638319661196459e-07, "loss": 0.1578, "num_tokens": 10735200.0, "reward": 0.76593017578125, "reward_std": 0.007996254600584507, "rewards//mean": 0.76593017578125, "rewards//std": 0.023951709270477295, "step": 1242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2486, "grad_norm": 31.44389533996582, "kl": 7.717126229777932, "learning_rate": 8.636142237028372e-07, "loss": 0.3087, "num_tokens": 10743880.0, "reward": 0.7412109375, "reward_std": 0.014628959819674492, "rewards//mean": 0.7412109375, "rewards//std": 0.03663065284490585, "step": 1243 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2488, "grad_norm": 31.918787002563477, "kl": 9.728628190234303, "learning_rate": 8.633963348222628e-07, "loss": 0.3891, "num_tokens": 10752560.0, "reward": 0.73858642578125, "reward_std": 0.008714969269931316, "rewards//mean": 0.73858642578125, "rewards//std": 0.03782325237989426, "step": 1244 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.249, "grad_norm": 9.533246040344238, "kl": 2.8249090276658535, "learning_rate": 8.631782995656882e-07, "loss": 0.113, "num_tokens": 10761192.0, "reward": 0.7611083984375, "reward_std": 0.006418757606297731, "rewards//mean": 0.7611083984375, "rewards//std": 0.022522857412695885, "step": 1245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2492, "grad_norm": 34.13203811645508, "kl": 11.04344275034964, "learning_rate": 8.62960118020938e-07, "loss": 0.4417, "num_tokens": 10769904.0, "reward": 0.7117919921875, "reward_std": 0.010154195129871368, "rewards//mean": 0.7117919921875, "rewards//std": 0.04697548598051071, "step": 1246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2494, "grad_norm": 16.348169326782227, "kl": 4.125708527863026, "learning_rate": 8.627417902758956e-07, "loss": 0.165, "num_tokens": 10778632.0, "reward": 0.79022216796875, "reward_std": 0.013444384559988976, "rewards//mean": 0.79022216796875, "rewards//std": 0.024926647543907166, "step": 1247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2496, "grad_norm": 4.244007110595703, "kl": 2.4565203841775656, "learning_rate": 8.625233164185034e-07, "loss": 0.0983, "num_tokens": 10787216.0, "reward": 0.763916015625, "reward_std": 0.009631536900997162, "rewards//mean": 0.763916015625, "rewards//std": 0.0243386123329401, "step": 1248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2498, "grad_norm": 3.8231735229492188, "kl": 2.839028410613537, "learning_rate": 8.623046965367628e-07, "loss": 0.1136, "num_tokens": 10795792.0, "reward": 0.74566650390625, "reward_std": 0.0094450693577528, "rewards//mean": 0.74566650390625, "rewards//std": 0.02701956033706665, "step": 1249 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.25, "grad_norm": 9.10566234588623, "kl": 6.971977606415749, "learning_rate": 8.620859307187338e-07, "loss": 0.2789, "num_tokens": 10804400.0, "reward": 0.7451171875, "reward_std": 0.014911534264683723, "rewards//mean": 0.7451171875, "rewards//std": 0.030544491484761238, "step": 1250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2502, "grad_norm": 20.85613250732422, "kl": 8.940727328881621, "learning_rate": 8.61867019052535e-07, "loss": 0.3576, "num_tokens": 10813208.0, "reward": 0.7559814453125, "reward_std": 0.011848023161292076, "rewards//mean": 0.7559814453125, "rewards//std": 0.033167317509651184, "step": 1251 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2504, "grad_norm": 13.552748680114746, "kl": 4.335039634257555, "learning_rate": 8.616479616263444e-07, "loss": 0.1734, "num_tokens": 10821848.0, "reward": 0.7596435546875, "reward_std": 0.01608794741332531, "rewards//mean": 0.7596435546875, "rewards//std": 0.02570437081158161, "step": 1252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2506, "grad_norm": 2.9355878829956055, "kl": 3.873683985322714, "learning_rate": 8.61428758528398e-07, "loss": 0.1549, "num_tokens": 10830376.0, "reward": 0.7449951171875, "reward_std": 0.007550119888037443, "rewards//mean": 0.7449951171875, "rewards//std": 0.017791688442230225, "step": 1253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2508, "grad_norm": 4.576368808746338, "kl": 5.775039687752724, "learning_rate": 8.612094098469909e-07, "loss": 0.231, "num_tokens": 10838992.0, "reward": 0.76055908203125, "reward_std": 0.017393331974744797, "rewards//mean": 0.76055908203125, "rewards//std": 0.026177842170000076, "step": 1254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.251, "grad_norm": 11.842081069946289, "kl": 7.278344709426165, "learning_rate": 8.609899156704767e-07, "loss": 0.2911, "num_tokens": 10847728.0, "reward": 0.7464599609375, "reward_std": 0.010564940050244331, "rewards//mean": 0.7464599609375, "rewards//std": 0.03639662638306618, "step": 1255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2512, "grad_norm": 6.426313877105713, "kl": 5.643575828522444, "learning_rate": 8.607702760872677e-07, "loss": 0.2257, "num_tokens": 10856544.0, "reward": 0.781005859375, "reward_std": 0.012030715122818947, "rewards//mean": 0.781005859375, "rewards//std": 0.04051655903458595, "step": 1256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2514, "grad_norm": 3.890913724899292, "kl": 3.2777351792901754, "learning_rate": 8.605504911858346e-07, "loss": 0.1311, "num_tokens": 10865104.0, "reward": 0.750244140625, "reward_std": 0.0073437620885670185, "rewards//mean": 0.750244140625, "rewards//std": 0.022807708010077477, "step": 1257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2516, "grad_norm": 8.680928230285645, "kl": 1.990102307870984, "learning_rate": 8.603305610547069e-07, "loss": 0.0796, "num_tokens": 10873816.0, "reward": 0.75885009765625, "reward_std": 0.004397744778543711, "rewards//mean": 0.75885009765625, "rewards//std": 0.029266826808452606, "step": 1258 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2518, "grad_norm": 4.191249370574951, "kl": 5.214279746636748, "learning_rate": 8.601104857824722e-07, "loss": 0.2086, "num_tokens": 10882440.0, "reward": 0.78961181640625, "reward_std": 0.014969921670854092, "rewards//mean": 0.78961181640625, "rewards//std": 0.03335590288043022, "step": 1259 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.252, "grad_norm": 8.555057525634766, "kl": 3.9011244364082813, "learning_rate": 8.598902654577768e-07, "loss": 0.156, "num_tokens": 10891048.0, "reward": 0.7681884765625, "reward_std": 0.019131936132907867, "rewards//mean": 0.7681884765625, "rewards//std": 0.030466832220554352, "step": 1260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2522, "grad_norm": 5.7329816818237305, "kl": 4.792279714718461, "learning_rate": 8.596699001693255e-07, "loss": 0.1917, "num_tokens": 10899720.0, "reward": 0.7528076171875, "reward_std": 0.012098362669348717, "rewards//mean": 0.7528076171875, "rewards//std": 0.032245345413684845, "step": 1261 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2524, "grad_norm": 7.551224708557129, "kl": 3.247972499579191, "learning_rate": 8.594493900058816e-07, "loss": 0.1299, "num_tokens": 10908304.0, "reward": 0.77276611328125, "reward_std": 0.015282051637768745, "rewards//mean": 0.77276611328125, "rewards//std": 0.04180915653705597, "step": 1262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2526, "grad_norm": 5.042646884918213, "kl": 5.161837503314018, "learning_rate": 8.592287350562663e-07, "loss": 0.2065, "num_tokens": 10916880.0, "reward": 0.79107666015625, "reward_std": 0.015731535851955414, "rewards//mean": 0.79107666015625, "rewards//std": 0.034597158432006836, "step": 1263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2528, "grad_norm": 14.095562934875488, "kl": 1.6300823576748371, "learning_rate": 8.590079354093593e-07, "loss": 0.0652, "num_tokens": 10925520.0, "reward": 0.77783203125, "reward_std": 0.0067242844961583614, "rewards//mean": 0.77783203125, "rewards//std": 0.022579578682780266, "step": 1264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.253, "grad_norm": 3.9633383750915527, "kl": 1.2054816894233227, "learning_rate": 8.587869911540992e-07, "loss": 0.0482, "num_tokens": 10934168.0, "reward": 0.7489013671875, "reward_std": 0.004060286097228527, "rewards//mean": 0.7489013671875, "rewards//std": 0.025358112528920174, "step": 1265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2532, "grad_norm": 4.64142370223999, "kl": 3.954777590930462, "learning_rate": 8.585659023794818e-07, "loss": 0.1582, "num_tokens": 10942832.0, "reward": 0.7490234375, "reward_std": 0.013307355344295502, "rewards//mean": 0.7490234375, "rewards//std": 0.034195851534605026, "step": 1266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2534, "grad_norm": 7.686950206756592, "kl": 3.5369584299623966, "learning_rate": 8.583446691745617e-07, "loss": 0.1415, "num_tokens": 10951568.0, "reward": 0.784912109375, "reward_std": 0.01466282270848751, "rewards//mean": 0.784912109375, "rewards//std": 0.026198145002126694, "step": 1267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2536, "grad_norm": 4.102766990661621, "kl": 4.478368993848562, "learning_rate": 8.581232916284517e-07, "loss": 0.1791, "num_tokens": 10960248.0, "reward": 0.75958251953125, "reward_std": 0.00903963390737772, "rewards//mean": 0.75958251953125, "rewards//std": 0.02239031344652176, "step": 1268 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2538, "grad_norm": 5.248315811157227, "kl": 2.6819275487214327, "learning_rate": 8.579017698303228e-07, "loss": 0.1073, "num_tokens": 10968904.0, "reward": 0.75897216796875, "reward_std": 0.009780505672097206, "rewards//mean": 0.75897216796875, "rewards//std": 0.032768476754426956, "step": 1269 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.254, "grad_norm": 4.820333003997803, "kl": 2.1646746415644884, "learning_rate": 8.576801038694039e-07, "loss": 0.0866, "num_tokens": 10977560.0, "reward": 0.76324462890625, "reward_std": 0.00664786109700799, "rewards//mean": 0.76324462890625, "rewards//std": 0.027353644371032715, "step": 1270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2542, "grad_norm": 7.026268482208252, "kl": 3.008241306990385, "learning_rate": 8.574582938349817e-07, "loss": 0.1203, "num_tokens": 10986264.0, "reward": 0.77130126953125, "reward_std": 0.01176033541560173, "rewards//mean": 0.77130126953125, "rewards//std": 0.026133855804800987, "step": 1271 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2544, "grad_norm": 13.753020286560059, "kl": 1.1150043960660696, "learning_rate": 8.572363398164016e-07, "loss": 0.0446, "num_tokens": 10994872.0, "reward": 0.75775146484375, "reward_std": 0.0015412584180012345, "rewards//mean": 0.75775146484375, "rewards//std": 0.019015923142433167, "step": 1272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2546, "grad_norm": 6.849703311920166, "kl": 1.1169192641973495, "learning_rate": 8.570142419030666e-07, "loss": 0.0447, "num_tokens": 11003560.0, "reward": 0.7681884765625, "reward_std": 0.0015413069631904364, "rewards//mean": 0.7681884765625, "rewards//std": 0.02633042074739933, "step": 1273 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2548, "grad_norm": 2.7238874435424805, "kl": 2.621772589161992, "learning_rate": 8.567920001844375e-07, "loss": 0.1049, "num_tokens": 11012160.0, "reward": 0.76214599609375, "reward_std": 0.007980536669492722, "rewards//mean": 0.76214599609375, "rewards//std": 0.02573402039706707, "step": 1274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.255, "grad_norm": 12.22774887084961, "kl": 4.507930127903819, "learning_rate": 8.565696147500337e-07, "loss": 0.1803, "num_tokens": 11020816.0, "reward": 0.75836181640625, "reward_std": 0.010539361275732517, "rewards//mean": 0.75836181640625, "rewards//std": 0.0307173989713192, "step": 1275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2552, "grad_norm": 7.131739139556885, "kl": 3.0699077248573303, "learning_rate": 8.563470856894314e-07, "loss": 0.1228, "num_tokens": 11029440.0, "reward": 0.76611328125, "reward_std": 0.008369958959519863, "rewards//mean": 0.76611328125, "rewards//std": 0.01891385391354561, "step": 1276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2554, "grad_norm": 4.130187511444092, "kl": 4.302004503086209, "learning_rate": 8.561244130922657e-07, "loss": 0.1721, "num_tokens": 11038104.0, "reward": 0.7452392578125, "reward_std": 0.013671613298356533, "rewards//mean": 0.7452392578125, "rewards//std": 0.034474264830350876, "step": 1277 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2556, "grad_norm": 6.271708965301514, "kl": 1.6784346252679825, "learning_rate": 8.559015970482291e-07, "loss": 0.0671, "num_tokens": 11046760.0, "reward": 0.7467041015625, "reward_std": 0.0062317755073308945, "rewards//mean": 0.7467041015625, "rewards//std": 0.03089313581585884, "step": 1278 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2558, "grad_norm": 27.68781089782715, "kl": 4.338345795869827, "learning_rate": 8.556786376470716e-07, "loss": 0.1735, "num_tokens": 11055496.0, "reward": 0.77911376953125, "reward_std": 0.01089534629136324, "rewards//mean": 0.77911376953125, "rewards//std": 0.03209824860095978, "step": 1279 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.256, "grad_norm": 8.381805419921875, "kl": 4.739690890535712, "learning_rate": 8.554555349786015e-07, "loss": 0.1896, "num_tokens": 11064112.0, "reward": 0.7952880859375, "reward_std": 0.013847950845956802, "rewards//mean": 0.7952880859375, "rewards//std": 0.031069036573171616, "step": 1280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2562, "grad_norm": 3.666189193725586, "kl": 1.6756311561912298, "learning_rate": 8.552322891326844e-07, "loss": 0.067, "num_tokens": 11072752.0, "reward": 0.77252197265625, "reward_std": 0.004807932768017054, "rewards//mean": 0.77252197265625, "rewards//std": 0.010433909483253956, "step": 1281 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2564, "grad_norm": 4.122167110443115, "kl": 2.7426191214472055, "learning_rate": 8.550089001992437e-07, "loss": 0.1097, "num_tokens": 11081424.0, "reward": 0.73468017578125, "reward_std": 0.008176295086741447, "rewards//mean": 0.73468017578125, "rewards//std": 0.036652494221925735, "step": 1282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2566, "grad_norm": 4.487239837646484, "kl": 3.345269527286291, "learning_rate": 8.547853682682604e-07, "loss": 0.1338, "num_tokens": 11089952.0, "reward": 0.77093505859375, "reward_std": 0.007087345235049725, "rewards//mean": 0.77093505859375, "rewards//std": 0.01962856762111187, "step": 1283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2568, "grad_norm": 7.786471366882324, "kl": 3.652106024324894, "learning_rate": 8.545616934297733e-07, "loss": 0.1461, "num_tokens": 11098544.0, "reward": 0.79718017578125, "reward_std": 0.016802387312054634, "rewards//mean": 0.79718017578125, "rewards//std": 0.026940450072288513, "step": 1284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.257, "grad_norm": 5.001603126525879, "kl": 1.5137909315526485, "learning_rate": 8.543378757738784e-07, "loss": 0.0606, "num_tokens": 11107168.0, "reward": 0.7647705078125, "reward_std": 0.0034436597488820553, "rewards//mean": 0.7647705078125, "rewards//std": 0.034319356083869934, "step": 1285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2572, "grad_norm": 5.244659900665283, "kl": 2.4232273045927286, "learning_rate": 8.541139153907295e-07, "loss": 0.0969, "num_tokens": 11115752.0, "reward": 0.75677490234375, "reward_std": 0.006669648922979832, "rewards//mean": 0.75677490234375, "rewards//std": 0.02266247384250164, "step": 1286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2574, "grad_norm": 8.395547866821289, "kl": 3.235109167173505, "learning_rate": 8.538898123705379e-07, "loss": 0.1294, "num_tokens": 11124360.0, "reward": 0.78277587890625, "reward_std": 0.008708701469004154, "rewards//mean": 0.78277587890625, "rewards//std": 0.02795793116092682, "step": 1287 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2576, "grad_norm": 1.7316111326217651, "kl": 1.5258556101471186, "learning_rate": 8.536655668035721e-07, "loss": 0.061, "num_tokens": 11133016.0, "reward": 0.767822265625, "reward_std": 0.004365556873381138, "rewards//mean": 0.767822265625, "rewards//std": 0.028398511931300163, "step": 1288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2578, "grad_norm": 6.039356708526611, "kl": 7.7991479728370905, "learning_rate": 8.534411787801586e-07, "loss": 0.312, "num_tokens": 11141656.0, "reward": 0.771240234375, "reward_std": 0.01720530167222023, "rewards//mean": 0.771240234375, "rewards//std": 0.03166480362415314, "step": 1289 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.258, "grad_norm": 5.303319931030273, "kl": 4.693038806319237, "learning_rate": 8.532166483906802e-07, "loss": 0.1877, "num_tokens": 11150280.0, "reward": 0.7457275390625, "reward_std": 0.008638778701424599, "rewards//mean": 0.7457275390625, "rewards//std": 0.024800391867756844, "step": 1290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2582, "grad_norm": 3.814932346343994, "kl": 5.850923229008913, "learning_rate": 8.529919757255781e-07, "loss": 0.234, "num_tokens": 11158864.0, "reward": 0.76611328125, "reward_std": 0.02036258764564991, "rewards//mean": 0.76611328125, "rewards//std": 0.03756749629974365, "step": 1291 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2584, "grad_norm": 4.260945796966553, "kl": 5.5106862504035234, "learning_rate": 8.527671608753506e-07, "loss": 0.2204, "num_tokens": 11167488.0, "reward": 0.76123046875, "reward_std": 0.013373324647545815, "rewards//mean": 0.76123046875, "rewards//std": 0.028969526290893555, "step": 1292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2586, "grad_norm": 5.248500347137451, "kl": 3.916138354688883, "learning_rate": 8.525422039305528e-07, "loss": 0.1566, "num_tokens": 11176208.0, "reward": 0.7728271484375, "reward_std": 0.01021310780197382, "rewards//mean": 0.7728271484375, "rewards//std": 0.027065252885222435, "step": 1293 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2588, "grad_norm": 7.21472692489624, "kl": 5.753823101520538, "learning_rate": 8.523171049817973e-07, "loss": 0.2302, "num_tokens": 11184800.0, "reward": 0.776123046875, "reward_std": 0.011476203799247742, "rewards//mean": 0.776123046875, "rewards//std": 0.026565372943878174, "step": 1294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.259, "grad_norm": 2.359610080718994, "kl": 3.251587826758623, "learning_rate": 8.520918641197541e-07, "loss": 0.1301, "num_tokens": 11193456.0, "reward": 0.76739501953125, "reward_std": 0.008114458061754704, "rewards//mean": 0.76739501953125, "rewards//std": 0.03128044307231903, "step": 1295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2592, "grad_norm": 2.9881999492645264, "kl": 2.696640431880951, "learning_rate": 8.518664814351502e-07, "loss": 0.1079, "num_tokens": 11202144.0, "reward": 0.739990234375, "reward_std": 0.005112895276397467, "rewards//mean": 0.739990234375, "rewards//std": 0.02935468964278698, "step": 1296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2594, "grad_norm": 4.5930891036987305, "kl": 6.463214572519064, "learning_rate": 8.516409570187696e-07, "loss": 0.2585, "num_tokens": 11210904.0, "reward": 0.75799560546875, "reward_std": 0.016281146556138992, "rewards//mean": 0.75799560546875, "rewards//std": 0.036241315305233, "step": 1297 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2596, "grad_norm": 8.910324096679688, "kl": 6.601942088454962, "learning_rate": 8.514152909614535e-07, "loss": 0.2641, "num_tokens": 11219616.0, "reward": 0.76300048828125, "reward_std": 0.015175841748714447, "rewards//mean": 0.76300048828125, "rewards//std": 0.03656483441591263, "step": 1298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2598, "grad_norm": 3.1343729496002197, "kl": 2.7468247059732676, "learning_rate": 8.511894833541005e-07, "loss": 0.1099, "num_tokens": 11228280.0, "reward": 0.750732421875, "reward_std": 0.004793817177414894, "rewards//mean": 0.750732421875, "rewards//std": 0.021842146292328835, "step": 1299 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.26, "grad_norm": 4.297388076782227, "kl": 2.919129230082035, "learning_rate": 8.509635342876654e-07, "loss": 0.1168, "num_tokens": 11236848.0, "reward": 0.7559814453125, "reward_std": 0.0069838822819292545, "rewards//mean": 0.7559814453125, "rewards//std": 0.028125077486038208, "step": 1300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2602, "grad_norm": 5.890142917633057, "kl": 4.626889752224088, "learning_rate": 8.507374438531606e-07, "loss": 0.1851, "num_tokens": 11245440.0, "reward": 0.77301025390625, "reward_std": 0.013959040865302086, "rewards//mean": 0.77301025390625, "rewards//std": 0.03568018972873688, "step": 1301 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2604, "grad_norm": 7.185187339782715, "kl": 3.8990981429815292, "learning_rate": 8.505112121416553e-07, "loss": 0.156, "num_tokens": 11254128.0, "reward": 0.733154296875, "reward_std": 0.004352999851107597, "rewards//mean": 0.733154296875, "rewards//std": 0.01750909723341465, "step": 1302 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2606, "grad_norm": 6.5421552658081055, "kl": 3.2562216632068157, "learning_rate": 8.502848392442758e-07, "loss": 0.1302, "num_tokens": 11262768.0, "reward": 0.78045654296875, "reward_std": 0.010951787233352661, "rewards//mean": 0.78045654296875, "rewards//std": 0.034168779850006104, "step": 1303 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2608, "grad_norm": 4.482645034790039, "kl": 4.552934428676963, "learning_rate": 8.500583252522052e-07, "loss": 0.1821, "num_tokens": 11271360.0, "reward": 0.76275634765625, "reward_std": 0.008597548119723797, "rewards//mean": 0.76275634765625, "rewards//std": 0.035531386733055115, "step": 1304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.261, "grad_norm": 6.0340962409973145, "kl": 3.4215122647583485, "learning_rate": 8.498316702566826e-07, "loss": 0.1369, "num_tokens": 11279992.0, "reward": 0.76104736328125, "reward_std": 0.006469708867371082, "rewards//mean": 0.76104736328125, "rewards//std": 0.015145717188715935, "step": 1305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2612, "grad_norm": 5.419297218322754, "kl": 2.951256576925516, "learning_rate": 8.496048743490053e-07, "loss": 0.1181, "num_tokens": 11288736.0, "reward": 0.76922607421875, "reward_std": 0.009774585254490376, "rewards//mean": 0.76922607421875, "rewards//std": 0.029250789433717728, "step": 1306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2614, "grad_norm": 9.354080200195312, "kl": 1.5646381042897701, "learning_rate": 8.493779376205264e-07, "loss": 0.0626, "num_tokens": 11297368.0, "reward": 0.79656982421875, "reward_std": 0.010128369554877281, "rewards//mean": 0.79656982421875, "rewards//std": 0.026063093915581703, "step": 1307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2616, "grad_norm": 2.8813416957855225, "kl": 1.8132019937038422, "learning_rate": 8.491508601626561e-07, "loss": 0.0725, "num_tokens": 11306032.0, "reward": 0.75103759765625, "reward_std": 0.005398713517934084, "rewards//mean": 0.75103759765625, "rewards//std": 0.03246075287461281, "step": 1308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2618, "grad_norm": 9.517343521118164, "kl": 3.6751442085951567, "learning_rate": 8.489236420668608e-07, "loss": 0.147, "num_tokens": 11314848.0, "reward": 0.7562255859375, "reward_std": 0.016644582152366638, "rewards//mean": 0.7562255859375, "rewards//std": 0.02966529130935669, "step": 1309 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.262, "grad_norm": 14.530468940734863, "kl": 3.6663004029542208, "learning_rate": 8.486962834246645e-07, "loss": 0.1467, "num_tokens": 11323456.0, "reward": 0.74462890625, "reward_std": 0.004980337806046009, "rewards//mean": 0.74462890625, "rewards//std": 0.028112424537539482, "step": 1310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2622, "grad_norm": 7.726293563842773, "kl": 2.5143831074237823, "learning_rate": 8.484687843276468e-07, "loss": 0.1006, "num_tokens": 11332048.0, "reward": 0.75604248046875, "reward_std": 0.009487086907029152, "rewards//mean": 0.75604248046875, "rewards//std": 0.03158097714185715, "step": 1311 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2624, "grad_norm": 7.72947359085083, "kl": 3.786453163251281, "learning_rate": 8.482411448674445e-07, "loss": 0.1515, "num_tokens": 11340712.0, "reward": 0.74591064453125, "reward_std": 0.006868995260447264, "rewards//mean": 0.74591064453125, "rewards//std": 0.032109566032886505, "step": 1312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2626, "grad_norm": 6.769286632537842, "kl": 5.255178485065699, "learning_rate": 8.480133651357505e-07, "loss": 0.2102, "num_tokens": 11349392.0, "reward": 0.74322509765625, "reward_std": 0.009714778512716293, "rewards//mean": 0.74322509765625, "rewards//std": 0.030680909752845764, "step": 1313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2628, "grad_norm": 6.920955657958984, "kl": 2.671338966116309, "learning_rate": 8.477854452243147e-07, "loss": 0.1069, "num_tokens": 11358080.0, "reward": 0.7919921875, "reward_std": 0.01285955123603344, "rewards//mean": 0.7919921875, "rewards//std": 0.02309396117925644, "step": 1314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.263, "grad_norm": 5.518157005310059, "kl": 3.1050837635993958, "learning_rate": 8.475573852249434e-07, "loss": 0.1242, "num_tokens": 11366696.0, "reward": 0.74749755859375, "reward_std": 0.008475309237837791, "rewards//mean": 0.74749755859375, "rewards//std": 0.02823164500296116, "step": 1315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2632, "grad_norm": 9.080916404724121, "kl": 2.1023597456514835, "learning_rate": 8.473291852294986e-07, "loss": 0.0841, "num_tokens": 11375288.0, "reward": 0.757080078125, "reward_std": 0.006348649971187115, "rewards//mean": 0.757080078125, "rewards//std": 0.019548293203115463, "step": 1316 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2634, "grad_norm": 5.56795072555542, "kl": 2.205072931945324, "learning_rate": 8.471008453298996e-07, "loss": 0.0882, "num_tokens": 11383904.0, "reward": 0.77349853515625, "reward_std": 0.007874330505728722, "rewards//mean": 0.77349853515625, "rewards//std": 0.02834402211010456, "step": 1317 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2636, "grad_norm": 6.041005611419678, "kl": 1.6031034365296364, "learning_rate": 8.468723656181218e-07, "loss": 0.0641, "num_tokens": 11392448.0, "reward": 0.7506103515625, "reward_std": 0.005055404268205166, "rewards//mean": 0.7506103515625, "rewards//std": 0.022991150617599487, "step": 1318 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2638, "grad_norm": 3.836426258087158, "kl": 1.4163221195340157, "learning_rate": 8.466437461861964e-07, "loss": 0.0567, "num_tokens": 11401088.0, "reward": 0.77691650390625, "reward_std": 0.0033918290864676237, "rewards//mean": 0.77691650390625, "rewards//std": 0.019289391115307808, "step": 1319 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.264, "grad_norm": 5.417503356933594, "kl": 4.588376244530082, "learning_rate": 8.464149871262116e-07, "loss": 0.1835, "num_tokens": 11409840.0, "reward": 0.75164794921875, "reward_std": 0.00984761118888855, "rewards//mean": 0.75164794921875, "rewards//std": 0.04103292152285576, "step": 1320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2642, "grad_norm": 2.41438889503479, "kl": 2.309561477974057, "learning_rate": 8.461860885303113e-07, "loss": 0.0924, "num_tokens": 11418512.0, "reward": 0.74456787109375, "reward_std": 0.009001797065138817, "rewards//mean": 0.74456787109375, "rewards//std": 0.029802938923239708, "step": 1321 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2644, "grad_norm": 3.3165946006774902, "kl": 4.257082613185048, "learning_rate": 8.459570504906961e-07, "loss": 0.1703, "num_tokens": 11427168.0, "reward": 0.7557373046875, "reward_std": 0.011855144053697586, "rewards//mean": 0.7557373046875, "rewards//std": 0.02796097658574581, "step": 1322 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2646, "grad_norm": 6.713140487670898, "kl": 1.3529539816081524, "learning_rate": 8.457278730996222e-07, "loss": 0.0541, "num_tokens": 11435776.0, "reward": 0.8125, "reward_std": 0.006250374019145966, "rewards//mean": 0.8125, "rewards//std": 0.01666027121245861, "step": 1323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2648, "grad_norm": 8.603462219238281, "kl": 4.811173427850008, "learning_rate": 8.454985564494024e-07, "loss": 0.1924, "num_tokens": 11444424.0, "reward": 0.7425537109375, "reward_std": 0.0073181819170713425, "rewards//mean": 0.7425537109375, "rewards//std": 0.034532178193330765, "step": 1324 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.265, "grad_norm": 8.883708000183105, "kl": 1.1453363094478846, "learning_rate": 8.452691006324054e-07, "loss": 0.0458, "num_tokens": 11453048.0, "reward": 0.74786376953125, "reward_std": 0.0022728105541318655, "rewards//mean": 0.74786376953125, "rewards//std": 0.028670594096183777, "step": 1325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2652, "grad_norm": 6.938794136047363, "kl": 4.304388772696257, "learning_rate": 8.45039505741056e-07, "loss": 0.1722, "num_tokens": 11461712.0, "reward": 0.74566650390625, "reward_std": 0.008371752686798573, "rewards//mean": 0.74566650390625, "rewards//std": 0.030931515619158745, "step": 1326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2654, "grad_norm": 5.500860691070557, "kl": 2.4787068367004395, "learning_rate": 8.448097718678348e-07, "loss": 0.0991, "num_tokens": 11470416.0, "reward": 0.7520751953125, "reward_std": 0.006374950520694256, "rewards//mean": 0.7520751953125, "rewards//std": 0.03016122244298458, "step": 1327 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2656, "grad_norm": 4.477947235107422, "kl": 2.00894608348608, "learning_rate": 8.44579899105279e-07, "loss": 0.0804, "num_tokens": 11479032.0, "reward": 0.75067138671875, "reward_std": 0.0037859920412302017, "rewards//mean": 0.75067138671875, "rewards//std": 0.021379046142101288, "step": 1328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2658, "grad_norm": 5.4936957359313965, "kl": 1.6012097373604774, "learning_rate": 8.443498875459808e-07, "loss": 0.064, "num_tokens": 11487568.0, "reward": 0.72161865234375, "reward_std": 0.0034891131799668074, "rewards//mean": 0.72161865234375, "rewards//std": 0.0310653205960989, "step": 1329 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.266, "grad_norm": 3.366276264190674, "kl": 1.7269084453582764, "learning_rate": 8.441197372825892e-07, "loss": 0.0691, "num_tokens": 11496160.0, "reward": 0.73370361328125, "reward_std": 0.004661104176193476, "rewards//mean": 0.73370361328125, "rewards//std": 0.032841384410858154, "step": 1330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2662, "grad_norm": 4.2201056480407715, "kl": 4.002227613702416, "learning_rate": 8.438894484078085e-07, "loss": 0.1601, "num_tokens": 11504792.0, "reward": 0.75140380859375, "reward_std": 0.015436064451932907, "rewards//mean": 0.75140380859375, "rewards//std": 0.034746479243040085, "step": 1331 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2664, "grad_norm": 8.48907470703125, "kl": 5.469716316089034, "learning_rate": 8.43659021014399e-07, "loss": 0.2188, "num_tokens": 11513432.0, "reward": 0.76434326171875, "reward_std": 0.011993806809186935, "rewards//mean": 0.76434326171875, "rewards//std": 0.0229155533015728, "step": 1332 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2666, "grad_norm": 6.576357364654541, "kl": 2.191881190985441, "learning_rate": 8.434284551951772e-07, "loss": 0.0877, "num_tokens": 11522112.0, "reward": 0.77593994140625, "reward_std": 0.009596865624189377, "rewards//mean": 0.77593994140625, "rewards//std": 0.02598922699689865, "step": 1333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2668, "grad_norm": 20.691680908203125, "kl": 2.7848943509161472, "learning_rate": 8.431977510430145e-07, "loss": 0.1114, "num_tokens": 11530648.0, "reward": 0.72088623046875, "reward_std": 0.00493771955370903, "rewards//mean": 0.72088623046875, "rewards//std": 0.03379638120532036, "step": 1334 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.267, "grad_norm": 8.741593360900879, "kl": 3.956906611099839, "learning_rate": 8.429669086508389e-07, "loss": 0.1583, "num_tokens": 11539376.0, "reward": 0.7659912109375, "reward_std": 0.009909494780004025, "rewards//mean": 0.7659912109375, "rewards//std": 0.040937863290309906, "step": 1335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2672, "grad_norm": 19.778757095336914, "kl": 4.849205795675516, "learning_rate": 8.427359281116333e-07, "loss": 0.194, "num_tokens": 11548024.0, "reward": 0.75140380859375, "reward_std": 0.01166295912116766, "rewards//mean": 0.75140380859375, "rewards//std": 0.025117216631770134, "step": 1336 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2674, "grad_norm": 7.019726753234863, "kl": 5.418259494006634, "learning_rate": 8.42504809518437e-07, "loss": 0.2167, "num_tokens": 11556656.0, "reward": 0.75677490234375, "reward_std": 0.011645495891571045, "rewards//mean": 0.75677490234375, "rewards//std": 0.03187344968318939, "step": 1337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2676, "grad_norm": 40.18573760986328, "kl": 5.802844006568193, "learning_rate": 8.422735529643443e-07, "loss": 0.2321, "num_tokens": 11565296.0, "reward": 0.76519775390625, "reward_std": 0.01373140700161457, "rewards//mean": 0.76519775390625, "rewards//std": 0.028306612744927406, "step": 1338 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2678, "grad_norm": 7.769367218017578, "kl": 3.249770663678646, "learning_rate": 8.420421585425055e-07, "loss": 0.13, "num_tokens": 11573864.0, "reward": 0.74432373046875, "reward_std": 0.010364457033574581, "rewards//mean": 0.74432373046875, "rewards//std": 0.033457860350608826, "step": 1339 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.268, "grad_norm": 20.78012466430664, "kl": 7.734453845769167, "learning_rate": 8.41810626346126e-07, "loss": 0.3094, "num_tokens": 11582424.0, "reward": 0.73883056640625, "reward_std": 0.014386819675564766, "rewards//mean": 0.73883056640625, "rewards//std": 0.03075679950416088, "step": 1340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2682, "grad_norm": 43.48139190673828, "kl": 6.283532673493028, "learning_rate": 8.415789564684673e-07, "loss": 0.2513, "num_tokens": 11590968.0, "reward": 0.7933349609375, "reward_std": 0.010344470851123333, "rewards//mean": 0.7933349609375, "rewards//std": 0.025178389623761177, "step": 1341 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2684, "grad_norm": 60.180294036865234, "kl": 10.318531692028046, "learning_rate": 8.413471490028455e-07, "loss": 0.4127, "num_tokens": 11599640.0, "reward": 0.76593017578125, "reward_std": 0.015049831941723824, "rewards//mean": 0.76593017578125, "rewards//std": 0.02918447181582451, "step": 1342 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2686, "grad_norm": 60.8212890625, "kl": 9.179529754444957, "learning_rate": 8.41115204042633e-07, "loss": 0.3672, "num_tokens": 11608232.0, "reward": 0.749267578125, "reward_std": 0.013325735926628113, "rewards//mean": 0.749267578125, "rewards//std": 0.03470119461417198, "step": 1343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2688, "grad_norm": 38.94044494628906, "kl": 5.907689101994038, "learning_rate": 8.408831216812573e-07, "loss": 0.2363, "num_tokens": 11616920.0, "reward": 0.758544921875, "reward_std": 0.015807736665010452, "rewards//mean": 0.758544921875, "rewards//std": 0.04591904953122139, "step": 1344 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.269, "grad_norm": 18.94415855407715, "kl": 3.504896441474557, "learning_rate": 8.406509020122008e-07, "loss": 0.1402, "num_tokens": 11625560.0, "reward": 0.77899169921875, "reward_std": 0.0050090402364730835, "rewards//mean": 0.77899169921875, "rewards//std": 0.026509970426559448, "step": 1345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2692, "grad_norm": 26.979238510131836, "kl": 6.458527412265539, "learning_rate": 8.404185451290017e-07, "loss": 0.2583, "num_tokens": 11634256.0, "reward": 0.7689208984375, "reward_std": 0.012575831264257431, "rewards//mean": 0.7689208984375, "rewards//std": 0.03487763553857803, "step": 1346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2694, "grad_norm": 5.83043909072876, "kl": 4.26373004168272, "learning_rate": 8.401860511252533e-07, "loss": 0.1705, "num_tokens": 11642840.0, "reward": 0.75396728515625, "reward_std": 0.00985700823366642, "rewards//mean": 0.75396728515625, "rewards//std": 0.03178403899073601, "step": 1347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2696, "grad_norm": 5.888754367828369, "kl": 2.50071469694376, "learning_rate": 8.399534200946043e-07, "loss": 0.1, "num_tokens": 11651408.0, "reward": 0.74603271484375, "reward_std": 0.006771939806640148, "rewards//mean": 0.74603271484375, "rewards//std": 0.02661653608083725, "step": 1348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2698, "grad_norm": 11.314641952514648, "kl": 2.699692741036415, "learning_rate": 8.397206521307583e-07, "loss": 0.108, "num_tokens": 11660048.0, "reward": 0.7738037109375, "reward_std": 0.010594477877020836, "rewards//mean": 0.7738037109375, "rewards//std": 0.025293558835983276, "step": 1349 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.27, "grad_norm": 8.233244895935059, "kl": 2.063253328204155, "learning_rate": 8.394877473274741e-07, "loss": 0.0825, "num_tokens": 11668728.0, "reward": 0.74749755859375, "reward_std": 0.0059649404138326645, "rewards//mean": 0.74749755859375, "rewards//std": 0.03423384577035904, "step": 1350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2702, "grad_norm": 4.355931758880615, "kl": 1.0865337681025267, "learning_rate": 8.392547057785661e-07, "loss": 0.0435, "num_tokens": 11677376.0, "reward": 0.77728271484375, "reward_std": 0.0041401139460504055, "rewards//mean": 0.77728271484375, "rewards//std": 0.018616894260048866, "step": 1351 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2704, "grad_norm": 5.585280895233154, "kl": 2.5505129732191563, "learning_rate": 8.39021527577903e-07, "loss": 0.102, "num_tokens": 11686016.0, "reward": 0.776611328125, "reward_std": 0.009554125368595123, "rewards//mean": 0.776611328125, "rewards//std": 0.022259533405303955, "step": 1352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2706, "grad_norm": 23.33195686340332, "kl": 3.4384165573865175, "learning_rate": 8.387882128194092e-07, "loss": 0.1375, "num_tokens": 11694712.0, "reward": 0.78271484375, "reward_std": 0.008728314191102982, "rewards//mean": 0.78271484375, "rewards//std": 0.03264866769313812, "step": 1353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2708, "grad_norm": 16.409711837768555, "kl": 1.1684235334396362, "learning_rate": 8.385547615970638e-07, "loss": 0.0467, "num_tokens": 11703384.0, "reward": 0.73870849609375, "reward_std": 0.0020206072367727757, "rewards//mean": 0.73870849609375, "rewards//std": 0.018676968291401863, "step": 1354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.271, "grad_norm": 6.225190162658691, "kl": 2.5341316740959883, "learning_rate": 8.38321174004901e-07, "loss": 0.1014, "num_tokens": 11712016.0, "reward": 0.77471923828125, "reward_std": 0.0039366185665130615, "rewards//mean": 0.77471923828125, "rewards//std": 0.020857207477092743, "step": 1355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2712, "grad_norm": 11.49312973022461, "kl": 3.6324777752161026, "learning_rate": 8.380874501370097e-07, "loss": 0.1453, "num_tokens": 11720648.0, "reward": 0.7725830078125, "reward_std": 0.014991494826972485, "rewards//mean": 0.7725830078125, "rewards//std": 0.03184476122260094, "step": 1356 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2714, "grad_norm": 1.3001525402069092, "kl": 1.5222731325775385, "learning_rate": 8.378535900875338e-07, "loss": 0.0609, "num_tokens": 11729304.0, "reward": 0.73431396484375, "reward_std": 0.002391017973423004, "rewards//mean": 0.73431396484375, "rewards//std": 0.02544415183365345, "step": 1357 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2716, "grad_norm": 8.484679222106934, "kl": 2.7289376724511385, "learning_rate": 8.376195939506725e-07, "loss": 0.1092, "num_tokens": 11738040.0, "reward": 0.74029541015625, "reward_std": 0.00747756939381361, "rewards//mean": 0.74029541015625, "rewards//std": 0.03327684476971626, "step": 1358 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2718, "grad_norm": 18.6961727142334, "kl": 4.8908359333872795, "learning_rate": 8.373854618206789e-07, "loss": 0.1956, "num_tokens": 11746752.0, "reward": 0.73046875, "reward_std": 0.02031049132347107, "rewards//mean": 0.73046875, "rewards//std": 0.04448913410305977, "step": 1359 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.272, "grad_norm": 19.100324630737305, "kl": 2.1134145110845566, "learning_rate": 8.371511937918617e-07, "loss": 0.0845, "num_tokens": 11755448.0, "reward": 0.72198486328125, "reward_std": 0.007530339062213898, "rewards//mean": 0.72198486328125, "rewards//std": 0.030421772971749306, "step": 1360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2722, "grad_norm": 11.395090103149414, "kl": 2.889318846166134, "learning_rate": 8.369167899585839e-07, "loss": 0.1156, "num_tokens": 11764152.0, "reward": 0.75225830078125, "reward_std": 0.008851965889334679, "rewards//mean": 0.75225830078125, "rewards//std": 0.029502255842089653, "step": 1361 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2724, "grad_norm": 14.356400489807129, "kl": 1.5334529355168343, "learning_rate": 8.366822504152636e-07, "loss": 0.0613, "num_tokens": 11772776.0, "reward": 0.73876953125, "reward_std": 0.00911199301481247, "rewards//mean": 0.73876953125, "rewards//std": 0.026157667860388756, "step": 1362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2726, "grad_norm": 8.904847145080566, "kl": 2.355218816548586, "learning_rate": 8.364475752563728e-07, "loss": 0.0942, "num_tokens": 11781408.0, "reward": 0.75860595703125, "reward_std": 0.009294064715504646, "rewards//mean": 0.75860595703125, "rewards//std": 0.034303195774555206, "step": 1363 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2728, "grad_norm": 18.4609317779541, "kl": 1.0844163838773966, "learning_rate": 8.362127645764389e-07, "loss": 0.0434, "num_tokens": 11789976.0, "reward": 0.74090576171875, "reward_std": 0.006823434494435787, "rewards//mean": 0.74090576171875, "rewards//std": 0.027805369347333908, "step": 1364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.273, "grad_norm": 35.80815887451172, "kl": 8.231054339557886, "learning_rate": 8.359778184700439e-07, "loss": 0.3292, "num_tokens": 11798632.0, "reward": 0.78948974609375, "reward_std": 0.014547184109687805, "rewards//mean": 0.78948974609375, "rewards//std": 0.02923266962170601, "step": 1365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2732, "grad_norm": 46.54033660888672, "kl": 5.669671170413494, "learning_rate": 8.357427370318238e-07, "loss": 0.2268, "num_tokens": 11807256.0, "reward": 0.7642822265625, "reward_std": 0.01499166153371334, "rewards//mean": 0.7642822265625, "rewards//std": 0.028656136244535446, "step": 1366 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2734, "grad_norm": 49.29010009765625, "kl": 7.49124613031745, "learning_rate": 8.355075203564692e-07, "loss": 0.2996, "num_tokens": 11815920.0, "reward": 0.74505615234375, "reward_std": 0.010848349891602993, "rewards//mean": 0.74505615234375, "rewards//std": 0.0375593900680542, "step": 1367 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2736, "grad_norm": 31.461627960205078, "kl": 4.508681073784828, "learning_rate": 8.352721685387256e-07, "loss": 0.1803, "num_tokens": 11824560.0, "reward": 0.75567626953125, "reward_std": 0.008087869733572006, "rewards//mean": 0.75567626953125, "rewards//std": 0.027734503149986267, "step": 1368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2738, "grad_norm": 15.764370918273926, "kl": 5.868150206282735, "learning_rate": 8.350366816733926e-07, "loss": 0.2347, "num_tokens": 11833184.0, "reward": 0.74560546875, "reward_std": 0.00955010112375021, "rewards//mean": 0.74560546875, "rewards//std": 0.02954077534377575, "step": 1369 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.274, "grad_norm": 21.798931121826172, "kl": 1.0845754779875278, "learning_rate": 8.348010598553243e-07, "loss": 0.0434, "num_tokens": 11841856.0, "reward": 0.7723388671875, "reward_std": 0.005296029150485992, "rewards//mean": 0.7723388671875, "rewards//std": 0.022845840081572533, "step": 1370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2742, "grad_norm": 10.106986999511719, "kl": 3.576843526214361, "learning_rate": 8.34565303179429e-07, "loss": 0.1431, "num_tokens": 11850632.0, "reward": 0.7630615234375, "reward_std": 0.011000830680131912, "rewards//mean": 0.7630615234375, "rewards//std": 0.029158851131796837, "step": 1371 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2744, "grad_norm": 9.401496887207031, "kl": 2.606949105858803, "learning_rate": 8.343294117406698e-07, "loss": 0.1043, "num_tokens": 11859328.0, "reward": 0.7569580078125, "reward_std": 0.0065363505855202675, "rewards//mean": 0.7569580078125, "rewards//std": 0.0202970989048481, "step": 1372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2746, "grad_norm": 9.911111831665039, "kl": 2.0001470036804676, "learning_rate": 8.340933856340635e-07, "loss": 0.08, "num_tokens": 11867920.0, "reward": 0.73931884765625, "reward_std": 0.008149401284754276, "rewards//mean": 0.73931884765625, "rewards//std": 0.03379145264625549, "step": 1373 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2748, "grad_norm": 13.342793464660645, "kl": 3.7259758468717337, "learning_rate": 8.338572249546812e-07, "loss": 0.149, "num_tokens": 11876616.0, "reward": 0.75372314453125, "reward_std": 0.00650325370952487, "rewards//mean": 0.75372314453125, "rewards//std": 0.0354849174618721, "step": 1374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.275, "grad_norm": 8.230053901672363, "kl": 2.122542344033718, "learning_rate": 8.336209297976489e-07, "loss": 0.0849, "num_tokens": 11885248.0, "reward": 0.77911376953125, "reward_std": 0.010866462253034115, "rewards//mean": 0.77911376953125, "rewards//std": 0.018570488318800926, "step": 1375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2752, "grad_norm": 7.580397129058838, "kl": 5.059437431395054, "learning_rate": 8.333845002581458e-07, "loss": 0.2024, "num_tokens": 11893872.0, "reward": 0.76678466796875, "reward_std": 0.017937738448381424, "rewards//mean": 0.76678466796875, "rewards//std": 0.03117768093943596, "step": 1376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2754, "grad_norm": 6.89148473739624, "kl": 4.216024033725262, "learning_rate": 8.331479364314059e-07, "loss": 0.1686, "num_tokens": 11902448.0, "reward": 0.74981689453125, "reward_std": 0.015681136399507523, "rewards//mean": 0.74981689453125, "rewards//std": 0.03388584405183792, "step": 1377 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2756, "grad_norm": 3.76511549949646, "kl": 3.6495047602802515, "learning_rate": 8.32911238412717e-07, "loss": 0.146, "num_tokens": 11911064.0, "reward": 0.7918701171875, "reward_std": 0.010602817870676517, "rewards//mean": 0.7918701171875, "rewards//std": 0.02308838814496994, "step": 1378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2758, "grad_norm": 8.92732048034668, "kl": 2.4589269440621138, "learning_rate": 8.326744062974211e-07, "loss": 0.0984, "num_tokens": 11919760.0, "reward": 0.7510986328125, "reward_std": 0.011387133970856667, "rewards//mean": 0.7510986328125, "rewards//std": 0.02497313730418682, "step": 1379 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.276, "grad_norm": 8.535314559936523, "kl": 1.194465760141611, "learning_rate": 8.324374401809142e-07, "loss": 0.0478, "num_tokens": 11928344.0, "reward": 0.73162841796875, "reward_std": 0.0032463876996189356, "rewards//mean": 0.73162841796875, "rewards//std": 0.03741361200809479, "step": 1380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2762, "grad_norm": 3.3985280990600586, "kl": 3.09967577457428, "learning_rate": 8.322003401586461e-07, "loss": 0.124, "num_tokens": 11936976.0, "reward": 0.786865234375, "reward_std": 0.013017643243074417, "rewards//mean": 0.786865234375, "rewards//std": 0.03151145577430725, "step": 1381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2764, "grad_norm": 6.333940029144287, "kl": 2.7226822581142187, "learning_rate": 8.319631063261207e-07, "loss": 0.1089, "num_tokens": 11945576.0, "reward": 0.75115966796875, "reward_std": 0.006218246184289455, "rewards//mean": 0.75115966796875, "rewards//std": 0.021773340180516243, "step": 1382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2766, "grad_norm": 0.2580198347568512, "kl": 0.5196405425667763, "learning_rate": 8.317257387788958e-07, "loss": 0.0208, "num_tokens": 11954136.0, "reward": 0.74786376953125, "reward_std": 0.0005179004510864615, "rewards//mean": 0.74786376953125, "rewards//std": 0.021484991535544395, "step": 1383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2768, "grad_norm": 4.238036155700684, "kl": 4.758801337331533, "learning_rate": 8.314882376125831e-07, "loss": 0.1904, "num_tokens": 11962824.0, "reward": 0.7857666015625, "reward_std": 0.01685452274978161, "rewards//mean": 0.7857666015625, "rewards//std": 0.03431759029626846, "step": 1384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.277, "grad_norm": 1.8679053783416748, "kl": 1.3025844171643257, "learning_rate": 8.312506029228477e-07, "loss": 0.0521, "num_tokens": 11971504.0, "reward": 0.7469482421875, "reward_std": 0.003025681944563985, "rewards//mean": 0.7469482421875, "rewards//std": 0.026977859437465668, "step": 1385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2772, "grad_norm": 4.578019618988037, "kl": 3.4632092732936144, "learning_rate": 8.310128348054093e-07, "loss": 0.1385, "num_tokens": 11980152.0, "reward": 0.758056640625, "reward_std": 0.006674261763691902, "rewards//mean": 0.758056640625, "rewards//std": 0.03284374624490738, "step": 1386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2774, "grad_norm": 2.8782236576080322, "kl": 1.7442466355860233, "learning_rate": 8.307749333560404e-07, "loss": 0.0698, "num_tokens": 11988744.0, "reward": 0.77886962890625, "reward_std": 0.005287739913910627, "rewards//mean": 0.77886962890625, "rewards//std": 0.018487975001335144, "step": 1387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2776, "grad_norm": 3.385528087615967, "kl": 1.5798235796391964, "learning_rate": 8.305368986705681e-07, "loss": 0.0632, "num_tokens": 11997312.0, "reward": 0.73919677734375, "reward_std": 0.004757335409522057, "rewards//mean": 0.73919677734375, "rewards//std": 0.033995117992162704, "step": 1388 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2778, "grad_norm": 6.672630310058594, "kl": 2.228483470156789, "learning_rate": 8.302987308448723e-07, "loss": 0.0891, "num_tokens": 12005928.0, "reward": 0.75494384765625, "reward_std": 0.003914062865078449, "rewards//mean": 0.75494384765625, "rewards//std": 0.034542862325906754, "step": 1389 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.278, "grad_norm": 1.9289408922195435, "kl": 0.9908090904355049, "learning_rate": 8.300604299748874e-07, "loss": 0.0396, "num_tokens": 12014688.0, "reward": 0.7354736328125, "reward_std": 0.001987332245334983, "rewards//mean": 0.7354736328125, "rewards//std": 0.02719247341156006, "step": 1390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2782, "grad_norm": 20.58250617980957, "kl": 3.8105711452662945, "learning_rate": 8.298219961566008e-07, "loss": 0.1524, "num_tokens": 12023352.0, "reward": 0.74493408203125, "reward_std": 0.014214625582098961, "rewards//mean": 0.74493408203125, "rewards//std": 0.03664836660027504, "step": 1391 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2784, "grad_norm": 7.755046367645264, "kl": 5.07947096042335, "learning_rate": 8.295834294860534e-07, "loss": 0.2032, "num_tokens": 12031984.0, "reward": 0.75604248046875, "reward_std": 0.009290643967688084, "rewards//mean": 0.75604248046875, "rewards//std": 0.02914814092218876, "step": 1392 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2786, "grad_norm": 3.1130611896514893, "kl": 3.0101313907653093, "learning_rate": 8.293447300593402e-07, "loss": 0.1204, "num_tokens": 12040632.0, "reward": 0.7606201171875, "reward_std": 0.008227781392633915, "rewards//mean": 0.7606201171875, "rewards//std": 0.02051076479256153, "step": 1393 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2788, "grad_norm": 5.169640064239502, "kl": 5.0863844864070415, "learning_rate": 8.291058979726091e-07, "loss": 0.2035, "num_tokens": 12049192.0, "reward": 0.7628173828125, "reward_std": 0.008807415142655373, "rewards//mean": 0.7628173828125, "rewards//std": 0.030149174854159355, "step": 1394 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.279, "grad_norm": 3.1735270023345947, "kl": 3.724377514794469, "learning_rate": 8.288669333220614e-07, "loss": 0.149, "num_tokens": 12057776.0, "reward": 0.7547607421875, "reward_std": 0.009010215289890766, "rewards//mean": 0.7547607421875, "rewards//std": 0.015096287243068218, "step": 1395 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2792, "grad_norm": 7.62028694152832, "kl": 6.1161501705646515, "learning_rate": 8.286278362039527e-07, "loss": 0.2446, "num_tokens": 12066448.0, "reward": 0.7623291015625, "reward_std": 0.010623966343700886, "rewards//mean": 0.7623291015625, "rewards//std": 0.024766186252236366, "step": 1396 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2794, "grad_norm": 1.2890839576721191, "kl": 1.417680436745286, "learning_rate": 8.283886067145906e-07, "loss": 0.0567, "num_tokens": 12075056.0, "reward": 0.7945556640625, "reward_std": 0.005159334745258093, "rewards//mean": 0.7945556640625, "rewards//std": 0.024623975157737732, "step": 1397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2796, "grad_norm": 3.1143851280212402, "kl": 2.9287510327994823, "learning_rate": 8.281492449503372e-07, "loss": 0.1172, "num_tokens": 12083688.0, "reward": 0.7630615234375, "reward_std": 0.005025902297347784, "rewards//mean": 0.7630615234375, "rewards//std": 0.023849500343203545, "step": 1398 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2798, "grad_norm": 5.231459140777588, "kl": 3.7067789547145367, "learning_rate": 8.279097510076069e-07, "loss": 0.1483, "num_tokens": 12092360.0, "reward": 0.779296875, "reward_std": 0.006181907840073109, "rewards//mean": 0.779296875, "rewards//std": 0.027116939425468445, "step": 1399 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.28, "grad_norm": 2.3263142108917236, "kl": 3.072752768173814, "learning_rate": 8.276701249828684e-07, "loss": 0.1229, "num_tokens": 12100960.0, "reward": 0.75860595703125, "reward_std": 0.009692623279988766, "rewards//mean": 0.75860595703125, "rewards//std": 0.03468237817287445, "step": 1400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2802, "grad_norm": 22.592851638793945, "kl": 1.1277550496160984, "learning_rate": 8.274303669726426e-07, "loss": 0.0451, "num_tokens": 12109584.0, "reward": 0.7689208984375, "reward_std": 0.004550518933683634, "rewards//mean": 0.7689208984375, "rewards//std": 0.036059003323316574, "step": 1401 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2804, "grad_norm": 2.93373441696167, "kl": 3.0542775746434927, "learning_rate": 8.271904770735041e-07, "loss": 0.1222, "num_tokens": 12118208.0, "reward": 0.77899169921875, "reward_std": 0.0068513015285134315, "rewards//mean": 0.77899169921875, "rewards//std": 0.022965705022215843, "step": 1402 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2806, "grad_norm": 25.050630569458008, "kl": 2.522257875651121, "learning_rate": 8.269504553820805e-07, "loss": 0.1009, "num_tokens": 12126832.0, "reward": 0.79827880859375, "reward_std": 0.008362634107470512, "rewards//mean": 0.79827880859375, "rewards//std": 0.021703006699681282, "step": 1403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2808, "grad_norm": 10.785796165466309, "kl": 7.249267783015966, "learning_rate": 8.267103019950528e-07, "loss": 0.29, "num_tokens": 12135432.0, "reward": 0.72796630859375, "reward_std": 0.01497596874833107, "rewards//mean": 0.72796630859375, "rewards//std": 0.03942888230085373, "step": 1404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.281, "grad_norm": 6.517972469329834, "kl": 4.577779654413462, "learning_rate": 8.264700170091543e-07, "loss": 0.1831, "num_tokens": 12143992.0, "reward": 0.75030517578125, "reward_std": 0.00809216033667326, "rewards//mean": 0.75030517578125, "rewards//std": 0.019304296001791954, "step": 1405 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2812, "grad_norm": 31.89670753479004, "kl": 9.678798243403435, "learning_rate": 8.262296005211721e-07, "loss": 0.3872, "num_tokens": 12152592.0, "reward": 0.74310302734375, "reward_std": 0.005614377558231354, "rewards//mean": 0.74310302734375, "rewards//std": 0.027715394273400307, "step": 1406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2814, "grad_norm": 32.459800720214844, "kl": 7.412335563451052, "learning_rate": 8.259890526279459e-07, "loss": 0.2965, "num_tokens": 12161296.0, "reward": 0.77996826171875, "reward_std": 0.011843318119645119, "rewards//mean": 0.77996826171875, "rewards//std": 0.023334499448537827, "step": 1407 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2816, "grad_norm": 9.2677001953125, "kl": 6.327021928504109, "learning_rate": 8.257483734263681e-07, "loss": 0.2531, "num_tokens": 12169976.0, "reward": 0.72503662109375, "reward_std": 0.015490911900997162, "rewards//mean": 0.72503662109375, "rewards//std": 0.04194832220673561, "step": 1408 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2818, "grad_norm": 8.746179580688477, "kl": 1.0971181858330965, "learning_rate": 8.255075630133845e-07, "loss": 0.0439, "num_tokens": 12178552.0, "reward": 0.77410888671875, "reward_std": 0.0032300525344908237, "rewards//mean": 0.77410888671875, "rewards//std": 0.018170008435845375, "step": 1409 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.282, "grad_norm": 19.859661102294922, "kl": 3.708829317241907, "learning_rate": 8.252666214859934e-07, "loss": 0.1484, "num_tokens": 12187224.0, "reward": 0.7470703125, "reward_std": 0.01625164970755577, "rewards//mean": 0.7470703125, "rewards//std": 0.03463919460773468, "step": 1410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2822, "grad_norm": 9.35738468170166, "kl": 4.432959834113717, "learning_rate": 8.250255489412462e-07, "loss": 0.1773, "num_tokens": 12195976.0, "reward": 0.76312255859375, "reward_std": 0.009246166795492172, "rewards//mean": 0.76312255859375, "rewards//std": 0.0362609401345253, "step": 1411 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2824, "grad_norm": 14.963501930236816, "kl": 5.9976518508046865, "learning_rate": 8.247843454762466e-07, "loss": 0.2399, "num_tokens": 12204544.0, "reward": 0.764404296875, "reward_std": 0.018967021256685257, "rewards//mean": 0.764404296875, "rewards//std": 0.03559018298983574, "step": 1412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2826, "grad_norm": 8.764498710632324, "kl": 3.0336758885532618, "learning_rate": 8.245430111881517e-07, "loss": 0.1213, "num_tokens": 12213224.0, "reward": 0.78759765625, "reward_std": 0.019061461091041565, "rewards//mean": 0.78759765625, "rewards//std": 0.032170381397008896, "step": 1413 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2828, "grad_norm": 10.85020923614502, "kl": 4.418082553893328, "learning_rate": 8.243015461741706e-07, "loss": 0.1767, "num_tokens": 12221920.0, "reward": 0.75579833984375, "reward_std": 0.008593752048909664, "rewards//mean": 0.75579833984375, "rewards//std": 0.03110233135521412, "step": 1414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.283, "grad_norm": 2.4739699363708496, "kl": 1.2441857382655144, "learning_rate": 8.240599505315654e-07, "loss": 0.0498, "num_tokens": 12230600.0, "reward": 0.77001953125, "reward_std": 0.004998262505978346, "rewards//mean": 0.77001953125, "rewards//std": 0.033214978873729706, "step": 1415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2832, "grad_norm": 6.969664096832275, "kl": 4.846530532464385, "learning_rate": 8.238182243576511e-07, "loss": 0.1939, "num_tokens": 12239248.0, "reward": 0.767822265625, "reward_std": 0.010738326236605644, "rewards//mean": 0.767822265625, "rewards//std": 0.024850696325302124, "step": 1416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2834, "grad_norm": 18.58360481262207, "kl": 3.6598623655736446, "learning_rate": 8.235763677497945e-07, "loss": 0.1464, "num_tokens": 12247824.0, "reward": 0.793701171875, "reward_std": 0.008233538828790188, "rewards//mean": 0.793701171875, "rewards//std": 0.025179890915751457, "step": 1417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2836, "grad_norm": 5.1898627281188965, "kl": 3.482665505260229, "learning_rate": 8.233343808054157e-07, "loss": 0.1393, "num_tokens": 12256496.0, "reward": 0.750732421875, "reward_std": 0.00788567028939724, "rewards//mean": 0.750732421875, "rewards//std": 0.03515194356441498, "step": 1418 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2838, "grad_norm": 5.801996231079102, "kl": 1.4907474257051945, "learning_rate": 8.23092263621987e-07, "loss": 0.0596, "num_tokens": 12265152.0, "reward": 0.75091552734375, "reward_std": 0.004526809323579073, "rewards//mean": 0.75091552734375, "rewards//std": 0.03150274604558945, "step": 1419 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.284, "grad_norm": 11.066574096679688, "kl": 3.5229289308190346, "learning_rate": 8.228500162970332e-07, "loss": 0.1409, "num_tokens": 12273816.0, "reward": 0.7413330078125, "reward_std": 0.011826589703559875, "rewards//mean": 0.7413330078125, "rewards//std": 0.040675222873687744, "step": 1420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2842, "grad_norm": 14.3084716796875, "kl": 4.823999624699354, "learning_rate": 8.226076389281314e-07, "loss": 0.193, "num_tokens": 12282528.0, "reward": 0.7667236328125, "reward_std": 0.014889299869537354, "rewards//mean": 0.7667236328125, "rewards//std": 0.031392816454172134, "step": 1421 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2844, "grad_norm": 6.342036724090576, "kl": 3.629468122497201, "learning_rate": 8.223651316129114e-07, "loss": 0.1452, "num_tokens": 12291168.0, "reward": 0.72991943359375, "reward_std": 0.014151573181152344, "rewards//mean": 0.72991943359375, "rewards//std": 0.024762749671936035, "step": 1422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2846, "grad_norm": 3.0061469078063965, "kl": 2.580713367089629, "learning_rate": 8.221224944490548e-07, "loss": 0.1032, "num_tokens": 12299824.0, "reward": 0.76361083984375, "reward_std": 0.006728575564920902, "rewards//mean": 0.76361083984375, "rewards//std": 0.03074203059077263, "step": 1423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2848, "grad_norm": 6.261436462402344, "kl": 2.5072155445814133, "learning_rate": 8.21879727534296e-07, "loss": 0.1003, "num_tokens": 12308464.0, "reward": 0.77105712890625, "reward_std": 0.004622997250407934, "rewards//mean": 0.77105712890625, "rewards//std": 0.017071232199668884, "step": 1424 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.285, "grad_norm": 6.13707160949707, "kl": 1.569300588220358, "learning_rate": 8.216368309664213e-07, "loss": 0.0628, "num_tokens": 12316984.0, "reward": 0.772705078125, "reward_std": 0.008392134681344032, "rewards//mean": 0.772705078125, "rewards//std": 0.029674729332327843, "step": 1425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2852, "grad_norm": 0.04022868350148201, "kl": 0.5108750946819782, "learning_rate": 8.213938048432696e-07, "loss": 0.0204, "num_tokens": 12325648.0, "reward": 0.76953125, "reward_std": 0.0, "rewards//mean": 0.76953125, "rewards//std": 0.018466776236891747, "step": 1426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2854, "grad_norm": 4.986400604248047, "kl": 2.078704100102186, "learning_rate": 8.211506492627318e-07, "loss": 0.0831, "num_tokens": 12334280.0, "reward": 0.74029541015625, "reward_std": 0.008886216208338737, "rewards//mean": 0.74029541015625, "rewards//std": 0.028030389919877052, "step": 1427 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2856, "grad_norm": 3.4062979221343994, "kl": 1.296766459941864, "learning_rate": 8.209073643227509e-07, "loss": 0.0519, "num_tokens": 12342960.0, "reward": 0.75018310546875, "reward_std": 0.004315837286412716, "rewards//mean": 0.75018310546875, "rewards//std": 0.0354849174618721, "step": 1428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2858, "grad_norm": 6.825105667114258, "kl": 1.2616166323423386, "learning_rate": 8.206639501213219e-07, "loss": 0.0505, "num_tokens": 12351680.0, "reward": 0.81048583984375, "reward_std": 0.0031310291960835457, "rewards//mean": 0.81048583984375, "rewards//std": 0.01588621363043785, "step": 1429 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 126.453125, "epoch": 0.286, "grad_norm": 15.299839973449707, "kl": 5.4106599148362875, "learning_rate": 8.204204067564924e-07, "loss": 0.1848, "num_tokens": 12360205.0, "reward": 0.76239013671875, "reward_std": 0.01022109854966402, "rewards//mean": 0.76239013671875, "rewards//std": 0.021559547632932663, "step": 1430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2862, "grad_norm": 3.5731098651885986, "kl": 3.9427863769233227, "learning_rate": 8.201767343263611e-07, "loss": 0.1577, "num_tokens": 12368885.0, "reward": 0.730712890625, "reward_std": 0.013175707310438156, "rewards//mean": 0.730712890625, "rewards//std": 0.04189203307032585, "step": 1431 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2864, "grad_norm": 11.419472694396973, "kl": 5.961509704589844, "learning_rate": 8.199329329290796e-07, "loss": 0.2385, "num_tokens": 12377629.0, "reward": 0.7703857421875, "reward_std": 0.01069999486207962, "rewards//mean": 0.7703857421875, "rewards//std": 0.03586537390947342, "step": 1432 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2866, "grad_norm": 5.172885894775391, "kl": 4.274353949353099, "learning_rate": 8.19689002662851e-07, "loss": 0.171, "num_tokens": 12386205.0, "reward": 0.7227783203125, "reward_std": 0.010881026275455952, "rewards//mean": 0.7227783203125, "rewards//std": 0.036551013588905334, "step": 1433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2868, "grad_norm": 5.536722183227539, "kl": 3.0592558216303587, "learning_rate": 8.194449436259303e-07, "loss": 0.1224, "num_tokens": 12394781.0, "reward": 0.7886962890625, "reward_std": 0.006747500505298376, "rewards//mean": 0.7886962890625, "rewards//std": 0.020047960802912712, "step": 1434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.287, "grad_norm": 6.103031635284424, "kl": 1.1958512663841248, "learning_rate": 8.192007559166247e-07, "loss": 0.0478, "num_tokens": 12403413.0, "reward": 0.73822021484375, "reward_std": 0.00039866380393505096, "rewards//mean": 0.73822021484375, "rewards//std": 0.02081579715013504, "step": 1435 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2872, "grad_norm": 2.698838710784912, "kl": 2.695367092266679, "learning_rate": 8.189564396332926e-07, "loss": 0.1078, "num_tokens": 12412045.0, "reward": 0.79327392578125, "reward_std": 0.008445193991065025, "rewards//mean": 0.79327392578125, "rewards//std": 0.02185799553990364, "step": 1436 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2874, "grad_norm": 1.4131667613983154, "kl": 1.029098181053996, "learning_rate": 8.187119948743449e-07, "loss": 0.0412, "num_tokens": 12420741.0, "reward": 0.73797607421875, "reward_std": 0.0010980158112943172, "rewards//mean": 0.73797607421875, "rewards//std": 0.01357283629477024, "step": 1437 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2876, "grad_norm": 3.045125722885132, "kl": 3.2402882408350706, "learning_rate": 8.184674217382437e-07, "loss": 0.1296, "num_tokens": 12429421.0, "reward": 0.724609375, "reward_std": 0.008203878998756409, "rewards//mean": 0.724609375, "rewards//std": 0.03579460829496384, "step": 1438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2878, "grad_norm": 3.918938159942627, "kl": 2.795470768585801, "learning_rate": 8.182227203235031e-07, "loss": 0.1118, "num_tokens": 12438141.0, "reward": 0.76739501953125, "reward_std": 0.008533384650945663, "rewards//mean": 0.76739501953125, "rewards//std": 0.029748542234301567, "step": 1439 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.288, "grad_norm": 3.2863383293151855, "kl": 2.1188965663313866, "learning_rate": 8.179778907286887e-07, "loss": 0.0848, "num_tokens": 12446805.0, "reward": 0.7318115234375, "reward_std": 0.005179004743695259, "rewards//mean": 0.7318115234375, "rewards//std": 0.023242603987455368, "step": 1440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2882, "grad_norm": 22.631269454956055, "kl": 8.918640416115522, "learning_rate": 8.177329330524181e-07, "loss": 0.3567, "num_tokens": 12455789.0, "reward": 0.6925048828125, "reward_std": 0.013418271206319332, "rewards//mean": 0.6925048828125, "rewards//std": 0.04893512278795242, "step": 1441 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2884, "grad_norm": 5.682319164276123, "kl": 4.445296753197908, "learning_rate": 8.1748784739336e-07, "loss": 0.1778, "num_tokens": 12464421.0, "reward": 0.74005126953125, "reward_std": 0.005198127590119839, "rewards//mean": 0.74005126953125, "rewards//std": 0.02654763124883175, "step": 1442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2886, "grad_norm": 1.4470422267913818, "kl": 1.0345830917358398, "learning_rate": 8.17242633850235e-07, "loss": 0.0414, "num_tokens": 12473037.0, "reward": 0.771484375, "reward_std": 0.0019523651571944356, "rewards//mean": 0.771484375, "rewards//std": 0.016703827306628227, "step": 1443 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2888, "grad_norm": 2.220828056335449, "kl": 2.345273908227682, "learning_rate": 8.16997292521815e-07, "loss": 0.0938, "num_tokens": 12481677.0, "reward": 0.78546142578125, "reward_std": 0.005727018695324659, "rewards//mean": 0.78546142578125, "rewards//std": 0.026605729013681412, "step": 1444 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.289, "grad_norm": 5.149527549743652, "kl": 4.182993354275823, "learning_rate": 8.167518235069234e-07, "loss": 0.1673, "num_tokens": 12490309.0, "reward": 0.771484375, "reward_std": 0.00904484186321497, "rewards//mean": 0.771484375, "rewards//std": 0.024449070915579796, "step": 1445 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2892, "grad_norm": 1.7928732633590698, "kl": 2.3073074258863926, "learning_rate": 8.165062269044352e-07, "loss": 0.0923, "num_tokens": 12498853.0, "reward": 0.74029541015625, "reward_std": 0.005118072032928467, "rewards//mean": 0.74029541015625, "rewards//std": 0.02886476367712021, "step": 1446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2894, "grad_norm": 2.6357505321502686, "kl": 1.0002813152968884, "learning_rate": 8.162605028132768e-07, "loss": 0.04, "num_tokens": 12507541.0, "reward": 0.78741455078125, "reward_std": 0.0024336238857358694, "rewards//mean": 0.78741455078125, "rewards//std": 0.01627499610185623, "step": 1447 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2896, "grad_norm": 12.989418983459473, "kl": 6.614552399143577, "learning_rate": 8.160146513324254e-07, "loss": 0.2646, "num_tokens": 12516237.0, "reward": 0.723388671875, "reward_std": 0.008351363241672516, "rewards//mean": 0.723388671875, "rewards//std": 0.02559962496161461, "step": 1448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2898, "grad_norm": 13.686028480529785, "kl": 2.834360908716917, "learning_rate": 8.157686725609105e-07, "loss": 0.1134, "num_tokens": 12524869.0, "reward": 0.78387451171875, "reward_std": 0.009840108454227448, "rewards//mean": 0.78387451171875, "rewards//std": 0.02367454767227173, "step": 1449 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.29, "grad_norm": 12.314616203308105, "kl": 6.877235503867269, "learning_rate": 8.155225665978118e-07, "loss": 0.2751, "num_tokens": 12533581.0, "reward": 0.74212646484375, "reward_std": 0.00912742130458355, "rewards//mean": 0.74212646484375, "rewards//std": 0.03233553096652031, "step": 1450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2902, "grad_norm": 3.5310564041137695, "kl": 1.5004207510501146, "learning_rate": 8.152763335422612e-07, "loss": 0.06, "num_tokens": 12542157.0, "reward": 0.77142333984375, "reward_std": 0.006713591516017914, "rewards//mean": 0.77142333984375, "rewards//std": 0.025065330788493156, "step": 1451 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2904, "grad_norm": 3.1590774059295654, "kl": 2.933615291491151, "learning_rate": 8.150299734934412e-07, "loss": 0.1173, "num_tokens": 12550805.0, "reward": 0.74053955078125, "reward_std": 0.0076324716210365295, "rewards//mean": 0.74053955078125, "rewards//std": 0.021951979026198387, "step": 1452 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2906, "grad_norm": 5.115265846252441, "kl": 3.4152608290314674, "learning_rate": 8.147834865505853e-07, "loss": 0.1366, "num_tokens": 12559397.0, "reward": 0.73199462890625, "reward_std": 0.012220550328493118, "rewards//mean": 0.73199462890625, "rewards//std": 0.041102588176727295, "step": 1453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2908, "grad_norm": 0.9487642645835876, "kl": 1.1078472509980202, "learning_rate": 8.145368728129789e-07, "loss": 0.0443, "num_tokens": 12567949.0, "reward": 0.74945068359375, "reward_std": 0.0013736373512074351, "rewards//mean": 0.74945068359375, "rewards//std": 0.02379508875310421, "step": 1454 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.291, "grad_norm": 1.6968648433685303, "kl": 1.4060956872999668, "learning_rate": 8.142901323799577e-07, "loss": 0.0562, "num_tokens": 12576581.0, "reward": 0.74066162109375, "reward_std": 0.0010767546482384205, "rewards//mean": 0.74066162109375, "rewards//std": 0.026608573272824287, "step": 1455 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2912, "grad_norm": 5.425151824951172, "kl": 2.681792926043272, "learning_rate": 8.140432653509087e-07, "loss": 0.1073, "num_tokens": 12585237.0, "reward": 0.76397705078125, "reward_std": 0.005783388391137123, "rewards//mean": 0.76397705078125, "rewards//std": 0.028295380994677544, "step": 1456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2914, "grad_norm": 3.9536635875701904, "kl": 1.6290787030011415, "learning_rate": 8.1379627182527e-07, "loss": 0.0652, "num_tokens": 12593853.0, "reward": 0.75933837890625, "reward_std": 0.0020354087464511395, "rewards//mean": 0.75933837890625, "rewards//std": 0.03408672288060188, "step": 1457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2916, "grad_norm": 18.338483810424805, "kl": 3.4971011132001877, "learning_rate": 8.135491519025306e-07, "loss": 0.1399, "num_tokens": 12602573.0, "reward": 0.75213623046875, "reward_std": 0.005087354686111212, "rewards//mean": 0.75213623046875, "rewards//std": 0.0330573134124279, "step": 1458 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2918, "grad_norm": 3.185584306716919, "kl": 4.476410383358598, "learning_rate": 8.133019056822302e-07, "loss": 0.1791, "num_tokens": 12611237.0, "reward": 0.76300048828125, "reward_std": 0.006838707718998194, "rewards//mean": 0.76300048828125, "rewards//std": 0.026199806481599808, "step": 1459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.292, "grad_norm": 5.106291770935059, "kl": 3.9950185026973486, "learning_rate": 8.130545332639597e-07, "loss": 0.1598, "num_tokens": 12619877.0, "reward": 0.74127197265625, "reward_std": 0.00920344889163971, "rewards//mean": 0.74127197265625, "rewards//std": 0.02624310366809368, "step": 1460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2922, "grad_norm": 22.210561752319336, "kl": 6.73808372579515, "learning_rate": 8.128070347473608e-07, "loss": 0.2695, "num_tokens": 12628621.0, "reward": 0.77655029296875, "reward_std": 0.007952040061354637, "rewards//mean": 0.77655029296875, "rewards//std": 0.030759258195757866, "step": 1461 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2924, "grad_norm": 28.265748977661133, "kl": 5.9607279896736145, "learning_rate": 8.125594102321255e-07, "loss": 0.2384, "num_tokens": 12637245.0, "reward": 0.77655029296875, "reward_std": 0.00983298197388649, "rewards//mean": 0.77655029296875, "rewards//std": 0.03253481537103653, "step": 1462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2926, "grad_norm": 13.571913719177246, "kl": 3.088777916505933, "learning_rate": 8.123116598179971e-07, "loss": 0.1236, "num_tokens": 12645997.0, "reward": 0.759765625, "reward_std": 0.011301582679152489, "rewards//mean": 0.759765625, "rewards//std": 0.03557742014527321, "step": 1463 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2928, "grad_norm": 53.40056610107422, "kl": 8.706040035933256, "learning_rate": 8.120637836047697e-07, "loss": 0.3482, "num_tokens": 12654605.0, "reward": 0.744384765625, "reward_std": 0.012703519314527512, "rewards//mean": 0.744384765625, "rewards//std": 0.03117142803966999, "step": 1464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.293, "grad_norm": 38.953224182128906, "kl": 6.153975997120142, "learning_rate": 8.118157816922874e-07, "loss": 0.2462, "num_tokens": 12663277.0, "reward": 0.725830078125, "reward_std": 0.009198610670864582, "rewards//mean": 0.725830078125, "rewards//std": 0.0242488831281662, "step": 1465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.078125, "epoch": 0.2932, "grad_norm": 20.69542694091797, "kl": 7.51901463791728, "learning_rate": 8.115676541804455e-07, "loss": 0.2882, "num_tokens": 12671834.0, "reward": 0.72149658203125, "reward_std": 0.01875142753124237, "rewards//mean": 0.72149658203125, "rewards//std": 0.042692676186561584, "step": 1466 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2934, "grad_norm": 14.31985855102539, "kl": 6.497452400624752, "learning_rate": 8.113194011691899e-07, "loss": 0.2599, "num_tokens": 12680426.0, "reward": 0.750244140625, "reward_std": 0.012425040826201439, "rewards//mean": 0.750244140625, "rewards//std": 0.031878259032964706, "step": 1467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2936, "grad_norm": 7.346076011657715, "kl": 2.856573548167944, "learning_rate": 8.110710227585167e-07, "loss": 0.1143, "num_tokens": 12689010.0, "reward": 0.78509521484375, "reward_std": 0.008657204918563366, "rewards//mean": 0.78509521484375, "rewards//std": 0.01946905069053173, "step": 1468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2938, "grad_norm": 16.012229919433594, "kl": 1.4431650210171938, "learning_rate": 8.108225190484726e-07, "loss": 0.0577, "num_tokens": 12697682.0, "reward": 0.73443603515625, "reward_std": 0.004821329843252897, "rewards//mean": 0.73443603515625, "rewards//std": 0.03247007727622986, "step": 1469 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.294, "grad_norm": 7.685550689697266, "kl": 4.789723701775074, "learning_rate": 8.105738901391551e-07, "loss": 0.1916, "num_tokens": 12706426.0, "reward": 0.79119873046875, "reward_std": 0.016323965042829514, "rewards//mean": 0.79119873046875, "rewards//std": 0.031148534268140793, "step": 1470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2942, "grad_norm": 13.609522819519043, "kl": 4.810040263459086, "learning_rate": 8.103251361307118e-07, "loss": 0.1924, "num_tokens": 12715122.0, "reward": 0.74664306640625, "reward_std": 0.0036240937188267708, "rewards//mean": 0.74664306640625, "rewards//std": 0.030583063140511513, "step": 1471 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2944, "grad_norm": 15.771655082702637, "kl": 3.396984687075019, "learning_rate": 8.100762571233408e-07, "loss": 0.1359, "num_tokens": 12723730.0, "reward": 0.77655029296875, "reward_std": 0.008351262658834457, "rewards//mean": 0.77655029296875, "rewards//std": 0.021566567942500114, "step": 1472 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2946, "grad_norm": 5.584359169006348, "kl": 2.603633690625429, "learning_rate": 8.098272532172905e-07, "loss": 0.1041, "num_tokens": 12732546.0, "reward": 0.75225830078125, "reward_std": 0.006755417678505182, "rewards//mean": 0.75225830078125, "rewards//std": 0.03003114089369774, "step": 1473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2948, "grad_norm": 7.366146564483643, "kl": 3.0774579513818026, "learning_rate": 8.095781245128597e-07, "loss": 0.1231, "num_tokens": 12741266.0, "reward": 0.752197265625, "reward_std": 0.015443527139723301, "rewards//mean": 0.752197265625, "rewards//std": 0.037492476403713226, "step": 1474 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.295, "grad_norm": 4.636621475219727, "kl": 3.1825755424797535, "learning_rate": 8.093288711103971e-07, "loss": 0.1273, "num_tokens": 12749826.0, "reward": 0.75592041015625, "reward_std": 0.005983960349112749, "rewards//mean": 0.75592041015625, "rewards//std": 0.028881540521979332, "step": 1475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2952, "grad_norm": 11.739696502685547, "kl": 2.0741084031760693, "learning_rate": 8.090794931103026e-07, "loss": 0.083, "num_tokens": 12758434.0, "reward": 0.760498046875, "reward_std": 0.006874276790767908, "rewards//mean": 0.760498046875, "rewards//std": 0.023559898138046265, "step": 1476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2954, "grad_norm": 6.742212772369385, "kl": 3.330030683428049, "learning_rate": 8.08829990613025e-07, "loss": 0.1332, "num_tokens": 12767034.0, "reward": 0.75030517578125, "reward_std": 0.00829287339001894, "rewards//mean": 0.75030517578125, "rewards//std": 0.03372015058994293, "step": 1477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2956, "grad_norm": 7.87825345993042, "kl": 3.7539873644709587, "learning_rate": 8.085803637190643e-07, "loss": 0.1502, "num_tokens": 12775674.0, "reward": 0.7900390625, "reward_std": 0.01460934802889824, "rewards//mean": 0.7900390625, "rewards//std": 0.03276345133781433, "step": 1478 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2958, "grad_norm": 3.2344954013824463, "kl": 3.3349881507456303, "learning_rate": 8.083306125289697e-07, "loss": 0.1334, "num_tokens": 12784202.0, "reward": 0.75860595703125, "reward_std": 0.010151240974664688, "rewards//mean": 0.75860595703125, "rewards//std": 0.034190040081739426, "step": 1479 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.296, "grad_norm": 7.749207496643066, "kl": 1.7467486821115017, "learning_rate": 8.080807371433414e-07, "loss": 0.0699, "num_tokens": 12792810.0, "reward": 0.76861572265625, "reward_std": 0.009853655472397804, "rewards//mean": 0.76861572265625, "rewards//std": 0.022258087992668152, "step": 1480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2962, "grad_norm": 5.620584487915039, "kl": 2.2123597227036953, "learning_rate": 8.07830737662829e-07, "loss": 0.0885, "num_tokens": 12801482.0, "reward": 0.74835205078125, "reward_std": 0.004649493843317032, "rewards//mean": 0.74835205078125, "rewards//std": 0.025198446586728096, "step": 1481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2964, "grad_norm": 3.8430168628692627, "kl": 3.042467314749956, "learning_rate": 8.075806141881325e-07, "loss": 0.1217, "num_tokens": 12810218.0, "reward": 0.7652587890625, "reward_std": 0.011134494096040726, "rewards//mean": 0.7652587890625, "rewards//std": 0.0349019318819046, "step": 1482 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2966, "grad_norm": 16.194698333740234, "kl": 1.9202016908675432, "learning_rate": 8.073303668200011e-07, "loss": 0.0768, "num_tokens": 12818898.0, "reward": 0.77398681640625, "reward_std": 0.006984813138842583, "rewards//mean": 0.77398681640625, "rewards//std": 0.02519904635846615, "step": 1483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.140625, "epoch": 0.2968, "grad_norm": 15.359682083129883, "kl": 3.4303321093320847, "learning_rate": 8.070799956592349e-07, "loss": 0.1215, "num_tokens": 12827403.0, "reward": 0.74493408203125, "reward_std": 0.00778286624699831, "rewards//mean": 0.74493408203125, "rewards//std": 0.025569375604391098, "step": 1484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.297, "grad_norm": 15.314162254333496, "kl": 0.5339925810694695, "learning_rate": 8.06829500806683e-07, "loss": 0.0214, "num_tokens": 12836035.0, "reward": 0.7694091796875, "reward_std": 0.0026868688873946667, "rewards//mean": 0.7694091796875, "rewards//std": 0.021349729970097542, "step": 1485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2972, "grad_norm": 13.411092758178711, "kl": 6.477550081908703, "learning_rate": 8.06578882363245e-07, "loss": 0.2591, "num_tokens": 12844763.0, "reward": 0.7552490234375, "reward_std": 0.019006986171007156, "rewards//mean": 0.7552490234375, "rewards//std": 0.03357556834816933, "step": 1486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2974, "grad_norm": 9.768086433410645, "kl": 1.3256912045180798, "learning_rate": 8.063281404298699e-07, "loss": 0.053, "num_tokens": 12853435.0, "reward": 0.7830810546875, "reward_std": 0.006776880472898483, "rewards//mean": 0.7830810546875, "rewards//std": 0.028202474117279053, "step": 1487 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2976, "grad_norm": 24.31314468383789, "kl": 3.6333701610565186, "learning_rate": 8.060772751075562e-07, "loss": 0.1453, "num_tokens": 12862083.0, "reward": 0.7589111328125, "reward_std": 0.012979390099644661, "rewards//mean": 0.7589111328125, "rewards//std": 0.02355572022497654, "step": 1488 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2978, "grad_norm": 11.574331283569336, "kl": 1.9029613602906466, "learning_rate": 8.058262864973528e-07, "loss": 0.0761, "num_tokens": 12870739.0, "reward": 0.744873046875, "reward_std": 0.00930815003812313, "rewards//mean": 0.744873046875, "rewards//std": 0.03298356384038925, "step": 1489 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.298, "grad_norm": 24.865310668945312, "kl": 3.1038744542747736, "learning_rate": 8.055751747003579e-07, "loss": 0.1242, "num_tokens": 12879339.0, "reward": 0.73614501953125, "reward_std": 0.01602857932448387, "rewards//mean": 0.73614501953125, "rewards//std": 0.033501263707876205, "step": 1490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2982, "grad_norm": 26.77971839904785, "kl": 5.77071532048285, "learning_rate": 8.053239398177191e-07, "loss": 0.2308, "num_tokens": 12887931.0, "reward": 0.746826171875, "reward_std": 0.0146062932908535, "rewards//mean": 0.746826171875, "rewards//std": 0.033652350306510925, "step": 1491 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2984, "grad_norm": 11.379695892333984, "kl": 3.4608402214944363, "learning_rate": 8.050725819506339e-07, "loss": 0.1384, "num_tokens": 12896563.0, "reward": 0.753173828125, "reward_std": 0.010060532949864864, "rewards//mean": 0.753173828125, "rewards//std": 0.029715511947870255, "step": 1492 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2986, "grad_norm": 17.708709716796875, "kl": 3.076208494603634, "learning_rate": 8.048211012003489e-07, "loss": 0.123, "num_tokens": 12905275.0, "reward": 0.74200439453125, "reward_std": 0.010404210537672043, "rewards//mean": 0.74200439453125, "rewards//std": 0.02500183694064617, "step": 1493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2988, "grad_norm": 11.853999137878418, "kl": 1.472416803240776, "learning_rate": 8.045694976681612e-07, "loss": 0.0589, "num_tokens": 12913971.0, "reward": 0.74090576171875, "reward_std": 0.004838705062866211, "rewards//mean": 0.74090576171875, "rewards//std": 0.03022608533501625, "step": 1494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.299, "grad_norm": 14.601651191711426, "kl": 1.9102000687271357, "learning_rate": 8.043177714554159e-07, "loss": 0.0764, "num_tokens": 12922619.0, "reward": 0.77532958984375, "reward_std": 0.007799575570970774, "rewards//mean": 0.77532958984375, "rewards//std": 0.027081118896603584, "step": 1495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2992, "grad_norm": 8.707847595214844, "kl": 1.6990231294184923, "learning_rate": 8.04065922663509e-07, "loss": 0.068, "num_tokens": 12931323.0, "reward": 0.79119873046875, "reward_std": 0.007749687880277634, "rewards//mean": 0.79119873046875, "rewards//std": 0.031241703778505325, "step": 1496 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2994, "grad_norm": 13.480512619018555, "kl": 4.101980570703745, "learning_rate": 8.038139513938845e-07, "loss": 0.1641, "num_tokens": 12940035.0, "reward": 0.74798583984375, "reward_std": 0.011941012926399708, "rewards//mean": 0.74798583984375, "rewards//std": 0.031703922897577286, "step": 1497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2996, "grad_norm": 6.592166423797607, "kl": 4.812883865088224, "learning_rate": 8.035618577480369e-07, "loss": 0.1925, "num_tokens": 12948675.0, "reward": 0.78741455078125, "reward_std": 0.016344359144568443, "rewards//mean": 0.78741455078125, "rewards//std": 0.026280570775270462, "step": 1498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.2998, "grad_norm": 4.115764617919922, "kl": 3.6408120151609182, "learning_rate": 8.033096418275092e-07, "loss": 0.1456, "num_tokens": 12957371.0, "reward": 0.72442626953125, "reward_std": 0.01087966002523899, "rewards//mean": 0.72442626953125, "rewards//std": 0.042329467833042145, "step": 1499 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3, "grad_norm": 8.221807479858398, "kl": 1.7563366871327162, "learning_rate": 8.030573037338941e-07, "loss": 0.0703, "num_tokens": 12966027.0, "reward": 0.77227783203125, "reward_std": 0.008386017754673958, "rewards//mean": 0.77227783203125, "rewards//std": 0.025786321610212326, "step": 1500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3002, "grad_norm": 5.701700210571289, "kl": 0.5587639026343822, "learning_rate": 8.028048435688333e-07, "loss": 0.0224, "num_tokens": 12974603.0, "reward": 0.79888916015625, "reward_std": 0.001270482549443841, "rewards//mean": 0.79888916015625, "rewards//std": 0.019586877897381783, "step": 1501 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3004, "grad_norm": 3.7195465564727783, "kl": 2.6717438362538815, "learning_rate": 8.025522614340177e-07, "loss": 0.1069, "num_tokens": 12983131.0, "reward": 0.74420166015625, "reward_std": 0.010117185302078724, "rewards//mean": 0.74420166015625, "rewards//std": 0.03271160647273064, "step": 1502 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3006, "grad_norm": 3.6980080604553223, "kl": 3.450033061206341, "learning_rate": 8.022995574311875e-07, "loss": 0.138, "num_tokens": 12991811.0, "reward": 0.7396240234375, "reward_std": 0.010044010356068611, "rewards//mean": 0.7396240234375, "rewards//std": 0.025602877140045166, "step": 1503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3008, "grad_norm": 2.8952481746673584, "kl": 0.790763495489955, "learning_rate": 8.020467316621316e-07, "loss": 0.0316, "num_tokens": 13000387.0, "reward": 0.78289794921875, "reward_std": 0.001645436743274331, "rewards//mean": 0.78289794921875, "rewards//std": 0.022193383425474167, "step": 1504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.301, "grad_norm": 2.669785976409912, "kl": 4.1806445978581905, "learning_rate": 8.017937842286882e-07, "loss": 0.1672, "num_tokens": 13009059.0, "reward": 0.75775146484375, "reward_std": 0.013202082365751266, "rewards//mean": 0.75775146484375, "rewards//std": 0.028843779116868973, "step": 1505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3012, "grad_norm": 2.4568898677825928, "kl": 0.747019100934267, "learning_rate": 8.015407152327447e-07, "loss": 0.0299, "num_tokens": 13017659.0, "reward": 0.74664306640625, "reward_std": 0.0028939624316990376, "rewards//mean": 0.74664306640625, "rewards//std": 0.025218263268470764, "step": 1506 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3014, "grad_norm": 1.8890511989593506, "kl": 1.1498551853001118, "learning_rate": 8.012875247762372e-07, "loss": 0.046, "num_tokens": 13026251.0, "reward": 0.78240966796875, "reward_std": 0.002637900412082672, "rewards//mean": 0.78240966796875, "rewards//std": 0.019828831776976585, "step": 1507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3016, "grad_norm": 4.576070308685303, "kl": 1.594864496961236, "learning_rate": 8.010342129611507e-07, "loss": 0.0638, "num_tokens": 13034867.0, "reward": 0.74298095703125, "reward_std": 0.0036703976802527905, "rewards//mean": 0.74298095703125, "rewards//std": 0.023085355758666992, "step": 1508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3018, "grad_norm": 3.2699155807495117, "kl": 1.748958621174097, "learning_rate": 8.007807798895193e-07, "loss": 0.07, "num_tokens": 13043611.0, "reward": 0.74957275390625, "reward_std": 0.004528774879872799, "rewards//mean": 0.74957275390625, "rewards//std": 0.026402974501252174, "step": 1509 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.302, "grad_norm": 4.456327438354492, "kl": 0.8866036608815193, "learning_rate": 8.005272256634257e-07, "loss": 0.0355, "num_tokens": 13052251.0, "reward": 0.75701904296875, "reward_std": 0.004231567494571209, "rewards//mean": 0.75701904296875, "rewards//std": 0.027980120852589607, "step": 1510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3022, "grad_norm": 1.179195523262024, "kl": 1.5088980309665203, "learning_rate": 8.002735503850015e-07, "loss": 0.0604, "num_tokens": 13060931.0, "reward": 0.77215576171875, "reward_std": 0.002582917455583811, "rewards//mean": 0.77215576171875, "rewards//std": 0.024725429713726044, "step": 1511 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3024, "grad_norm": 5.7419586181640625, "kl": 2.186433929949999, "learning_rate": 8.000197541564271e-07, "loss": 0.0875, "num_tokens": 13069595.0, "reward": 0.7386474609375, "reward_std": 0.003107402939349413, "rewards//mean": 0.7386474609375, "rewards//std": 0.030700454488396645, "step": 1512 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3026, "grad_norm": 3.0364153385162354, "kl": 2.8216484375298023, "learning_rate": 7.997658370799316e-07, "loss": 0.1129, "num_tokens": 13078187.0, "reward": 0.7657470703125, "reward_std": 0.007853852584958076, "rewards//mean": 0.7657470703125, "rewards//std": 0.022173339501023293, "step": 1513 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3028, "grad_norm": 7.140042781829834, "kl": 0.7660270165652037, "learning_rate": 7.995117992577928e-07, "loss": 0.0306, "num_tokens": 13086763.0, "reward": 0.7581787109375, "reward_std": 0.004557499662041664, "rewards//mean": 0.7581787109375, "rewards//std": 0.019926784560084343, "step": 1514 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.303, "grad_norm": 1.8625723123550415, "kl": 0.9544672500342131, "learning_rate": 7.992576407923372e-07, "loss": 0.0382, "num_tokens": 13095403.0, "reward": 0.75994873046875, "reward_std": 0.002039259532466531, "rewards//mean": 0.75994873046875, "rewards//std": 0.024058910086750984, "step": 1515 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3032, "grad_norm": 3.971250295639038, "kl": 2.17875063046813, "learning_rate": 7.990033617859395e-07, "loss": 0.0872, "num_tokens": 13103963.0, "reward": 0.74371337890625, "reward_std": 0.0056187184527516365, "rewards//mean": 0.74371337890625, "rewards//std": 0.025071369484066963, "step": 1516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3034, "grad_norm": 8.928879737854004, "kl": 2.4854452945291996, "learning_rate": 7.987489623410235e-07, "loss": 0.0994, "num_tokens": 13112563.0, "reward": 0.74102783203125, "reward_std": 0.0031742537394165993, "rewards//mean": 0.74102783203125, "rewards//std": 0.02809673734009266, "step": 1517 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3036, "grad_norm": 1.7843027114868164, "kl": 1.4118167273700237, "learning_rate": 7.984944425600613e-07, "loss": 0.0565, "num_tokens": 13121155.0, "reward": 0.76629638671875, "reward_std": 0.005580378696322441, "rewards//mean": 0.76629638671875, "rewards//std": 0.025841444730758667, "step": 1518 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3038, "grad_norm": 4.163170337677002, "kl": 1.2572334352880716, "learning_rate": 7.982398025455732e-07, "loss": 0.0503, "num_tokens": 13129923.0, "reward": 0.78656005859375, "reward_std": 0.0058036986738443375, "rewards//mean": 0.78656005859375, "rewards//std": 0.02161354385316372, "step": 1519 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.304, "grad_norm": 1.428679347038269, "kl": 0.9906759764999151, "learning_rate": 7.979850424001282e-07, "loss": 0.0396, "num_tokens": 13138555.0, "reward": 0.76422119140625, "reward_std": 0.0012864710297435522, "rewards//mean": 0.76422119140625, "rewards//std": 0.017817514017224312, "step": 1520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3042, "grad_norm": 3.2875068187713623, "kl": 1.7363167777657509, "learning_rate": 7.97730162226344e-07, "loss": 0.0695, "num_tokens": 13147267.0, "reward": 0.75335693359375, "reward_std": 0.0042083170264959335, "rewards//mean": 0.75335693359375, "rewards//std": 0.021697426214814186, "step": 1521 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3044, "grad_norm": 9.200188636779785, "kl": 6.625841850414872, "learning_rate": 7.974751621268858e-07, "loss": 0.265, "num_tokens": 13155963.0, "reward": 0.76275634765625, "reward_std": 0.011108377017080784, "rewards//mean": 0.76275634765625, "rewards//std": 0.033315036445856094, "step": 1522 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3046, "grad_norm": 1.9781839847564697, "kl": 1.9428846444934607, "learning_rate": 7.972200422044682e-07, "loss": 0.0777, "num_tokens": 13164587.0, "reward": 0.75537109375, "reward_std": 0.005958822090178728, "rewards//mean": 0.75537109375, "rewards//std": 0.026852156966924667, "step": 1523 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3048, "grad_norm": 6.879197120666504, "kl": 2.205456007272005, "learning_rate": 7.969648025618529e-07, "loss": 0.0882, "num_tokens": 13173219.0, "reward": 0.7874755859375, "reward_std": 0.008457712829113007, "rewards//mean": 0.7874755859375, "rewards//std": 0.02719470113515854, "step": 1524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.305, "grad_norm": 6.487525939941406, "kl": 1.2132704816758633, "learning_rate": 7.967094433018508e-07, "loss": 0.0485, "num_tokens": 13181795.0, "reward": 0.765380859375, "reward_std": 0.0031874789856374264, "rewards//mean": 0.765380859375, "rewards//std": 0.015899665653705597, "step": 1525 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3052, "grad_norm": 3.5539145469665527, "kl": 3.9248960874974728, "learning_rate": 7.964539645273202e-07, "loss": 0.157, "num_tokens": 13190347.0, "reward": 0.7828369140625, "reward_std": 0.00914781354367733, "rewards//mean": 0.7828369140625, "rewards//std": 0.021572621539235115, "step": 1526 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3054, "grad_norm": 5.06986141204834, "kl": 4.15170344337821, "learning_rate": 7.961983663411684e-07, "loss": 0.1661, "num_tokens": 13198987.0, "reward": 0.7686767578125, "reward_std": 0.007378567010164261, "rewards//mean": 0.7686767578125, "rewards//std": 0.030524414032697678, "step": 1527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3056, "grad_norm": 1.2802878618240356, "kl": 1.1794881578534842, "learning_rate": 7.959426488463499e-07, "loss": 0.0472, "num_tokens": 13207627.0, "reward": 0.74786376953125, "reward_std": 0.0021249987185001373, "rewards//mean": 0.74786376953125, "rewards//std": 0.01725907437503338, "step": 1528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3058, "grad_norm": 8.874640464782715, "kl": 4.919972432777286, "learning_rate": 7.956868121458677e-07, "loss": 0.1968, "num_tokens": 13216195.0, "reward": 0.7379150390625, "reward_std": 0.007485592737793922, "rewards//mean": 0.7379150390625, "rewards//std": 0.025588683784008026, "step": 1529 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.306, "grad_norm": 12.111811637878418, "kl": 3.570584613829851, "learning_rate": 7.954308563427732e-07, "loss": 0.1428, "num_tokens": 13224811.0, "reward": 0.76019287109375, "reward_std": 0.006022367626428604, "rewards//mean": 0.76019287109375, "rewards//std": 0.0230584554374218, "step": 1530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3062, "grad_norm": 1.4922840595245361, "kl": 2.025520922616124, "learning_rate": 7.951747815401649e-07, "loss": 0.081, "num_tokens": 13233507.0, "reward": 0.7764892578125, "reward_std": 0.0044884709641337395, "rewards//mean": 0.7764892578125, "rewards//std": 0.031004654243588448, "step": 1531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3064, "grad_norm": 1.084610939025879, "kl": 1.5145159009844065, "learning_rate": 7.949185878411899e-07, "loss": 0.0606, "num_tokens": 13242139.0, "reward": 0.80169677734375, "reward_std": 0.004900915548205376, "rewards//mean": 0.80169677734375, "rewards//std": 0.024868888780474663, "step": 1532 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3066, "grad_norm": 4.220397472381592, "kl": 4.190284017473459, "learning_rate": 7.946622753490432e-07, "loss": 0.1676, "num_tokens": 13250763.0, "reward": 0.7738037109375, "reward_std": 0.011110790073871613, "rewards//mean": 0.7738037109375, "rewards//std": 0.03179338201880455, "step": 1533 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3068, "grad_norm": 3.8574328422546387, "kl": 3.023826414719224, "learning_rate": 7.94405844166967e-07, "loss": 0.121, "num_tokens": 13259403.0, "reward": 0.7447509765625, "reward_std": 0.006219988223165274, "rewards//mean": 0.7447509765625, "rewards//std": 0.03421863913536072, "step": 1534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.307, "grad_norm": 7.903994083404541, "kl": 4.962822580710053, "learning_rate": 7.941492943982521e-07, "loss": 0.1985, "num_tokens": 13268131.0, "reward": 0.74761962890625, "reward_std": 0.009986553341150284, "rewards//mean": 0.74761962890625, "rewards//std": 0.032731037586927414, "step": 1535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3072, "grad_norm": 12.15711498260498, "kl": 4.283036205917597, "learning_rate": 7.938926261462365e-07, "loss": 0.1713, "num_tokens": 13276771.0, "reward": 0.747314453125, "reward_std": 0.00761657627299428, "rewards//mean": 0.747314453125, "rewards//std": 0.02559962496161461, "step": 1536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3074, "grad_norm": 1.8574138879776, "kl": 2.006930122151971, "learning_rate": 7.936358395143063e-07, "loss": 0.0803, "num_tokens": 13285411.0, "reward": 0.7672119140625, "reward_std": 0.005736600141972303, "rewards//mean": 0.7672119140625, "rewards//std": 0.01953396387398243, "step": 1537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3076, "grad_norm": 9.763838768005371, "kl": 6.778828650712967, "learning_rate": 7.93378934605895e-07, "loss": 0.2712, "num_tokens": 13294099.0, "reward": 0.76153564453125, "reward_std": 0.012372822500765324, "rewards//mean": 0.76153564453125, "rewards//std": 0.023760074749588966, "step": 1538 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3078, "grad_norm": 2.0672945976257324, "kl": 1.9476182386279106, "learning_rate": 7.93121911524484e-07, "loss": 0.0779, "num_tokens": 13302675.0, "reward": 0.7374267578125, "reward_std": 0.003018442541360855, "rewards//mean": 0.7374267578125, "rewards//std": 0.028429146856069565, "step": 1539 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.308, "grad_norm": 2.7123074531555176, "kl": 2.072697926312685, "learning_rate": 7.928647703736023e-07, "loss": 0.0829, "num_tokens": 13311403.0, "reward": 0.7547607421875, "reward_std": 0.004106986802071333, "rewards//mean": 0.7547607421875, "rewards//std": 0.02630511112511158, "step": 1540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3082, "grad_norm": 1.635036587715149, "kl": 0.9788522999733686, "learning_rate": 7.926075112568258e-07, "loss": 0.0392, "num_tokens": 13319987.0, "reward": 0.75177001953125, "reward_std": 0.0025895023718476295, "rewards//mean": 0.75177001953125, "rewards//std": 0.02867904119193554, "step": 1541 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3084, "grad_norm": 2.1284656524658203, "kl": 1.0090431105345488, "learning_rate": 7.923501342777787e-07, "loss": 0.0404, "num_tokens": 13328547.0, "reward": 0.73284912109375, "reward_std": 0.002874338533729315, "rewards//mean": 0.73284912109375, "rewards//std": 0.035764940083026886, "step": 1542 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3086, "grad_norm": 2.770582914352417, "kl": 3.5458790939301252, "learning_rate": 7.920926395401326e-07, "loss": 0.1418, "num_tokens": 13337123.0, "reward": 0.7674560546875, "reward_std": 0.007663349155336618, "rewards//mean": 0.7674560546875, "rewards//std": 0.0270092636346817, "step": 1543 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3088, "grad_norm": 12.36461067199707, "kl": 6.357189066708088, "learning_rate": 7.918350271476063e-07, "loss": 0.2543, "num_tokens": 13345795.0, "reward": 0.77752685546875, "reward_std": 0.010957293212413788, "rewards//mean": 0.77752685546875, "rewards//std": 0.027272172272205353, "step": 1544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.309, "grad_norm": 2.1590182781219482, "kl": 2.6033093258738518, "learning_rate": 7.915772972039659e-07, "loss": 0.1041, "num_tokens": 13354387.0, "reward": 0.754150390625, "reward_std": 0.006933914497494698, "rewards//mean": 0.754150390625, "rewards//std": 0.025428753346204758, "step": 1545 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3092, "grad_norm": 2.306547164916992, "kl": 3.196092039346695, "learning_rate": 7.913194498130251e-07, "loss": 0.1278, "num_tokens": 13363075.0, "reward": 0.7781982421875, "reward_std": 0.007881474681198597, "rewards//mean": 0.7781982421875, "rewards//std": 0.030060676857829094, "step": 1546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3094, "grad_norm": 6.8349289894104, "kl": 1.675778154283762, "learning_rate": 7.910614850786447e-07, "loss": 0.067, "num_tokens": 13371691.0, "reward": 0.7474365234375, "reward_std": 0.0034942326601594687, "rewards//mean": 0.7474365234375, "rewards//std": 0.018390795215964317, "step": 1547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3096, "grad_norm": 1.6075439453125, "kl": 1.2292317170649767, "learning_rate": 7.90803403104733e-07, "loss": 0.0492, "num_tokens": 13380339.0, "reward": 0.76837158203125, "reward_std": 0.0036253032740205526, "rewards//mean": 0.76837158203125, "rewards//std": 0.022731170058250427, "step": 1548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3098, "grad_norm": 2.0693655014038086, "kl": 2.8236304074525833, "learning_rate": 7.905452039952451e-07, "loss": 0.1129, "num_tokens": 13388947.0, "reward": 0.79052734375, "reward_std": 0.008789876475930214, "rewards//mean": 0.79052734375, "rewards//std": 0.028977885842323303, "step": 1549 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.31, "grad_norm": 1.548401951789856, "kl": 2.3623798824846745, "learning_rate": 7.90286887854184e-07, "loss": 0.0945, "num_tokens": 13397571.0, "reward": 0.75714111328125, "reward_std": 0.0065483031794428825, "rewards//mean": 0.75714111328125, "rewards//std": 0.025128664448857307, "step": 1550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3102, "grad_norm": 3.1247942447662354, "kl": 2.1018613893538713, "learning_rate": 7.900284547855991e-07, "loss": 0.0841, "num_tokens": 13406251.0, "reward": 0.72906494140625, "reward_std": 0.002748192986473441, "rewards//mean": 0.72906494140625, "rewards//std": 0.03136887773871422, "step": 1551 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.546875, "epoch": 0.3104, "grad_norm": 3.9941465854644775, "kl": 4.231639681383967, "learning_rate": 7.897699048935873e-07, "loss": 0.1661, "num_tokens": 13414870.0, "reward": 0.76788330078125, "reward_std": 0.014554089866578579, "rewards//mean": 0.76788330078125, "rewards//std": 0.03652341291308403, "step": 1552 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3106, "grad_norm": 9.030753135681152, "kl": 3.4044176265597343, "learning_rate": 7.895112382822924e-07, "loss": 0.1362, "num_tokens": 13423446.0, "reward": 0.7880859375, "reward_std": 0.016512135043740273, "rewards//mean": 0.7880859375, "rewards//std": 0.03450608626008034, "step": 1553 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3108, "grad_norm": 1.22676682472229, "kl": 1.1323249340057373, "learning_rate": 7.892524550559055e-07, "loss": 0.0453, "num_tokens": 13432078.0, "reward": 0.7481689453125, "reward_std": 0.002001028275117278, "rewards//mean": 0.7481689453125, "rewards//std": 0.017538035288453102, "step": 1554 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.311, "grad_norm": 0.7332450747489929, "kl": 1.2387255914509296, "learning_rate": 7.889935553186641e-07, "loss": 0.0495, "num_tokens": 13440750.0, "reward": 0.7340087890625, "reward_std": 0.0024168689269572496, "rewards//mean": 0.7340087890625, "rewards//std": 0.039553701877593994, "step": 1555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3112, "grad_norm": 4.305971145629883, "kl": 5.2117039784789085, "learning_rate": 7.887345391748532e-07, "loss": 0.2085, "num_tokens": 13449462.0, "reward": 0.7496337890625, "reward_std": 0.012276681140065193, "rewards//mean": 0.7496337890625, "rewards//std": 0.0312478207051754, "step": 1556 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3114, "grad_norm": 0.24466800689697266, "kl": 0.5341769643127918, "learning_rate": 7.884754067288046e-07, "loss": 0.0214, "num_tokens": 13457958.0, "reward": 0.7513427734375, "reward_std": 0.0004253430524840951, "rewards//mean": 0.7513427734375, "rewards//std": 0.019626734778285027, "step": 1557 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3116, "grad_norm": 3.6603965759277344, "kl": 2.2707056906074286, "learning_rate": 7.882161580848966e-07, "loss": 0.0908, "num_tokens": 13466590.0, "reward": 0.77972412109375, "reward_std": 0.003847534768283367, "rewards//mean": 0.77972412109375, "rewards//std": 0.029549933969974518, "step": 1558 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3118, "grad_norm": 10.382216453552246, "kl": 5.203917965292931, "learning_rate": 7.879567933475546e-07, "loss": 0.2082, "num_tokens": 13475222.0, "reward": 0.7879638671875, "reward_std": 0.0056140488013625145, "rewards//mean": 0.7879638671875, "rewards//std": 0.02510855160653591, "step": 1559 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.312, "grad_norm": 6.544485092163086, "kl": 1.3014779221266508, "learning_rate": 7.876973126212506e-07, "loss": 0.0521, "num_tokens": 13483806.0, "reward": 0.77545166015625, "reward_std": 0.005389615427702665, "rewards//mean": 0.77545166015625, "rewards//std": 0.02592974714934826, "step": 1560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3122, "grad_norm": 2.880796194076538, "kl": 3.00633068010211, "learning_rate": 7.874377160105036e-07, "loss": 0.1203, "num_tokens": 13492502.0, "reward": 0.73931884765625, "reward_std": 0.007274072151631117, "rewards//mean": 0.73931884765625, "rewards//std": 0.0314142070710659, "step": 1561 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3124, "grad_norm": 10.305108070373535, "kl": 3.9166896883398294, "learning_rate": 7.871780036198788e-07, "loss": 0.1567, "num_tokens": 13501158.0, "reward": 0.7200927734375, "reward_std": 0.005273417569696903, "rewards//mean": 0.7200927734375, "rewards//std": 0.0368233397603035, "step": 1562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3126, "grad_norm": 2.5040998458862305, "kl": 4.020249454304576, "learning_rate": 7.869181755539887e-07, "loss": 0.1608, "num_tokens": 13509718.0, "reward": 0.78753662109375, "reward_std": 0.014026065357029438, "rewards//mean": 0.78753662109375, "rewards//std": 0.0272560715675354, "step": 1563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3128, "grad_norm": 4.298035621643066, "kl": 3.5542656742036343, "learning_rate": 7.866582319174917e-07, "loss": 0.1422, "num_tokens": 13518430.0, "reward": 0.7562255859375, "reward_std": 0.006951578892767429, "rewards//mean": 0.7562255859375, "rewards//std": 0.031263317912817, "step": 1564 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.313, "grad_norm": 4.246382236480713, "kl": 2.2568327765911818, "learning_rate": 7.863981728150931e-07, "loss": 0.0903, "num_tokens": 13527118.0, "reward": 0.77130126953125, "reward_std": 0.007427331525832415, "rewards//mean": 0.77130126953125, "rewards//std": 0.02944575995206833, "step": 1565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.046875, "epoch": 0.3132, "grad_norm": 12.954119682312012, "kl": 2.3872750215232372, "learning_rate": 7.861379983515448e-07, "loss": 0.0804, "num_tokens": 13535865.0, "reward": 0.75164794921875, "reward_std": 0.007791175507009029, "rewards//mean": 0.75164794921875, "rewards//std": 0.024258168414235115, "step": 1566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3134, "grad_norm": 7.969794273376465, "kl": 3.3750481456518173, "learning_rate": 7.858777086316451e-07, "loss": 0.135, "num_tokens": 13544649.0, "reward": 0.760498046875, "reward_std": 0.006085801403969526, "rewards//mean": 0.760498046875, "rewards//std": 0.023927126079797745, "step": 1567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3136, "grad_norm": 13.998820304870605, "kl": 3.3849109187722206, "learning_rate": 7.856173037602382e-07, "loss": 0.1354, "num_tokens": 13553329.0, "reward": 0.74395751953125, "reward_std": 0.008572198450565338, "rewards//mean": 0.74395751953125, "rewards//std": 0.031086266040802002, "step": 1568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3138, "grad_norm": 6.771536350250244, "kl": 2.87066737562418, "learning_rate": 7.853567838422159e-07, "loss": 0.1148, "num_tokens": 13561985.0, "reward": 0.7691650390625, "reward_std": 0.00547350337728858, "rewards//mean": 0.7691650390625, "rewards//std": 0.02769770473241806, "step": 1569 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.314, "grad_norm": 12.953600883483887, "kl": 5.080168567597866, "learning_rate": 7.850961489825149e-07, "loss": 0.2032, "num_tokens": 13570689.0, "reward": 0.71453857421875, "reward_std": 0.007240781560540199, "rewards//mean": 0.71453857421875, "rewards//std": 0.02928389050066471, "step": 1570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3142, "grad_norm": 14.087126731872559, "kl": 5.163237893953919, "learning_rate": 7.848353992861194e-07, "loss": 0.2065, "num_tokens": 13579393.0, "reward": 0.7786865234375, "reward_std": 0.006209178827702999, "rewards//mean": 0.7786865234375, "rewards//std": 0.02369667775928974, "step": 1571 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3144, "grad_norm": 2.7185895442962646, "kl": 1.6526428014039993, "learning_rate": 7.84574534858059e-07, "loss": 0.0661, "num_tokens": 13587945.0, "reward": 0.79345703125, "reward_std": 0.005284490995109081, "rewards//mean": 0.79345703125, "rewards//std": 0.01581757515668869, "step": 1572 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3146, "grad_norm": 6.753876209259033, "kl": 1.8430143650621176, "learning_rate": 7.8431355580341e-07, "loss": 0.0737, "num_tokens": 13596641.0, "reward": 0.7794189453125, "reward_std": 0.005295715294778347, "rewards//mean": 0.7794189453125, "rewards//std": 0.022977977991104126, "step": 1573 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3148, "grad_norm": 7.430104732513428, "kl": 3.0881308056414127, "learning_rate": 7.840524622272948e-07, "loss": 0.1235, "num_tokens": 13605297.0, "reward": 0.75640869140625, "reward_std": 0.01122256275266409, "rewards//mean": 0.75640869140625, "rewards//std": 0.02808542177081108, "step": 1574 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.315, "grad_norm": 3.793759822845459, "kl": 2.775339739397168, "learning_rate": 7.837912542348817e-07, "loss": 0.111, "num_tokens": 13613921.0, "reward": 0.802978515625, "reward_std": 0.008220259100198746, "rewards//mean": 0.802978515625, "rewards//std": 0.029139375314116478, "step": 1575 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3152, "grad_norm": 7.971090316772461, "kl": 3.0225213952362537, "learning_rate": 7.835299319313853e-07, "loss": 0.1209, "num_tokens": 13622457.0, "reward": 0.78424072265625, "reward_std": 0.008467039093375206, "rewards//mean": 0.78424072265625, "rewards//std": 0.026709645986557007, "step": 1576 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3154, "grad_norm": 7.319133281707764, "kl": 4.196964494884014, "learning_rate": 7.832684954220663e-07, "loss": 0.1679, "num_tokens": 13631257.0, "reward": 0.764892578125, "reward_std": 0.008041007444262505, "rewards//mean": 0.764892578125, "rewards//std": 0.022454533725976944, "step": 1577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3156, "grad_norm": 6.501193046569824, "kl": 0.7581717055290937, "learning_rate": 7.830069448122312e-07, "loss": 0.0303, "num_tokens": 13639849.0, "reward": 0.78497314453125, "reward_std": 0.002913340460509062, "rewards//mean": 0.78497314453125, "rewards//std": 0.01798073947429657, "step": 1578 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3158, "grad_norm": 11.395877838134766, "kl": 4.177056362852454, "learning_rate": 7.827452802072327e-07, "loss": 0.1671, "num_tokens": 13648569.0, "reward": 0.74993896484375, "reward_std": 0.008374565280973911, "rewards//mean": 0.74993896484375, "rewards//std": 0.027432115748524666, "step": 1579 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.316, "grad_norm": 6.650692462921143, "kl": 3.6675681360065937, "learning_rate": 7.82483501712469e-07, "loss": 0.1467, "num_tokens": 13657185.0, "reward": 0.72760009765625, "reward_std": 0.007356700487434864, "rewards//mean": 0.72760009765625, "rewards//std": 0.03624005988240242, "step": 1580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3162, "grad_norm": 16.119176864624023, "kl": 2.5368894282728434, "learning_rate": 7.822216094333847e-07, "loss": 0.1015, "num_tokens": 13665769.0, "reward": 0.7923583984375, "reward_std": 0.008061882108449936, "rewards//mean": 0.7923583984375, "rewards//std": 0.024815035983920097, "step": 1581 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3164, "grad_norm": 8.031620025634766, "kl": 4.719241995364428, "learning_rate": 7.819596034754696e-07, "loss": 0.1888, "num_tokens": 13674369.0, "reward": 0.76568603515625, "reward_std": 0.012494444847106934, "rewards//mean": 0.76568603515625, "rewards//std": 0.028172602877020836, "step": 1582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3166, "grad_norm": 12.50458812713623, "kl": 5.18662153929472, "learning_rate": 7.816974839442603e-07, "loss": 0.2075, "num_tokens": 13683049.0, "reward": 0.75146484375, "reward_std": 0.01645568758249283, "rewards//mean": 0.75146484375, "rewards//std": 0.03689746931195259, "step": 1583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3168, "grad_norm": 4.855544090270996, "kl": 3.400411695241928, "learning_rate": 7.814352509453379e-07, "loss": 0.136, "num_tokens": 13691641.0, "reward": 0.76513671875, "reward_std": 0.009719028137624264, "rewards//mean": 0.76513671875, "rewards//std": 0.028301332145929337, "step": 1584 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.317, "grad_norm": 10.149500846862793, "kl": 2.5843121875077486, "learning_rate": 7.811729045843301e-07, "loss": 0.1034, "num_tokens": 13700241.0, "reward": 0.7584228515625, "reward_std": 0.006244299001991749, "rewards//mean": 0.7584228515625, "rewards//std": 0.029718313366174698, "step": 1585 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3172, "grad_norm": 7.332499980926514, "kl": 2.5256293322890997, "learning_rate": 7.8091044496691e-07, "loss": 0.101, "num_tokens": 13708873.0, "reward": 0.74774169921875, "reward_std": 0.008119644597172737, "rewards//mean": 0.74774169921875, "rewards//std": 0.03190051019191742, "step": 1586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3174, "grad_norm": 8.883455276489258, "kl": 2.6917338632047176, "learning_rate": 7.806478721987963e-07, "loss": 0.1077, "num_tokens": 13717625.0, "reward": 0.7562255859375, "reward_std": 0.006581498775631189, "rewards//mean": 0.7562255859375, "rewards//std": 0.023691566661000252, "step": 1587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3176, "grad_norm": 10.507652282714844, "kl": 2.3051068540662527, "learning_rate": 7.803851863857532e-07, "loss": 0.0922, "num_tokens": 13726369.0, "reward": 0.75238037109375, "reward_std": 0.006877091713249683, "rewards//mean": 0.75238037109375, "rewards//std": 0.02831944450736046, "step": 1588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3178, "grad_norm": 6.7308173179626465, "kl": 1.5977623760700226, "learning_rate": 7.801223876335907e-07, "loss": 0.0639, "num_tokens": 13735057.0, "reward": 0.758056640625, "reward_std": 0.007009593769907951, "rewards//mean": 0.758056640625, "rewards//std": 0.023259857669472694, "step": 1589 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.318, "grad_norm": 15.043256759643555, "kl": 4.218444623053074, "learning_rate": 7.798594760481637e-07, "loss": 0.1687, "num_tokens": 13743641.0, "reward": 0.7542724609375, "reward_std": 0.010292530059814453, "rewards//mean": 0.7542724609375, "rewards//std": 0.02739877812564373, "step": 1590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3182, "grad_norm": 25.150230407714844, "kl": 6.923338778316975, "learning_rate": 7.795964517353733e-07, "loss": 0.2769, "num_tokens": 13752361.0, "reward": 0.74615478515625, "reward_std": 0.009582172147929668, "rewards//mean": 0.74615478515625, "rewards//std": 0.024144954979419708, "step": 1591 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3184, "grad_norm": 35.85035705566406, "kl": 5.761877480894327, "learning_rate": 7.793333148011657e-07, "loss": 0.2305, "num_tokens": 13760945.0, "reward": 0.74853515625, "reward_std": 0.007426196709275246, "rewards//mean": 0.74853515625, "rewards//std": 0.030675049871206284, "step": 1592 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3186, "grad_norm": 7.998895645141602, "kl": 4.327491544187069, "learning_rate": 7.790700653515323e-07, "loss": 0.1731, "num_tokens": 13769489.0, "reward": 0.71014404296875, "reward_std": 0.013705659657716751, "rewards//mean": 0.71014404296875, "rewards//std": 0.037732694298028946, "step": 1593 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3188, "grad_norm": 18.91761016845703, "kl": 4.072094418108463, "learning_rate": 7.788067034925099e-07, "loss": 0.1629, "num_tokens": 13778065.0, "reward": 0.7537841796875, "reward_std": 0.01519739255309105, "rewards//mean": 0.7537841796875, "rewards//std": 0.028763698413968086, "step": 1594 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.319, "grad_norm": 14.40042781829834, "kl": 4.263888690620661, "learning_rate": 7.785432293301806e-07, "loss": 0.1706, "num_tokens": 13786681.0, "reward": 0.728759765625, "reward_std": 0.008491072803735733, "rewards//mean": 0.728759765625, "rewards//std": 0.0288217943161726, "step": 1595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3192, "grad_norm": 5.8197808265686035, "kl": 3.3678077179938555, "learning_rate": 7.78279642970672e-07, "loss": 0.1347, "num_tokens": 13795241.0, "reward": 0.7589111328125, "reward_std": 0.011701389215886593, "rewards//mean": 0.7589111328125, "rewards//std": 0.03187517076730728, "step": 1596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3194, "grad_norm": 24.04300308227539, "kl": 2.8448747098445892, "learning_rate": 7.780159445201562e-07, "loss": 0.1138, "num_tokens": 13803929.0, "reward": 0.80096435546875, "reward_std": 0.01415582187473774, "rewards//mean": 0.80096435546875, "rewards//std": 0.026842502877116203, "step": 1597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3196, "grad_norm": 6.142871856689453, "kl": 2.120186196640134, "learning_rate": 7.777521340848514e-07, "loss": 0.0848, "num_tokens": 13812577.0, "reward": 0.71734619140625, "reward_std": 0.008194508031010628, "rewards//mean": 0.71734619140625, "rewards//std": 0.032692164182662964, "step": 1598 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3198, "grad_norm": 3.9844765663146973, "kl": 1.9246549867093563, "learning_rate": 7.774882117710202e-07, "loss": 0.077, "num_tokens": 13821161.0, "reward": 0.749267578125, "reward_std": 0.005446576979011297, "rewards//mean": 0.749267578125, "rewards//std": 0.024928545579314232, "step": 1599 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.32, "grad_norm": 7.272011756896973, "kl": 2.294439125806093, "learning_rate": 7.772241776849704e-07, "loss": 0.0918, "num_tokens": 13829705.0, "reward": 0.7069091796875, "reward_std": 0.007744522299617529, "rewards//mean": 0.7069091796875, "rewards//std": 0.02167622372508049, "step": 1600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3202, "grad_norm": 7.847100257873535, "kl": 3.460626281797886, "learning_rate": 7.769600319330552e-07, "loss": 0.1384, "num_tokens": 13838377.0, "reward": 0.76690673828125, "reward_std": 0.012261524796485901, "rewards//mean": 0.76690673828125, "rewards//std": 0.03251154348254204, "step": 1601 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3204, "grad_norm": 13.213078498840332, "kl": 3.530375551432371, "learning_rate": 7.76695774621672e-07, "loss": 0.1412, "num_tokens": 13847049.0, "reward": 0.75537109375, "reward_std": 0.009636061266064644, "rewards//mean": 0.75537109375, "rewards//std": 0.03106733039021492, "step": 1602 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3206, "grad_norm": 7.9046478271484375, "kl": 2.3466283716261387, "learning_rate": 7.764314058572639e-07, "loss": 0.0939, "num_tokens": 13855689.0, "reward": 0.75860595703125, "reward_std": 0.00639187078922987, "rewards//mean": 0.75860595703125, "rewards//std": 0.016879508271813393, "step": 1603 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3208, "grad_norm": 11.421862602233887, "kl": 3.2277444936335087, "learning_rate": 7.761669257463187e-07, "loss": 0.1291, "num_tokens": 13864481.0, "reward": 0.7777099609375, "reward_std": 0.012739087454974651, "rewards//mean": 0.7777099609375, "rewards//std": 0.01985372230410576, "step": 1604 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.321, "grad_norm": 22.2473201751709, "kl": 3.5649806205183268, "learning_rate": 7.759023343953688e-07, "loss": 0.1426, "num_tokens": 13873025.0, "reward": 0.75970458984375, "reward_std": 0.013574929907917976, "rewards//mean": 0.75970458984375, "rewards//std": 0.030040716752409935, "step": 1605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3212, "grad_norm": 25.993104934692383, "kl": 7.29346152022481, "learning_rate": 7.756376319109916e-07, "loss": 0.2917, "num_tokens": 13881697.0, "reward": 0.73590087890625, "reward_std": 0.011688966304063797, "rewards//mean": 0.73590087890625, "rewards//std": 0.026389211416244507, "step": 1606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3214, "grad_norm": 18.03403663635254, "kl": 5.4808325953781605, "learning_rate": 7.753728183998092e-07, "loss": 0.2192, "num_tokens": 13890337.0, "reward": 0.75201416015625, "reward_std": 0.013721934519708157, "rewards//mean": 0.75201416015625, "rewards//std": 0.030804993584752083, "step": 1607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3216, "grad_norm": 31.654752731323242, "kl": 6.4295050743967295, "learning_rate": 7.751078939684885e-07, "loss": 0.2572, "num_tokens": 13898905.0, "reward": 0.75146484375, "reward_std": 0.011243881657719612, "rewards//mean": 0.75146484375, "rewards//std": 0.02818985842168331, "step": 1608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3218, "grad_norm": 59.448848724365234, "kl": 13.091281961649656, "learning_rate": 7.748428587237411e-07, "loss": 0.5237, "num_tokens": 13907505.0, "reward": 0.73382568359375, "reward_std": 0.01739353872835636, "rewards//mean": 0.73382568359375, "rewards//std": 0.030120227485895157, "step": 1609 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.322, "grad_norm": 41.87702178955078, "kl": 7.90415133535862, "learning_rate": 7.74577712772323e-07, "loss": 0.3162, "num_tokens": 13916113.0, "reward": 0.7431640625, "reward_std": 0.01467541791498661, "rewards//mean": 0.7431640625, "rewards//std": 0.0174969881772995, "step": 1610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3222, "grad_norm": 18.246580123901367, "kl": 5.010819291695952, "learning_rate": 7.743124562210351e-07, "loss": 0.2004, "num_tokens": 13924729.0, "reward": 0.713134765625, "reward_std": 0.013421990908682346, "rewards//mean": 0.713134765625, "rewards//std": 0.026216628029942513, "step": 1611 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3224, "grad_norm": 11.278694152832031, "kl": 3.67126352712512, "learning_rate": 7.740470891767224e-07, "loss": 0.1469, "num_tokens": 13933313.0, "reward": 0.7525634765625, "reward_std": 0.006975987926125526, "rewards//mean": 0.7525634765625, "rewards//std": 0.02472703717648983, "step": 1612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3226, "grad_norm": 21.367422103881836, "kl": 4.67998156696558, "learning_rate": 7.737816117462751e-07, "loss": 0.1872, "num_tokens": 13941849.0, "reward": 0.74945068359375, "reward_std": 0.007777191698551178, "rewards//mean": 0.74945068359375, "rewards//std": 0.030590981245040894, "step": 1613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3228, "grad_norm": 14.104653358459473, "kl": 4.814919566735625, "learning_rate": 7.735160240366274e-07, "loss": 0.1926, "num_tokens": 13950593.0, "reward": 0.7359619140625, "reward_std": 0.007620183285325766, "rewards//mean": 0.7359619140625, "rewards//std": 0.027802443131804466, "step": 1614 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.323, "grad_norm": 5.918192386627197, "kl": 3.786400757730007, "learning_rate": 7.732503261547578e-07, "loss": 0.1515, "num_tokens": 13959273.0, "reward": 0.77899169921875, "reward_std": 0.009284375235438347, "rewards//mean": 0.77899169921875, "rewards//std": 0.022859998047351837, "step": 1615 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3232, "grad_norm": 29.60202407836914, "kl": 2.178092934191227, "learning_rate": 7.729845182076895e-07, "loss": 0.0871, "num_tokens": 13967897.0, "reward": 0.79571533203125, "reward_std": 0.017498420551419258, "rewards//mean": 0.79571533203125, "rewards//std": 0.02256004326045513, "step": 1616 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3234, "grad_norm": 15.12800121307373, "kl": 2.791096108034253, "learning_rate": 7.7271860030249e-07, "loss": 0.1116, "num_tokens": 13976521.0, "reward": 0.7176513671875, "reward_std": 0.015062645077705383, "rewards//mean": 0.7176513671875, "rewards//std": 0.038934990763664246, "step": 1617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3236, "grad_norm": 8.760200500488281, "kl": 3.269177110865712, "learning_rate": 7.72452572546271e-07, "loss": 0.1308, "num_tokens": 13985233.0, "reward": 0.7557373046875, "reward_std": 0.009893681854009628, "rewards//mean": 0.7557373046875, "rewards//std": 0.03662383556365967, "step": 1618 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3238, "grad_norm": 7.397759914398193, "kl": 3.589187180623412, "learning_rate": 7.721864350461882e-07, "loss": 0.1436, "num_tokens": 13993897.0, "reward": 0.7772216796875, "reward_std": 0.018703140318393707, "rewards//mean": 0.7772216796875, "rewards//std": 0.03549542278051376, "step": 1619 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.324, "grad_norm": 8.775015830993652, "kl": 1.8012400344014168, "learning_rate": 7.71920187909442e-07, "loss": 0.072, "num_tokens": 14002497.0, "reward": 0.75531005859375, "reward_std": 0.006133618764579296, "rewards//mean": 0.75531005859375, "rewards//std": 0.029358750209212303, "step": 1620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3242, "grad_norm": 11.978819847106934, "kl": 5.873415801674128, "learning_rate": 7.716538312432765e-07, "loss": 0.2349, "num_tokens": 14011017.0, "reward": 0.73675537109375, "reward_std": 0.019236888736486435, "rewards//mean": 0.73675537109375, "rewards//std": 0.030715428292751312, "step": 1621 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3244, "grad_norm": 6.883742332458496, "kl": 3.0566181652247906, "learning_rate": 7.713873651549804e-07, "loss": 0.1223, "num_tokens": 14019617.0, "reward": 0.7467041015625, "reward_std": 0.009721335954964161, "rewards//mean": 0.7467041015625, "rewards//std": 0.027234749868512154, "step": 1622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3246, "grad_norm": 8.357880592346191, "kl": 1.9255514815449715, "learning_rate": 7.71120789751886e-07, "loss": 0.077, "num_tokens": 14028185.0, "reward": 0.75762939453125, "reward_std": 0.01105282362550497, "rewards//mean": 0.75762939453125, "rewards//std": 0.027510913088917732, "step": 1623 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3248, "grad_norm": 7.992527484893799, "kl": 2.4378102142363787, "learning_rate": 7.7085410514137e-07, "loss": 0.0975, "num_tokens": 14036937.0, "reward": 0.750244140625, "reward_std": 0.009366312995553017, "rewards//mean": 0.750244140625, "rewards//std": 0.03406723216176033, "step": 1624 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.325, "grad_norm": 12.08099365234375, "kl": 2.1106253564357758, "learning_rate": 7.705873114308527e-07, "loss": 0.0844, "num_tokens": 14045609.0, "reward": 0.761962890625, "reward_std": 0.00817366223782301, "rewards//mean": 0.761962890625, "rewards//std": 0.01795978471636772, "step": 1625 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3252, "grad_norm": 9.843433380126953, "kl": 2.6523684319108725, "learning_rate": 7.703204087277988e-07, "loss": 0.1061, "num_tokens": 14054305.0, "reward": 0.75311279296875, "reward_std": 0.011103453114628792, "rewards//mean": 0.75311279296875, "rewards//std": 0.022110015153884888, "step": 1626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3254, "grad_norm": 10.099394798278809, "kl": 1.7653774097561836, "learning_rate": 7.700533971397165e-07, "loss": 0.0706, "num_tokens": 14062945.0, "reward": 0.73114013671875, "reward_std": 0.006639300845563412, "rewards//mean": 0.73114013671875, "rewards//std": 0.030762212350964546, "step": 1627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3256, "grad_norm": 6.093928813934326, "kl": 1.878224153071642, "learning_rate": 7.697862767741583e-07, "loss": 0.0751, "num_tokens": 14071561.0, "reward": 0.7745361328125, "reward_std": 0.007871415466070175, "rewards//mean": 0.7745361328125, "rewards//std": 0.026212874799966812, "step": 1628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3258, "grad_norm": 5.065017223358154, "kl": 1.2661594208329916, "learning_rate": 7.695190477387199e-07, "loss": 0.0506, "num_tokens": 14080193.0, "reward": 0.7513427734375, "reward_std": 0.004636221565306187, "rewards//mean": 0.7513427734375, "rewards//std": 0.020484179258346558, "step": 1629 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.326, "grad_norm": 27.482603073120117, "kl": 3.3264848925173283, "learning_rate": 7.692517101410414e-07, "loss": 0.1331, "num_tokens": 14088921.0, "reward": 0.77496337890625, "reward_std": 0.015468500554561615, "rewards//mean": 0.77496337890625, "rewards//std": 0.027220504358410835, "step": 1630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3262, "grad_norm": 10.311243057250977, "kl": 3.5869624000042677, "learning_rate": 7.689842640888063e-07, "loss": 0.1435, "num_tokens": 14097505.0, "reward": 0.7471923828125, "reward_std": 0.009189065545797348, "rewards//mean": 0.7471923828125, "rewards//std": 0.02355572022497654, "step": 1631 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3264, "grad_norm": 9.306674003601074, "kl": 3.5584700033068657, "learning_rate": 7.687167096897418e-07, "loss": 0.1423, "num_tokens": 14106249.0, "reward": 0.7293701171875, "reward_std": 0.01151098869740963, "rewards//mean": 0.7293701171875, "rewards//std": 0.02178211510181427, "step": 1632 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3266, "grad_norm": 13.561746597290039, "kl": 2.1063948441296816, "learning_rate": 7.684490470516185e-07, "loss": 0.0843, "num_tokens": 14114817.0, "reward": 0.76678466796875, "reward_std": 0.004899102263152599, "rewards//mean": 0.76678466796875, "rewards//std": 0.021739942952990532, "step": 1633 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3268, "grad_norm": 7.863563537597656, "kl": 4.624258419498801, "learning_rate": 7.681812762822515e-07, "loss": 0.185, "num_tokens": 14123481.0, "reward": 0.75518798828125, "reward_std": 0.012642089277505875, "rewards//mean": 0.75518798828125, "rewards//std": 0.03485565632581711, "step": 1634 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.327, "grad_norm": 22.702239990234375, "kl": 2.840575873851776, "learning_rate": 7.679133974894982e-07, "loss": 0.1136, "num_tokens": 14132065.0, "reward": 0.75201416015625, "reward_std": 0.00959020759910345, "rewards//mean": 0.75201416015625, "rewards//std": 0.028237005695700645, "step": 1635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3272, "grad_norm": 5.975203990936279, "kl": 2.8444022815674543, "learning_rate": 7.676454107812607e-07, "loss": 0.1138, "num_tokens": 14140681.0, "reward": 0.78369140625, "reward_std": 0.009267503395676613, "rewards//mean": 0.78369140625, "rewards//std": 0.02929377555847168, "step": 1636 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3274, "grad_norm": 14.63658332824707, "kl": 7.024058857932687, "learning_rate": 7.673773162654836e-07, "loss": 0.281, "num_tokens": 14149393.0, "reward": 0.749755859375, "reward_std": 0.011115207336843014, "rewards//mean": 0.749755859375, "rewards//std": 0.029926680028438568, "step": 1637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3276, "grad_norm": 12.250699043273926, "kl": 3.8545846436172724, "learning_rate": 7.671091140501555e-07, "loss": 0.1542, "num_tokens": 14158001.0, "reward": 0.75225830078125, "reward_std": 0.011953351087868214, "rewards//mean": 0.75225830078125, "rewards//std": 0.021062320098280907, "step": 1638 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.265625, "epoch": 0.3278, "grad_norm": 14.134915351867676, "kl": 5.8936877734959126, "learning_rate": 7.668408042433081e-07, "loss": 0.224, "num_tokens": 14166562.0, "reward": 0.7552490234375, "reward_std": 0.009656651876866817, "rewards//mean": 0.7552490234375, "rewards//std": 0.03004254214465618, "step": 1639 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.328, "grad_norm": 10.148066520690918, "kl": 6.543967846781015, "learning_rate": 7.665723869530169e-07, "loss": 0.2618, "num_tokens": 14175130.0, "reward": 0.76983642578125, "reward_std": 0.014795171096920967, "rewards//mean": 0.76983642578125, "rewards//std": 0.026994338259100914, "step": 1640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3282, "grad_norm": 26.754764556884766, "kl": 8.376775816082954, "learning_rate": 7.663038622873999e-07, "loss": 0.3351, "num_tokens": 14183738.0, "reward": 0.740234375, "reward_std": 0.014836485497653484, "rewards//mean": 0.740234375, "rewards//std": 0.03266720846295357, "step": 1641 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.546875, "epoch": 0.3284, "grad_norm": 11.834562301635742, "kl": 4.77639720775187, "learning_rate": 7.660352303546192e-07, "loss": 0.1959, "num_tokens": 14192349.0, "reward": 0.72686767578125, "reward_std": 0.010114647448062897, "rewards//mean": 0.72686767578125, "rewards//std": 0.03528556600213051, "step": 1642 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3286, "grad_norm": 27.761306762695312, "kl": 9.532374881207943, "learning_rate": 7.657664912628794e-07, "loss": 0.3813, "num_tokens": 14201133.0, "reward": 0.7391357421875, "reward_std": 0.01447632908821106, "rewards//mean": 0.7391357421875, "rewards//std": 0.03351961448788643, "step": 1643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3288, "grad_norm": 12.942228317260742, "kl": 5.6833315175026655, "learning_rate": 7.654976451204287e-07, "loss": 0.2273, "num_tokens": 14209693.0, "reward": 0.72845458984375, "reward_std": 0.011466937139630318, "rewards//mean": 0.72845458984375, "rewards//std": 0.030040716752409935, "step": 1644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.329, "grad_norm": 8.533147811889648, "kl": 3.2423823177814484, "learning_rate": 7.652286920355583e-07, "loss": 0.1297, "num_tokens": 14218405.0, "reward": 0.7584228515625, "reward_std": 0.010816293768584728, "rewards//mean": 0.7584228515625, "rewards//std": 0.033539481461048126, "step": 1645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3292, "grad_norm": 19.20831298828125, "kl": 5.504130233079195, "learning_rate": 7.649596321166024e-07, "loss": 0.2202, "num_tokens": 14227165.0, "reward": 0.76483154296875, "reward_std": 0.011215681210160255, "rewards//mean": 0.76483154296875, "rewards//std": 0.025996796786785126, "step": 1646 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3294, "grad_norm": 6.512764930725098, "kl": 4.2323793433606625, "learning_rate": 7.646904654719385e-07, "loss": 0.1693, "num_tokens": 14235821.0, "reward": 0.7633056640625, "reward_std": 0.01945539563894272, "rewards//mean": 0.7633056640625, "rewards//std": 0.026854129508137703, "step": 1647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.984375, "epoch": 0.3296, "grad_norm": 7.150025367736816, "kl": 3.510749952867627, "learning_rate": 7.644211922099867e-07, "loss": 0.1404, "num_tokens": 14244436.0, "reward": 0.74188232421875, "reward_std": 0.0065304902382195, "rewards//mean": 0.74188232421875, "rewards//std": 0.022737829014658928, "step": 1648 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3298, "grad_norm": 18.398927688598633, "kl": 4.953672690317035, "learning_rate": 7.641518124392103e-07, "loss": 0.1981, "num_tokens": 14253076.0, "reward": 0.7508544921875, "reward_std": 0.005613813176751137, "rewards//mean": 0.7508544921875, "rewards//std": 0.018939048051834106, "step": 1649 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.33, "grad_norm": 5.904971122741699, "kl": 2.5764774549752474, "learning_rate": 7.638823262681154e-07, "loss": 0.1031, "num_tokens": 14261660.0, "reward": 0.77178955078125, "reward_std": 0.010225379839539528, "rewards//mean": 0.77178955078125, "rewards//std": 0.031217467039823532, "step": 1650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3302, "grad_norm": 11.138781547546387, "kl": 3.0971457418054342, "learning_rate": 7.636127338052511e-07, "loss": 0.1239, "num_tokens": 14270276.0, "reward": 0.7667236328125, "reward_std": 0.012113604694604874, "rewards//mean": 0.7667236328125, "rewards//std": 0.027396569028496742, "step": 1651 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3304, "grad_norm": 10.846475601196289, "kl": 2.1628370974212885, "learning_rate": 7.633430351592093e-07, "loss": 0.0865, "num_tokens": 14278900.0, "reward": 0.75048828125, "reward_std": 0.004749837331473827, "rewards//mean": 0.75048828125, "rewards//std": 0.028641607612371445, "step": 1652 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3306, "grad_norm": 15.916720390319824, "kl": 1.1707215327769518, "learning_rate": 7.630732304386243e-07, "loss": 0.0468, "num_tokens": 14287516.0, "reward": 0.77703857421875, "reward_std": 0.004662022925913334, "rewards//mean": 0.77703857421875, "rewards//std": 0.027039162814617157, "step": 1653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3308, "grad_norm": 22.2278995513916, "kl": 2.5529266484081745, "learning_rate": 7.628033197521735e-07, "loss": 0.1021, "num_tokens": 14296180.0, "reward": 0.74749755859375, "reward_std": 0.008782346732914448, "rewards//mean": 0.74749755859375, "rewards//std": 0.019995320588350296, "step": 1654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.331, "grad_norm": 15.795402526855469, "kl": 1.6755111087113619, "learning_rate": 7.625333032085769e-07, "loss": 0.067, "num_tokens": 14304812.0, "reward": 0.74957275390625, "reward_std": 0.008638689294457436, "rewards//mean": 0.74957275390625, "rewards//std": 0.022607633844017982, "step": 1655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3312, "grad_norm": 6.203474521636963, "kl": 2.146116741001606, "learning_rate": 7.622631809165972e-07, "loss": 0.0858, "num_tokens": 14313468.0, "reward": 0.77423095703125, "reward_std": 0.010866011492908001, "rewards//mean": 0.77423095703125, "rewards//std": 0.026722678914666176, "step": 1656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3314, "grad_norm": 6.657282829284668, "kl": 2.3803008049726486, "learning_rate": 7.619929529850396e-07, "loss": 0.0952, "num_tokens": 14322052.0, "reward": 0.7696533203125, "reward_std": 0.006360728293657303, "rewards//mean": 0.7696533203125, "rewards//std": 0.025901490822434425, "step": 1657 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3316, "grad_norm": 8.862725257873535, "kl": 2.4458378311246634, "learning_rate": 7.617226195227517e-07, "loss": 0.0978, "num_tokens": 14330684.0, "reward": 0.778076171875, "reward_std": 0.006107522174715996, "rewards//mean": 0.778076171875, "rewards//std": 0.018905848264694214, "step": 1658 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3318, "grad_norm": 8.068026542663574, "kl": 3.9540002550929785, "learning_rate": 7.614521806386243e-07, "loss": 0.1582, "num_tokens": 14339292.0, "reward": 0.74945068359375, "reward_std": 0.006683437153697014, "rewards//mean": 0.74945068359375, "rewards//std": 0.01489273738116026, "step": 1659 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.332, "grad_norm": 31.912376403808594, "kl": 7.891320111230016, "learning_rate": 7.611816364415895e-07, "loss": 0.3157, "num_tokens": 14347980.0, "reward": 0.72967529296875, "reward_std": 0.02069058082997799, "rewards//mean": 0.72967529296875, "rewards//std": 0.049942903220653534, "step": 1660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3322, "grad_norm": 11.64772891998291, "kl": 1.9665944166481495, "learning_rate": 7.60910987040623e-07, "loss": 0.0787, "num_tokens": 14356572.0, "reward": 0.76629638671875, "reward_std": 0.005575934890657663, "rewards//mean": 0.76629638671875, "rewards//std": 0.016507720574736595, "step": 1661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3324, "grad_norm": 16.980592727661133, "kl": 6.588000485673547, "learning_rate": 7.606402325447419e-07, "loss": 0.2635, "num_tokens": 14365332.0, "reward": 0.7310791015625, "reward_std": 0.016998887062072754, "rewards//mean": 0.7310791015625, "rewards//std": 0.03285365551710129, "step": 1662 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3326, "grad_norm": 29.69980812072754, "kl": 7.07955215126276, "learning_rate": 7.603693730630066e-07, "loss": 0.2832, "num_tokens": 14373972.0, "reward": 0.74029541015625, "reward_std": 0.011502914130687714, "rewards//mean": 0.74029541015625, "rewards//std": 0.029667524620890617, "step": 1663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3328, "grad_norm": 16.89678192138672, "kl": 6.799922564998269, "learning_rate": 7.600984087045186e-07, "loss": 0.272, "num_tokens": 14382628.0, "reward": 0.74896240234375, "reward_std": 0.011954502202570438, "rewards//mean": 0.74896240234375, "rewards//std": 0.03989831358194351, "step": 1664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.333, "grad_norm": 11.457413673400879, "kl": 2.7482737116515636, "learning_rate": 7.598273395784229e-07, "loss": 0.1099, "num_tokens": 14391236.0, "reward": 0.78363037109375, "reward_std": 0.008042111992835999, "rewards//mean": 0.78363037109375, "rewards//std": 0.028481479734182358, "step": 1665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3332, "grad_norm": 12.461380958557129, "kl": 4.245392709970474, "learning_rate": 7.59556165793906e-07, "loss": 0.1698, "num_tokens": 14399820.0, "reward": 0.72528076171875, "reward_std": 0.006810404360294342, "rewards//mean": 0.72528076171875, "rewards//std": 0.03248918801546097, "step": 1666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3334, "grad_norm": 6.8713226318359375, "kl": 3.91130817681551, "learning_rate": 7.592848874601963e-07, "loss": 0.1565, "num_tokens": 14408460.0, "reward": 0.76318359375, "reward_std": 0.012699104845523834, "rewards//mean": 0.76318359375, "rewards//std": 0.03380763530731201, "step": 1667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3336, "grad_norm": 7.350086212158203, "kl": 5.778000093996525, "learning_rate": 7.590135046865651e-07, "loss": 0.2311, "num_tokens": 14417084.0, "reward": 0.76361083984375, "reward_std": 0.011498469859361649, "rewards//mean": 0.76361083984375, "rewards//std": 0.03844371438026428, "step": 1668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3338, "grad_norm": 11.982793807983398, "kl": 5.074105879291892, "learning_rate": 7.587420175823252e-07, "loss": 0.203, "num_tokens": 14425684.0, "reward": 0.746337890625, "reward_std": 0.017222050577402115, "rewards//mean": 0.746337890625, "rewards//std": 0.028270291164517403, "step": 1669 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.1875, "epoch": 0.334, "grad_norm": 19.137075424194336, "kl": 3.788899175822735, "learning_rate": 7.584704262568314e-07, "loss": 0.1383, "num_tokens": 14434200.0, "reward": 0.75244140625, "reward_std": 0.008026419207453728, "rewards//mean": 0.75244140625, "rewards//std": 0.026268543675541878, "step": 1670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3342, "grad_norm": 8.649948120117188, "kl": 2.3480862453579903, "learning_rate": 7.581987308194809e-07, "loss": 0.0939, "num_tokens": 14442848.0, "reward": 0.76666259765625, "reward_std": 0.01028990838676691, "rewards//mean": 0.76666259765625, "rewards//std": 0.028299124911427498, "step": 1671 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3344, "grad_norm": 16.29898452758789, "kl": 5.2496380638331175, "learning_rate": 7.579269313797125e-07, "loss": 0.21, "num_tokens": 14451376.0, "reward": 0.7310791015625, "reward_std": 0.00796603225171566, "rewards//mean": 0.7310791015625, "rewards//std": 0.025640688836574554, "step": 1672 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3346, "grad_norm": 5.378942966461182, "kl": 4.060715341940522, "learning_rate": 7.576550280470071e-07, "loss": 0.1624, "num_tokens": 14460112.0, "reward": 0.76409912109375, "reward_std": 0.01511688157916069, "rewards//mean": 0.76409912109375, "rewards//std": 0.02715814672410488, "step": 1673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3348, "grad_norm": 9.949033737182617, "kl": 1.6426987908780575, "learning_rate": 7.573830209308872e-07, "loss": 0.0657, "num_tokens": 14468744.0, "reward": 0.7333984375, "reward_std": 0.008231493644416332, "rewards//mean": 0.7333984375, "rewards//std": 0.031117962673306465, "step": 1674 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.335, "grad_norm": 6.0058064460754395, "kl": 2.6325108893215656, "learning_rate": 7.57110910140917e-07, "loss": 0.1053, "num_tokens": 14477368.0, "reward": 0.755126953125, "reward_std": 0.010620414279401302, "rewards//mean": 0.755126953125, "rewards//std": 0.02269059419631958, "step": 1675 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3352, "grad_norm": 6.265772819519043, "kl": 2.351148357614875, "learning_rate": 7.568386957867032e-07, "loss": 0.094, "num_tokens": 14486008.0, "reward": 0.76678466796875, "reward_std": 0.007720914203673601, "rewards//mean": 0.76678466796875, "rewards//std": 0.02256004326045513, "step": 1676 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3354, "grad_norm": 7.135132789611816, "kl": 2.418541956692934, "learning_rate": 7.565663779778933e-07, "loss": 0.0967, "num_tokens": 14494608.0, "reward": 0.74639892578125, "reward_std": 0.0088096484541893, "rewards//mean": 0.74639892578125, "rewards//std": 0.02282288484275341, "step": 1677 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3356, "grad_norm": 16.368267059326172, "kl": 3.6359867565333843, "learning_rate": 7.562939568241771e-07, "loss": 0.1454, "num_tokens": 14503208.0, "reward": 0.76263427734375, "reward_std": 0.010166294872760773, "rewards//mean": 0.76263427734375, "rewards//std": 0.014939420856535435, "step": 1678 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3358, "grad_norm": 5.093266010284424, "kl": 1.491914639249444, "learning_rate": 7.560214324352858e-07, "loss": 0.0597, "num_tokens": 14511840.0, "reward": 0.7373046875, "reward_std": 0.003280874341726303, "rewards//mean": 0.7373046875, "rewards//std": 0.026955686509609222, "step": 1679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.336, "grad_norm": 8.144636154174805, "kl": 2.80340170674026, "learning_rate": 7.55748804920992e-07, "loss": 0.1121, "num_tokens": 14520544.0, "reward": 0.76519775390625, "reward_std": 0.012464512139558792, "rewards//mean": 0.76519775390625, "rewards//std": 0.03017847053706646, "step": 1680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3362, "grad_norm": 11.081323623657227, "kl": 6.226683981716633, "learning_rate": 7.554760743911103e-07, "loss": 0.2491, "num_tokens": 14529264.0, "reward": 0.73345947265625, "reward_std": 0.01712070405483246, "rewards//mean": 0.73345947265625, "rewards//std": 0.03184684365987778, "step": 1681 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3364, "grad_norm": 6.9883222579956055, "kl": 2.3179250191897154, "learning_rate": 7.552032409554962e-07, "loss": 0.0927, "num_tokens": 14537864.0, "reward": 0.76922607421875, "reward_std": 0.008112422190606594, "rewards//mean": 0.76922607421875, "rewards//std": 0.020271258428692818, "step": 1682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3366, "grad_norm": 6.01211404800415, "kl": 2.5319530349224806, "learning_rate": 7.549303047240474e-07, "loss": 0.1013, "num_tokens": 14546472.0, "reward": 0.77532958984375, "reward_std": 0.006156784947961569, "rewards//mean": 0.77532958984375, "rewards//std": 0.03255574777722359, "step": 1683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3368, "grad_norm": 9.942745208740234, "kl": 2.4515086635947227, "learning_rate": 7.54657265806702e-07, "loss": 0.0981, "num_tokens": 14555080.0, "reward": 0.7755126953125, "reward_std": 0.009586556814610958, "rewards//mean": 0.7755126953125, "rewards//std": 0.02556264027953148, "step": 1684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.337, "grad_norm": 16.847671508789062, "kl": 4.299041600897908, "learning_rate": 7.543841243134408e-07, "loss": 0.172, "num_tokens": 14563712.0, "reward": 0.7554931640625, "reward_std": 0.013671418651938438, "rewards//mean": 0.7554931640625, "rewards//std": 0.0289964247494936, "step": 1685 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3372, "grad_norm": 11.128751754760742, "kl": 4.528183134272695, "learning_rate": 7.541108803542845e-07, "loss": 0.1811, "num_tokens": 14572320.0, "reward": 0.7733154296875, "reward_std": 0.020395830273628235, "rewards//mean": 0.7733154296875, "rewards//std": 0.031156614422798157, "step": 1686 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3374, "grad_norm": 36.73457336425781, "kl": 7.407161274924874, "learning_rate": 7.538375340392961e-07, "loss": 0.2963, "num_tokens": 14580984.0, "reward": 0.7178955078125, "reward_std": 0.01460280641913414, "rewards//mean": 0.7178955078125, "rewards//std": 0.03694973886013031, "step": 1687 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.703125, "epoch": 0.3376, "grad_norm": 19.612701416015625, "kl": 7.426185131072998, "learning_rate": 7.535640854785791e-07, "loss": 0.2928, "num_tokens": 14589573.0, "reward": 0.7728271484375, "reward_std": 0.02155712991952896, "rewards//mean": 0.7728271484375, "rewards//std": 0.028461076319217682, "step": 1688 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3378, "grad_norm": 16.157751083374023, "kl": 6.999772354960442, "learning_rate": 7.532905347822791e-07, "loss": 0.28, "num_tokens": 14598197.0, "reward": 0.72467041015625, "reward_std": 0.022015413269400597, "rewards//mean": 0.72467041015625, "rewards//std": 0.038431111723184586, "step": 1689 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.338, "grad_norm": 6.553281784057617, "kl": 4.209701828658581, "learning_rate": 7.530168820605818e-07, "loss": 0.1684, "num_tokens": 14606869.0, "reward": 0.74798583984375, "reward_std": 0.012180760502815247, "rewards//mean": 0.74798583984375, "rewards//std": 0.029198473319411278, "step": 1690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3382, "grad_norm": 32.15467071533203, "kl": 9.1272879447788, "learning_rate": 7.527431274237149e-07, "loss": 0.3651, "num_tokens": 14615597.0, "reward": 0.71746826171875, "reward_std": 0.011943129822611809, "rewards//mean": 0.71746826171875, "rewards//std": 0.028194623067975044, "step": 1691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3384, "grad_norm": 6.4996209144592285, "kl": 5.02642223611474, "learning_rate": 7.524692709819463e-07, "loss": 0.2011, "num_tokens": 14624213.0, "reward": 0.76025390625, "reward_std": 0.016095131635665894, "rewards//mean": 0.76025390625, "rewards//std": 0.02068750187754631, "step": 1692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3386, "grad_norm": 11.062028884887695, "kl": 4.206334099173546, "learning_rate": 7.521953128455855e-07, "loss": 0.1683, "num_tokens": 14632813.0, "reward": 0.7607421875, "reward_std": 0.007870189845561981, "rewards//mean": 0.7607421875, "rewards//std": 0.025995122268795967, "step": 1693 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3388, "grad_norm": 11.231415748596191, "kl": 3.808822438120842, "learning_rate": 7.519212531249829e-07, "loss": 0.1524, "num_tokens": 14641413.0, "reward": 0.74603271484375, "reward_std": 0.009955466724932194, "rewards//mean": 0.74603271484375, "rewards//std": 0.022331418469548225, "step": 1694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.339, "grad_norm": 6.933574676513672, "kl": 2.020120469853282, "learning_rate": 7.516470919305298e-07, "loss": 0.0808, "num_tokens": 14650029.0, "reward": 0.74200439453125, "reward_std": 0.010920392349362373, "rewards//mean": 0.74200439453125, "rewards//std": 0.023688610643148422, "step": 1695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3392, "grad_norm": 12.914809226989746, "kl": 2.8975869584828615, "learning_rate": 7.513728293726579e-07, "loss": 0.1159, "num_tokens": 14658589.0, "reward": 0.7923583984375, "reward_std": 0.009334616363048553, "rewards//mean": 0.7923583984375, "rewards//std": 0.016619790345430374, "step": 1696 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3394, "grad_norm": 15.476243019104004, "kl": 1.4073762707412243, "learning_rate": 7.510984655618406e-07, "loss": 0.0563, "num_tokens": 14667221.0, "reward": 0.7314453125, "reward_std": 0.0075613269582390785, "rewards//mean": 0.7314453125, "rewards//std": 0.026955686509609222, "step": 1697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3396, "grad_norm": 15.04116439819336, "kl": 3.4532814882695675, "learning_rate": 7.508240006085913e-07, "loss": 0.1381, "num_tokens": 14675909.0, "reward": 0.755615234375, "reward_std": 0.0094823706895113, "rewards//mean": 0.755615234375, "rewards//std": 0.018040519207715988, "step": 1698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3398, "grad_norm": 10.518776893615723, "kl": 2.346025651320815, "learning_rate": 7.505494346234647e-07, "loss": 0.0938, "num_tokens": 14684477.0, "reward": 0.76666259765625, "reward_std": 0.011838393285870552, "rewards//mean": 0.76666259765625, "rewards//std": 0.024697864428162575, "step": 1699 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.34, "grad_norm": 13.792692184448242, "kl": 6.545206168666482, "learning_rate": 7.502747677170555e-07, "loss": 0.2618, "num_tokens": 14693117.0, "reward": 0.75, "reward_std": 0.01461731269955635, "rewards//mean": 0.75, "rewards//std": 0.029181944206357002, "step": 1700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3402, "grad_norm": 8.20025634765625, "kl": 4.336450120434165, "learning_rate": 7.5e-07, "loss": 0.1735, "num_tokens": 14701757.0, "reward": 0.7445068359375, "reward_std": 0.013547065667808056, "rewards//mean": 0.7445068359375, "rewards//std": 0.03472801297903061, "step": 1701 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3404, "grad_norm": 13.76013469696045, "kl": 4.508783459663391, "learning_rate": 7.497251315829743e-07, "loss": 0.1804, "num_tokens": 14710437.0, "reward": 0.75933837890625, "reward_std": 0.008789284154772758, "rewards//mean": 0.75933837890625, "rewards//std": 0.01940830983221531, "step": 1702 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3406, "grad_norm": 14.223562240600586, "kl": 5.793676137924194, "learning_rate": 7.494501625766955e-07, "loss": 0.2317, "num_tokens": 14719157.0, "reward": 0.76434326171875, "reward_std": 0.011211156845092773, "rewards//mean": 0.76434326171875, "rewards//std": 0.028211798518896103, "step": 1703 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3408, "grad_norm": 7.0896711349487305, "kl": 4.318609192967415, "learning_rate": 7.491750930919212e-07, "loss": 0.1727, "num_tokens": 14727781.0, "reward": 0.76995849609375, "reward_std": 0.00895887054502964, "rewards//mean": 0.76995849609375, "rewards//std": 0.02394981123507023, "step": 1704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.341, "grad_norm": 12.570860862731934, "kl": 4.320924017578363, "learning_rate": 7.488999232394491e-07, "loss": 0.1728, "num_tokens": 14736469.0, "reward": 0.75244140625, "reward_std": 0.006509346887469292, "rewards//mean": 0.75244140625, "rewards//std": 0.024324923753738403, "step": 1705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3412, "grad_norm": 16.738834381103516, "kl": 2.4809902161359787, "learning_rate": 7.486246531301177e-07, "loss": 0.0992, "num_tokens": 14745133.0, "reward": 0.72711181640625, "reward_std": 0.013168182224035263, "rewards//mean": 0.72711181640625, "rewards//std": 0.03143443912267685, "step": 1706 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.46875, "epoch": 0.3414, "grad_norm": 6.93234395980835, "kl": 5.587183387950063, "learning_rate": 7.483492828748056e-07, "loss": 0.2142, "num_tokens": 14753827.0, "reward": 0.7552490234375, "reward_std": 0.016453320160508156, "rewards//mean": 0.7552490234375, "rewards//std": 0.02872367389500141, "step": 1707 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3416, "grad_norm": 22.446189880371094, "kl": 2.741853207349777, "learning_rate": 7.480738125844322e-07, "loss": 0.1097, "num_tokens": 14762443.0, "reward": 0.73297119140625, "reward_std": 0.011288801208138466, "rewards//mean": 0.73297119140625, "rewards//std": 0.030681896954774857, "step": 1708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3418, "grad_norm": 12.996790885925293, "kl": 2.3409229721874, "learning_rate": 7.477982423699567e-07, "loss": 0.0936, "num_tokens": 14771099.0, "reward": 0.772705078125, "reward_std": 0.008740413933992386, "rewards//mean": 0.772705078125, "rewards//std": 0.01942399889230728, "step": 1709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.342, "grad_norm": 8.097086906433105, "kl": 1.8513435777276754, "learning_rate": 7.475225723423788e-07, "loss": 0.0741, "num_tokens": 14779779.0, "reward": 0.7666015625, "reward_std": 0.007709752302616835, "rewards//mean": 0.7666015625, "rewards//std": 0.021202094852924347, "step": 1710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3422, "grad_norm": 12.267953872680664, "kl": 5.723224747925997, "learning_rate": 7.472468026127384e-07, "loss": 0.2289, "num_tokens": 14788555.0, "reward": 0.76629638671875, "reward_std": 0.009873709641397, "rewards//mean": 0.76629638671875, "rewards//std": 0.02834455668926239, "step": 1711 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3424, "grad_norm": 4.721216201782227, "kl": 4.228861682116985, "learning_rate": 7.469709332921154e-07, "loss": 0.1692, "num_tokens": 14797115.0, "reward": 0.73907470703125, "reward_std": 0.011778753250837326, "rewards//mean": 0.73907470703125, "rewards//std": 0.027945473790168762, "step": 1712 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3426, "grad_norm": 7.927215576171875, "kl": 4.117089059203863, "learning_rate": 7.4669496449163e-07, "loss": 0.1647, "num_tokens": 14805763.0, "reward": 0.74078369140625, "reward_std": 0.01242013182491064, "rewards//mean": 0.74078369140625, "rewards//std": 0.020825976505875587, "step": 1713 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3428, "grad_norm": 16.150463104248047, "kl": 6.161867793649435, "learning_rate": 7.464188963224427e-07, "loss": 0.2465, "num_tokens": 14814371.0, "reward": 0.75897216796875, "reward_std": 0.011923184618353844, "rewards//mean": 0.75897216796875, "rewards//std": 0.03031909465789795, "step": 1714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.734375, "epoch": 0.343, "grad_norm": 8.781957626342773, "kl": 3.492230284959078, "learning_rate": 7.461427288957531e-07, "loss": 0.1432, "num_tokens": 14822954.0, "reward": 0.74908447265625, "reward_std": 0.008261223323643208, "rewards//mean": 0.74908447265625, "rewards//std": 0.02625233121216297, "step": 1715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3432, "grad_norm": 17.5076961517334, "kl": 3.583744313567877, "learning_rate": 7.45866462322802e-07, "loss": 0.1433, "num_tokens": 14831530.0, "reward": 0.7747802734375, "reward_std": 0.009402611292898655, "rewards//mean": 0.7747802734375, "rewards//std": 0.019872011616826057, "step": 1716 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3434, "grad_norm": 11.033143997192383, "kl": 5.401227429509163, "learning_rate": 7.45590096714869e-07, "loss": 0.216, "num_tokens": 14840114.0, "reward": 0.72332763671875, "reward_std": 0.020782142877578735, "rewards//mean": 0.72332763671875, "rewards//std": 0.029228009283542633, "step": 1717 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3436, "grad_norm": 16.427082061767578, "kl": 6.726500168442726, "learning_rate": 7.453136321832745e-07, "loss": 0.2691, "num_tokens": 14848722.0, "reward": 0.760009765625, "reward_std": 0.012372981756925583, "rewards//mean": 0.760009765625, "rewards//std": 0.021473098546266556, "step": 1718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3438, "grad_norm": 26.458110809326172, "kl": 7.284464478492737, "learning_rate": 7.450370688393784e-07, "loss": 0.2914, "num_tokens": 14857322.0, "reward": 0.73175048828125, "reward_std": 0.012284617871046066, "rewards//mean": 0.73175048828125, "rewards//std": 0.02642074227333069, "step": 1719 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.344, "grad_norm": 13.820018768310547, "kl": 5.9415582139045, "learning_rate": 7.447604067945802e-07, "loss": 0.2377, "num_tokens": 14866018.0, "reward": 0.72735595703125, "reward_std": 0.013435333967208862, "rewards//mean": 0.72735595703125, "rewards//std": 0.0270917359739542, "step": 1720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3442, "grad_norm": 14.459245681762695, "kl": 5.953585060313344, "learning_rate": 7.444836461603194e-07, "loss": 0.2381, "num_tokens": 14874850.0, "reward": 0.75592041015625, "reward_std": 0.011368580162525177, "rewards//mean": 0.75592041015625, "rewards//std": 0.024279998615384102, "step": 1721 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3444, "grad_norm": 11.407780647277832, "kl": 5.483324466273189, "learning_rate": 7.442067870480751e-07, "loss": 0.2193, "num_tokens": 14883418.0, "reward": 0.7259521484375, "reward_std": 0.009960510767996311, "rewards//mean": 0.7259521484375, "rewards//std": 0.03520938754081726, "step": 1722 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3446, "grad_norm": 10.866833686828613, "kl": 4.219879038631916, "learning_rate": 7.439298295693663e-07, "loss": 0.1688, "num_tokens": 14892090.0, "reward": 0.7445068359375, "reward_std": 0.006772883702069521, "rewards//mean": 0.7445068359375, "rewards//std": 0.03010294772684574, "step": 1723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3448, "grad_norm": 36.98476791381836, "kl": 3.116904040798545, "learning_rate": 7.436527738357513e-07, "loss": 0.1247, "num_tokens": 14900690.0, "reward": 0.76971435546875, "reward_std": 0.01532891858369112, "rewards//mean": 0.76971435546875, "rewards//std": 0.03278140723705292, "step": 1724 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.345, "grad_norm": 5.003154754638672, "kl": 2.0285655725747347, "learning_rate": 7.433756199588282e-07, "loss": 0.0811, "num_tokens": 14909338.0, "reward": 0.7744140625, "reward_std": 0.004872136749327183, "rewards//mean": 0.7744140625, "rewards//std": 0.02111050859093666, "step": 1725 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3452, "grad_norm": 18.3594913482666, "kl": 1.8213788475841284, "learning_rate": 7.430983680502343e-07, "loss": 0.0729, "num_tokens": 14918018.0, "reward": 0.75994873046875, "reward_std": 0.00850547756999731, "rewards//mean": 0.75994873046875, "rewards//std": 0.02573225647211075, "step": 1726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3454, "grad_norm": 6.738495826721191, "kl": 1.195437179878354, "learning_rate": 7.42821018221647e-07, "loss": 0.0478, "num_tokens": 14926762.0, "reward": 0.78759765625, "reward_std": 0.003485372057184577, "rewards//mean": 0.78759765625, "rewards//std": 0.02826707810163498, "step": 1727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3456, "grad_norm": 7.730818271636963, "kl": 2.8814690820872784, "learning_rate": 7.425435705847825e-07, "loss": 0.1153, "num_tokens": 14935474.0, "reward": 0.7613525390625, "reward_std": 0.010984411463141441, "rewards//mean": 0.7613525390625, "rewards//std": 0.01968218944966793, "step": 1728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3458, "grad_norm": 10.770354270935059, "kl": 2.826941965147853, "learning_rate": 7.422660252513968e-07, "loss": 0.1131, "num_tokens": 14944122.0, "reward": 0.75079345703125, "reward_std": 0.010136321187019348, "rewards//mean": 0.75079345703125, "rewards//std": 0.021352121606469154, "step": 1729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.346, "grad_norm": 8.526028633117676, "kl": 1.2991268746554852, "learning_rate": 7.41988382333285e-07, "loss": 0.052, "num_tokens": 14952810.0, "reward": 0.74993896484375, "reward_std": 0.00819031335413456, "rewards//mean": 0.74993896484375, "rewards//std": 0.02521466091275215, "step": 1730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3462, "grad_norm": 6.339461803436279, "kl": 1.9494109824299812, "learning_rate": 7.417106419422818e-07, "loss": 0.078, "num_tokens": 14961378.0, "reward": 0.7952880859375, "reward_std": 0.0145040825009346, "rewards//mean": 0.7952880859375, "rewards//std": 0.027425285428762436, "step": 1731 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3464, "grad_norm": 19.363574981689453, "kl": 1.2932778727263212, "learning_rate": 7.41432804190261e-07, "loss": 0.0517, "num_tokens": 14969962.0, "reward": 0.72003173828125, "reward_std": 0.003600679337978363, "rewards//mean": 0.72003173828125, "rewards//std": 0.027648132294416428, "step": 1732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3466, "grad_norm": 6.703650951385498, "kl": 3.404814265668392, "learning_rate": 7.411548691891357e-07, "loss": 0.1362, "num_tokens": 14978610.0, "reward": 0.7841796875, "reward_std": 0.017170464619994164, "rewards//mean": 0.7841796875, "rewards//std": 0.03392563387751579, "step": 1733 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3468, "grad_norm": 4.404158592224121, "kl": 2.9254848584532738, "learning_rate": 7.408768370508576e-07, "loss": 0.117, "num_tokens": 14987322.0, "reward": 0.76251220703125, "reward_std": 0.01290039625018835, "rewards//mean": 0.76251220703125, "rewards//std": 0.028272902593016624, "step": 1734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.347, "grad_norm": 22.63078498840332, "kl": 2.2710730005055666, "learning_rate": 7.405987078874185e-07, "loss": 0.0908, "num_tokens": 14995930.0, "reward": 0.7584228515625, "reward_std": 0.012374240905046463, "rewards//mean": 0.7584228515625, "rewards//std": 0.03130590170621872, "step": 1735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3472, "grad_norm": 13.92486572265625, "kl": 2.887278689071536, "learning_rate": 7.403204818108487e-07, "loss": 0.1155, "num_tokens": 15004594.0, "reward": 0.7598876953125, "reward_std": 0.011876486241817474, "rewards//mean": 0.7598876953125, "rewards//std": 0.03911187872290611, "step": 1736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3474, "grad_norm": 7.4634690284729, "kl": 2.634060813114047, "learning_rate": 7.400421589332174e-07, "loss": 0.1054, "num_tokens": 15013186.0, "reward": 0.76824951171875, "reward_std": 0.00886482559144497, "rewards//mean": 0.76824951171875, "rewards//std": 0.0291574876755476, "step": 1737 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3476, "grad_norm": 8.981084823608398, "kl": 4.878067035228014, "learning_rate": 7.397637393666333e-07, "loss": 0.1951, "num_tokens": 15021962.0, "reward": 0.7420654296875, "reward_std": 0.01109348889440298, "rewards//mean": 0.7420654296875, "rewards//std": 0.024269787594676018, "step": 1738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3478, "grad_norm": 14.383384704589844, "kl": 2.241973152384162, "learning_rate": 7.394852232232436e-07, "loss": 0.0897, "num_tokens": 15030642.0, "reward": 0.79779052734375, "reward_std": 0.011168712750077248, "rewards//mean": 0.79779052734375, "rewards//std": 0.02340381033718586, "step": 1739 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.348, "grad_norm": 7.720381736755371, "kl": 3.893526293337345, "learning_rate": 7.392066106152345e-07, "loss": 0.1557, "num_tokens": 15039298.0, "reward": 0.7689208984375, "reward_std": 0.008886430412530899, "rewards//mean": 0.7689208984375, "rewards//std": 0.025546055287122726, "step": 1740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3482, "grad_norm": 5.949435710906982, "kl": 2.1611736863851547, "learning_rate": 7.389279016548316e-07, "loss": 0.0864, "num_tokens": 15047898.0, "reward": 0.75262451171875, "reward_std": 0.008822827599942684, "rewards//mean": 0.75262451171875, "rewards//std": 0.028772840276360512, "step": 1741 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3484, "grad_norm": 3.424663543701172, "kl": 2.818216495215893, "learning_rate": 7.386490964542982e-07, "loss": 0.1127, "num_tokens": 15056522.0, "reward": 0.78424072265625, "reward_std": 0.009205842390656471, "rewards//mean": 0.78424072265625, "rewards//std": 0.02906336635351181, "step": 1742 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3486, "grad_norm": 17.820810317993164, "kl": 2.490190641954541, "learning_rate": 7.383701951259375e-07, "loss": 0.0996, "num_tokens": 15065074.0, "reward": 0.7767333984375, "reward_std": 0.006568721029907465, "rewards//mean": 0.7767333984375, "rewards//std": 0.02129577472805977, "step": 1743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3488, "grad_norm": 9.225910186767578, "kl": 5.337249953299761, "learning_rate": 7.380911977820906e-07, "loss": 0.2135, "num_tokens": 15073762.0, "reward": 0.7352294921875, "reward_std": 0.015421277843415737, "rewards//mean": 0.7352294921875, "rewards//std": 0.024304691702127457, "step": 1744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.349, "grad_norm": 21.589153289794922, "kl": 6.036288775503635, "learning_rate": 7.378121045351377e-07, "loss": 0.2415, "num_tokens": 15082394.0, "reward": 0.744873046875, "reward_std": 0.010433953255414963, "rewards//mean": 0.744873046875, "rewards//std": 0.023165950551629066, "step": 1745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3492, "grad_norm": 11.4340181350708, "kl": 1.4951294343918562, "learning_rate": 7.375329154974975e-07, "loss": 0.0598, "num_tokens": 15091026.0, "reward": 0.7608642578125, "reward_std": 0.005236084572970867, "rewards//mean": 0.7608642578125, "rewards//std": 0.02316170372068882, "step": 1746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3494, "grad_norm": 12.729737281799316, "kl": 5.8137662298977375, "learning_rate": 7.372536307816272e-07, "loss": 0.2326, "num_tokens": 15099618.0, "reward": 0.759765625, "reward_std": 0.010998114943504333, "rewards//mean": 0.759765625, "rewards//std": 0.01820257492363453, "step": 1747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3496, "grad_norm": 15.836647033691406, "kl": 5.377154625952244, "learning_rate": 7.369742505000231e-07, "loss": 0.2151, "num_tokens": 15108242.0, "reward": 0.75054931640625, "reward_std": 0.010375864803791046, "rewards//mean": 0.75054931640625, "rewards//std": 0.036457452923059464, "step": 1748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3498, "grad_norm": 28.29893684387207, "kl": 6.580530092120171, "learning_rate": 7.366947747652191e-07, "loss": 0.2632, "num_tokens": 15116810.0, "reward": 0.72918701171875, "reward_std": 0.01071253977715969, "rewards//mean": 0.72918701171875, "rewards//std": 0.026929209008812904, "step": 1749 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.35, "grad_norm": 19.780168533325195, "kl": 7.418893063440919, "learning_rate": 7.364152036897882e-07, "loss": 0.2968, "num_tokens": 15125418.0, "reward": 0.7392578125, "reward_std": 0.013927556574344635, "rewards//mean": 0.7392578125, "rewards//std": 0.03685477748513222, "step": 1750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3502, "grad_norm": 14.399723052978516, "kl": 6.29911071062088, "learning_rate": 7.361355373863413e-07, "loss": 0.252, "num_tokens": 15133986.0, "reward": 0.73876953125, "reward_std": 0.011047488078474998, "rewards//mean": 0.73876953125, "rewards//std": 0.03426307067275047, "step": 1751 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3504, "grad_norm": 17.051923751831055, "kl": 2.556887049227953, "learning_rate": 7.358557759675284e-07, "loss": 0.1023, "num_tokens": 15142714.0, "reward": 0.7666015625, "reward_std": 0.01128496415913105, "rewards//mean": 0.7666015625, "rewards//std": 0.02542041800916195, "step": 1752 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3506, "grad_norm": 11.291234016418457, "kl": 4.278929254040122, "learning_rate": 7.35575919546037e-07, "loss": 0.1712, "num_tokens": 15151538.0, "reward": 0.7235107421875, "reward_std": 0.010619303211569786, "rewards//mean": 0.7235107421875, "rewards//std": 0.03514914587140083, "step": 1753 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3508, "grad_norm": 5.1102423667907715, "kl": 2.5491420943289995, "learning_rate": 7.352959682345935e-07, "loss": 0.102, "num_tokens": 15160170.0, "reward": 0.7515869140625, "reward_std": 0.00870327465236187, "rewards//mean": 0.7515869140625, "rewards//std": 0.020795151591300964, "step": 1754 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.351, "grad_norm": 8.712926864624023, "kl": 3.3026189766824245, "learning_rate": 7.350159221459621e-07, "loss": 0.1321, "num_tokens": 15168810.0, "reward": 0.7755126953125, "reward_std": 0.009872986003756523, "rewards//mean": 0.7755126953125, "rewards//std": 0.022988514974713326, "step": 1755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3512, "grad_norm": 5.928198337554932, "kl": 4.434307049959898, "learning_rate": 7.347357813929454e-07, "loss": 0.1774, "num_tokens": 15177346.0, "reward": 0.75616455078125, "reward_std": 0.010966308414936066, "rewards//mean": 0.75616455078125, "rewards//std": 0.02040819823741913, "step": 1756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3514, "grad_norm": 9.019213676452637, "kl": 3.6429110188037157, "learning_rate": 7.344555460883839e-07, "loss": 0.1457, "num_tokens": 15185914.0, "reward": 0.76507568359375, "reward_std": 0.011363131925463676, "rewards//mean": 0.76507568359375, "rewards//std": 0.019745443016290665, "step": 1757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3516, "grad_norm": 3.725701332092285, "kl": 2.5000802017748356, "learning_rate": 7.341752163451567e-07, "loss": 0.1, "num_tokens": 15194562.0, "reward": 0.73394775390625, "reward_std": 0.0064627304673194885, "rewards//mean": 0.73394775390625, "rewards//std": 0.027233846485614777, "step": 1758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3518, "grad_norm": 7.178076267242432, "kl": 2.457119194790721, "learning_rate": 7.338947922761802e-07, "loss": 0.0983, "num_tokens": 15203194.0, "reward": 0.75592041015625, "reward_std": 0.00877198576927185, "rewards//mean": 0.75592041015625, "rewards//std": 0.024230070412158966, "step": 1759 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.90625, "epoch": 0.352, "grad_norm": 5.549074649810791, "kl": 3.2554349713027477, "learning_rate": 7.336142739944093e-07, "loss": 0.1287, "num_tokens": 15211908.0, "reward": 0.7672119140625, "reward_std": 0.011690272949635983, "rewards//mean": 0.7672119140625, "rewards//std": 0.029046498239040375, "step": 1760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3522, "grad_norm": 10.549675941467285, "kl": 1.002486851066351, "learning_rate": 7.333336616128369e-07, "loss": 0.0401, "num_tokens": 15220524.0, "reward": 0.76007080078125, "reward_std": 0.003659485839307308, "rewards//mean": 0.76007080078125, "rewards//std": 0.01995212212204933, "step": 1761 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3524, "grad_norm": 10.45427417755127, "kl": 2.251881780102849, "learning_rate": 7.330529552444932e-07, "loss": 0.0901, "num_tokens": 15229196.0, "reward": 0.74853515625, "reward_std": 0.006151488516479731, "rewards//mean": 0.74853515625, "rewards//std": 0.022654540836811066, "step": 1762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3526, "grad_norm": 15.576757431030273, "kl": 2.9452833347022533, "learning_rate": 7.327721550024475e-07, "loss": 0.1178, "num_tokens": 15237892.0, "reward": 0.74267578125, "reward_std": 0.010885363444685936, "rewards//mean": 0.74267578125, "rewards//std": 0.026707449927926064, "step": 1763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3528, "grad_norm": 5.1858415603637695, "kl": 1.606558632105589, "learning_rate": 7.324912609998053e-07, "loss": 0.0643, "num_tokens": 15246564.0, "reward": 0.733154296875, "reward_std": 0.004549759905785322, "rewards//mean": 0.733154296875, "rewards//std": 0.025189509615302086, "step": 1764 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.353, "grad_norm": 9.590002059936523, "kl": 4.183308666571975, "learning_rate": 7.322102733497109e-07, "loss": 0.1673, "num_tokens": 15255244.0, "reward": 0.770263671875, "reward_std": 0.00666767219081521, "rewards//mean": 0.770263671875, "rewards//std": 0.031938981264829636, "step": 1765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3532, "grad_norm": 10.281412124633789, "kl": 3.2724026404321194, "learning_rate": 7.319291921653463e-07, "loss": 0.1309, "num_tokens": 15263876.0, "reward": 0.73870849609375, "reward_std": 0.004603174515068531, "rewards//mean": 0.73870849609375, "rewards//std": 0.022968998178839684, "step": 1766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3534, "grad_norm": 8.301042556762695, "kl": 5.278777930885553, "learning_rate": 7.316480175599308e-07, "loss": 0.2112, "num_tokens": 15272540.0, "reward": 0.78741455078125, "reward_std": 0.015570227056741714, "rewards//mean": 0.78741455078125, "rewards//std": 0.02587188594043255, "step": 1767 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3536, "grad_norm": 9.332090377807617, "kl": 2.2536631040275097, "learning_rate": 7.313667496467215e-07, "loss": 0.0901, "num_tokens": 15281124.0, "reward": 0.74981689453125, "reward_std": 0.005827593617141247, "rewards//mean": 0.74981689453125, "rewards//std": 0.014506562612950802, "step": 1768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 126.765625, "epoch": 0.3538, "grad_norm": 3.0023088455200195, "kl": 2.0433195158839226, "learning_rate": 7.310853885390132e-07, "loss": 0.0576, "num_tokens": 15289653.0, "reward": 0.78173828125, "reward_std": 0.005920770578086376, "rewards//mean": 0.78173828125, "rewards//std": 0.01923132874071598, "step": 1769 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.354, "grad_norm": 7.975004196166992, "kl": 5.059855833649635, "learning_rate": 7.308039343501379e-07, "loss": 0.2024, "num_tokens": 15298357.0, "reward": 0.72723388671875, "reward_std": 0.009100313298404217, "rewards//mean": 0.72723388671875, "rewards//std": 0.028608225286006927, "step": 1770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3542, "grad_norm": 10.475563049316406, "kl": 2.0413386076688766, "learning_rate": 7.305223871934656e-07, "loss": 0.0817, "num_tokens": 15307029.0, "reward": 0.7882080078125, "reward_std": 0.0049152253195643425, "rewards//mean": 0.7882080078125, "rewards//std": 0.01619539223611355, "step": 1771 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3544, "grad_norm": 16.09853744506836, "kl": 4.93830094113946, "learning_rate": 7.302407471824033e-07, "loss": 0.1975, "num_tokens": 15315717.0, "reward": 0.7646484375, "reward_std": 0.007313862442970276, "rewards//mean": 0.7646484375, "rewards//std": 0.03219672292470932, "step": 1772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3546, "grad_norm": 3.7485849857330322, "kl": 2.3264099545776844, "learning_rate": 7.299590144303954e-07, "loss": 0.0931, "num_tokens": 15324325.0, "reward": 0.76885986328125, "reward_std": 0.007083921227604151, "rewards//mean": 0.76885986328125, "rewards//std": 0.02356688305735588, "step": 1773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3548, "grad_norm": 12.827655792236328, "kl": 3.729407176375389, "learning_rate": 7.296771890509242e-07, "loss": 0.1492, "num_tokens": 15332901.0, "reward": 0.76171875, "reward_std": 0.004367573652416468, "rewards//mean": 0.76171875, "rewards//std": 0.025581879541277885, "step": 1774 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.355, "grad_norm": 10.825053215026855, "kl": 8.491058219224215, "learning_rate": 7.293952711575086e-07, "loss": 0.3396, "num_tokens": 15341501.0, "reward": 0.74847412109375, "reward_std": 0.01544426754117012, "rewards//mean": 0.74847412109375, "rewards//std": 0.03469241410493851, "step": 1775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3552, "grad_norm": 12.783354759216309, "kl": 3.17761186696589, "learning_rate": 7.291132608637052e-07, "loss": 0.1271, "num_tokens": 15350125.0, "reward": 0.78692626953125, "reward_std": 0.009350663051009178, "rewards//mean": 0.78692626953125, "rewards//std": 0.02011081762611866, "step": 1776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3554, "grad_norm": 8.209219932556152, "kl": 3.952581450343132, "learning_rate": 7.288311582831077e-07, "loss": 0.1581, "num_tokens": 15358829.0, "reward": 0.77197265625, "reward_std": 0.006296072620898485, "rewards//mean": 0.77197265625, "rewards//std": 0.02374039590358734, "step": 1777 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3556, "grad_norm": 19.983972549438477, "kl": 9.637716632336378, "learning_rate": 7.285489635293471e-07, "loss": 0.3855, "num_tokens": 15367453.0, "reward": 0.74346923828125, "reward_std": 0.018547261133790016, "rewards//mean": 0.74346923828125, "rewards//std": 0.02717096358537674, "step": 1778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3558, "grad_norm": 22.738313674926758, "kl": 7.230593521147966, "learning_rate": 7.282666767160912e-07, "loss": 0.2892, "num_tokens": 15376125.0, "reward": 0.7393798828125, "reward_std": 0.008969064801931381, "rewards//mean": 0.7393798828125, "rewards//std": 0.02510855160653591, "step": 1779 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.356, "grad_norm": 25.0104923248291, "kl": 5.67492107860744, "learning_rate": 7.279842979570453e-07, "loss": 0.227, "num_tokens": 15384829.0, "reward": 0.7806396484375, "reward_std": 0.004135373514145613, "rewards//mean": 0.7806396484375, "rewards//std": 0.018125487491488457, "step": 1780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3562, "grad_norm": 6.182112216949463, "kl": 3.115319015458226, "learning_rate": 7.277018273659516e-07, "loss": 0.1246, "num_tokens": 15393437.0, "reward": 0.78009033203125, "reward_std": 0.005007099360227585, "rewards//mean": 0.78009033203125, "rewards//std": 0.027389591559767723, "step": 1781 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3564, "grad_norm": 27.994529724121094, "kl": 5.9446770660579205, "learning_rate": 7.274192650565889e-07, "loss": 0.2378, "num_tokens": 15402053.0, "reward": 0.7587890625, "reward_std": 0.011876787059009075, "rewards//mean": 0.7587890625, "rewards//std": 0.02433985471725464, "step": 1782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3566, "grad_norm": 28.30736541748047, "kl": 9.795169910416007, "learning_rate": 7.271366111427734e-07, "loss": 0.3918, "num_tokens": 15410757.0, "reward": 0.7508544921875, "reward_std": 0.01354462280869484, "rewards//mean": 0.7508544921875, "rewards//std": 0.03079301305115223, "step": 1783 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3568, "grad_norm": 11.573566436767578, "kl": 2.8760644625872374, "learning_rate": 7.26853865738358e-07, "loss": 0.115, "num_tokens": 15419405.0, "reward": 0.751953125, "reward_std": 0.004005537834018469, "rewards//mean": 0.751953125, "rewards//std": 0.02214088849723339, "step": 1784 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.357, "grad_norm": 12.344344139099121, "kl": 5.548849571496248, "learning_rate": 7.265710289572328e-07, "loss": 0.222, "num_tokens": 15427997.0, "reward": 0.77801513671875, "reward_std": 0.016352981328964233, "rewards//mean": 0.77801513671875, "rewards//std": 0.03220841661095619, "step": 1785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3572, "grad_norm": 10.783181190490723, "kl": 1.9642281122505665, "learning_rate": 7.262881009133241e-07, "loss": 0.0786, "num_tokens": 15436629.0, "reward": 0.753173828125, "reward_std": 0.0023689866065979004, "rewards//mean": 0.753173828125, "rewards//std": 0.018803080543875694, "step": 1786 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3574, "grad_norm": 5.746514797210693, "kl": 3.0543887242674828, "learning_rate": 7.260050817205955e-07, "loss": 0.1222, "num_tokens": 15445237.0, "reward": 0.76434326171875, "reward_std": 0.01005857065320015, "rewards//mean": 0.76434326171875, "rewards//std": 0.02969251573085785, "step": 1787 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3576, "grad_norm": 9.588447570800781, "kl": 6.2093190886080265, "learning_rate": 7.25721971493047e-07, "loss": 0.2484, "num_tokens": 15453805.0, "reward": 0.75616455078125, "reward_std": 0.010142859071493149, "rewards//mean": 0.75616455078125, "rewards//std": 0.028669537976384163, "step": 1788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3578, "grad_norm": 9.859051704406738, "kl": 5.229010626673698, "learning_rate": 7.254387703447153e-07, "loss": 0.2092, "num_tokens": 15462397.0, "reward": 0.76214599609375, "reward_std": 0.01515856571495533, "rewards//mean": 0.76214599609375, "rewards//std": 0.030986279249191284, "step": 1789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.358, "grad_norm": 8.220314025878906, "kl": 4.179319351911545, "learning_rate": 7.25155478389674e-07, "loss": 0.1672, "num_tokens": 15471029.0, "reward": 0.76385498046875, "reward_std": 0.01024538278579712, "rewards//mean": 0.76385498046875, "rewards//std": 0.024119237437844276, "step": 1790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3582, "grad_norm": 4.266323089599609, "kl": 4.320960188284516, "learning_rate": 7.248720957420329e-07, "loss": 0.1728, "num_tokens": 15479709.0, "reward": 0.77239990234375, "reward_std": 0.010997525416314602, "rewards//mean": 0.77239990234375, "rewards//std": 0.026256943121552467, "step": 1791 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3584, "grad_norm": 7.368237018585205, "kl": 2.9156788084656, "learning_rate": 7.245886225159386e-07, "loss": 0.1166, "num_tokens": 15488325.0, "reward": 0.717041015625, "reward_std": 0.009482947178184986, "rewards//mean": 0.717041015625, "rewards//std": 0.045399196445941925, "step": 1792 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3586, "grad_norm": 14.691852569580078, "kl": 1.7800093349069357, "learning_rate": 7.243050588255737e-07, "loss": 0.0712, "num_tokens": 15496949.0, "reward": 0.7899169921875, "reward_std": 0.011507879011332989, "rewards//mean": 0.7899169921875, "rewards//std": 0.028069043532013893, "step": 1793 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3588, "grad_norm": 15.345220565795898, "kl": 5.1895691603422165, "learning_rate": 7.240214047851581e-07, "loss": 0.2076, "num_tokens": 15505589.0, "reward": 0.7374267578125, "reward_std": 0.0077485572546720505, "rewards//mean": 0.7374267578125, "rewards//std": 0.024394214153289795, "step": 1794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.359, "grad_norm": 20.31637954711914, "kl": 6.3183458019047976, "learning_rate": 7.237376605089476e-07, "loss": 0.2527, "num_tokens": 15514253.0, "reward": 0.776123046875, "reward_std": 0.010714283213019371, "rewards//mean": 0.776123046875, "rewards//std": 0.025712909176945686, "step": 1795 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3592, "grad_norm": 7.15415620803833, "kl": 4.081360448151827, "learning_rate": 7.234538261112341e-07, "loss": 0.1633, "num_tokens": 15522845.0, "reward": 0.7509765625, "reward_std": 0.010711642913520336, "rewards//mean": 0.7509765625, "rewards//std": 0.02026761695742607, "step": 1796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3594, "grad_norm": 5.400107383728027, "kl": 3.705445107072592, "learning_rate": 7.23169901706346e-07, "loss": 0.1482, "num_tokens": 15531453.0, "reward": 0.74517822265625, "reward_std": 0.009501457214355469, "rewards//mean": 0.74517822265625, "rewards//std": 0.03087371401488781, "step": 1797 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3596, "grad_norm": 10.734845161437988, "kl": 5.157490076497197, "learning_rate": 7.228858874086484e-07, "loss": 0.2063, "num_tokens": 15540037.0, "reward": 0.7705078125, "reward_std": 0.010244790464639664, "rewards//mean": 0.7705078125, "rewards//std": 0.01782611384987831, "step": 1798 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3598, "grad_norm": 10.73138427734375, "kl": 2.7920976169407368, "learning_rate": 7.226017833325419e-07, "loss": 0.1117, "num_tokens": 15548613.0, "reward": 0.74395751953125, "reward_std": 0.007455380167812109, "rewards//mean": 0.74395751953125, "rewards//std": 0.02261633798480034, "step": 1799 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.36, "grad_norm": 10.345914840698242, "kl": 3.87006332911551, "learning_rate": 7.223175895924637e-07, "loss": 0.1548, "num_tokens": 15557189.0, "reward": 0.79193115234375, "reward_std": 0.008266190066933632, "rewards//mean": 0.79193115234375, "rewards//std": 0.022000884637236595, "step": 1800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3602, "grad_norm": 10.194170951843262, "kl": 5.555856931954622, "learning_rate": 7.220333063028871e-07, "loss": 0.2222, "num_tokens": 15565741.0, "reward": 0.7454833984375, "reward_std": 0.0249344352632761, "rewards//mean": 0.7454833984375, "rewards//std": 0.03213813528418541, "step": 1801 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3604, "grad_norm": 11.115693092346191, "kl": 4.155972799286246, "learning_rate": 7.217489335783211e-07, "loss": 0.1662, "num_tokens": 15574533.0, "reward": 0.79888916015625, "reward_std": 0.013462554663419724, "rewards//mean": 0.79888916015625, "rewards//std": 0.03218161314725876, "step": 1802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3606, "grad_norm": 17.077611923217773, "kl": 2.78132888302207, "learning_rate": 7.214644715333114e-07, "loss": 0.1113, "num_tokens": 15583125.0, "reward": 0.75701904296875, "reward_std": 0.013729956932365894, "rewards//mean": 0.75701904296875, "rewards//std": 0.025884754955768585, "step": 1803 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3608, "grad_norm": 12.110036849975586, "kl": 4.024367205798626, "learning_rate": 7.211799202824388e-07, "loss": 0.161, "num_tokens": 15591757.0, "reward": 0.74847412109375, "reward_std": 0.011821866966784, "rewards//mean": 0.74847412109375, "rewards//std": 0.028208041563630104, "step": 1804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.361, "grad_norm": 12.529596328735352, "kl": 2.554512431845069, "learning_rate": 7.20895279940321e-07, "loss": 0.1022, "num_tokens": 15600341.0, "reward": 0.748046875, "reward_std": 0.006430302746593952, "rewards//mean": 0.748046875, "rewards//std": 0.02427009865641594, "step": 1805 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3612, "grad_norm": 18.0447998046875, "kl": 2.894049622118473, "learning_rate": 7.206105506216106e-07, "loss": 0.1158, "num_tokens": 15608949.0, "reward": 0.7611083984375, "reward_std": 0.01057471800595522, "rewards//mean": 0.7611083984375, "rewards//std": 0.026182828471064568, "step": 1806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3614, "grad_norm": 4.179669380187988, "kl": 3.3387136291712523, "learning_rate": 7.203257324409971e-07, "loss": 0.1335, "num_tokens": 15617549.0, "reward": 0.76190185546875, "reward_std": 0.009570148773491383, "rewards//mean": 0.76190185546875, "rewards//std": 0.03266298025846481, "step": 1807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3616, "grad_norm": 7.232208251953125, "kl": 3.486246306449175, "learning_rate": 7.200408255132045e-07, "loss": 0.1394, "num_tokens": 15626093.0, "reward": 0.7607421875, "reward_std": 0.011610465124249458, "rewards//mean": 0.7607421875, "rewards//std": 0.02033919282257557, "step": 1808 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3618, "grad_norm": 16.169591903686523, "kl": 1.3782273046672344, "learning_rate": 7.19755829952994e-07, "loss": 0.0551, "num_tokens": 15634789.0, "reward": 0.793212890625, "reward_std": 0.006641528103500605, "rewards//mean": 0.793212890625, "rewards//std": 0.015975650399923325, "step": 1809 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.362, "grad_norm": 12.412141799926758, "kl": 3.3542172610759735, "learning_rate": 7.194707458751615e-07, "loss": 0.1342, "num_tokens": 15643493.0, "reward": 0.7469482421875, "reward_std": 0.007622177712619305, "rewards//mean": 0.7469482421875, "rewards//std": 0.027982624247670174, "step": 1810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3622, "grad_norm": 11.963812828063965, "kl": 4.771061383187771, "learning_rate": 7.191855733945386e-07, "loss": 0.1908, "num_tokens": 15652381.0, "reward": 0.7396240234375, "reward_std": 0.01201414130628109, "rewards//mean": 0.7396240234375, "rewards//std": 0.03593115508556366, "step": 1811 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3624, "grad_norm": 10.7211332321167, "kl": 1.8340128250420094, "learning_rate": 7.189003126259931e-07, "loss": 0.0734, "num_tokens": 15661125.0, "reward": 0.7752685546875, "reward_std": 0.0092721376568079, "rewards//mean": 0.7752685546875, "rewards//std": 0.02249864861369133, "step": 1812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3626, "grad_norm": 14.77794361114502, "kl": 1.1398207396268845, "learning_rate": 7.186149636844279e-07, "loss": 0.0456, "num_tokens": 15669845.0, "reward": 0.778076171875, "reward_std": 0.003906923346221447, "rewards//mean": 0.778076171875, "rewards//std": 0.015309864655137062, "step": 1813 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3628, "grad_norm": 17.214759826660156, "kl": 2.4941138923168182, "learning_rate": 7.183295266847814e-07, "loss": 0.0998, "num_tokens": 15678469.0, "reward": 0.7567138671875, "reward_std": 0.007382056210190058, "rewards//mean": 0.7567138671875, "rewards//std": 0.02023734524846077, "step": 1814 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.363, "grad_norm": 8.335517883300781, "kl": 2.8564509116113186, "learning_rate": 7.180440017420276e-07, "loss": 0.1143, "num_tokens": 15687037.0, "reward": 0.747802734375, "reward_std": 0.005393965169787407, "rewards//mean": 0.747802734375, "rewards//std": 0.027320489287376404, "step": 1815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3632, "grad_norm": 8.306028366088867, "kl": 3.8441889118403196, "learning_rate": 7.177583889711762e-07, "loss": 0.1538, "num_tokens": 15695685.0, "reward": 0.77984619140625, "reward_std": 0.00937097892165184, "rewards//mean": 0.77984619140625, "rewards//std": 0.025385782122612, "step": 1816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3634, "grad_norm": 15.702668190002441, "kl": 5.227973237633705, "learning_rate": 7.174726884872715e-07, "loss": 0.2091, "num_tokens": 15704253.0, "reward": 0.7515869140625, "reward_std": 0.009687444195151329, "rewards//mean": 0.7515869140625, "rewards//std": 0.029876815155148506, "step": 1817 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3636, "grad_norm": 18.41916275024414, "kl": 6.280253957957029, "learning_rate": 7.17186900405394e-07, "loss": 0.2512, "num_tokens": 15712845.0, "reward": 0.74658203125, "reward_std": 0.017452437430620193, "rewards//mean": 0.74658203125, "rewards//std": 0.0323505662381649, "step": 1818 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3638, "grad_norm": 28.84602165222168, "kl": 7.063274849206209, "learning_rate": 7.169010248406588e-07, "loss": 0.2825, "num_tokens": 15721461.0, "reward": 0.75006103515625, "reward_std": 0.011782647110521793, "rewards//mean": 0.75006103515625, "rewards//std": 0.028768105432391167, "step": 1819 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.364, "grad_norm": 6.0525007247924805, "kl": 3.8506972566246986, "learning_rate": 7.16615061908217e-07, "loss": 0.154, "num_tokens": 15730149.0, "reward": 0.7410888671875, "reward_std": 0.007551171816885471, "rewards//mean": 0.7410888671875, "rewards//std": 0.03739281743764877, "step": 1820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3642, "grad_norm": 7.779174327850342, "kl": 2.575003158301115, "learning_rate": 7.163290117232541e-07, "loss": 0.103, "num_tokens": 15738717.0, "reward": 0.75946044921875, "reward_std": 0.006207785569131374, "rewards//mean": 0.75946044921875, "rewards//std": 0.020085208117961884, "step": 1821 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3644, "grad_norm": 2.5102193355560303, "kl": 1.606106111779809, "learning_rate": 7.160428744009912e-07, "loss": 0.0642, "num_tokens": 15747325.0, "reward": 0.73175048828125, "reward_std": 0.004206686746329069, "rewards//mean": 0.73175048828125, "rewards//std": 0.022359192371368408, "step": 1822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3646, "grad_norm": 7.623538017272949, "kl": 4.200133021920919, "learning_rate": 7.157566500566842e-07, "loss": 0.168, "num_tokens": 15755973.0, "reward": 0.78436279296875, "reward_std": 0.011930635198950768, "rewards//mean": 0.78436279296875, "rewards//std": 0.023490967229008675, "step": 1823 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3648, "grad_norm": 6.204391002655029, "kl": 2.8439120296388865, "learning_rate": 7.154703388056244e-07, "loss": 0.1138, "num_tokens": 15764549.0, "reward": 0.76434326171875, "reward_std": 0.008812109008431435, "rewards//mean": 0.76434326171875, "rewards//std": 0.01907394640147686, "step": 1824 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.365, "grad_norm": 11.921187400817871, "kl": 1.9189935978502035, "learning_rate": 7.15183940763138e-07, "loss": 0.0768, "num_tokens": 15773165.0, "reward": 0.75701904296875, "reward_std": 0.005657212808728218, "rewards//mean": 0.75701904296875, "rewards//std": 0.0242119450122118, "step": 1825 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3652, "grad_norm": 12.116460800170898, "kl": 2.1174923833459616, "learning_rate": 7.148974560445858e-07, "loss": 0.0847, "num_tokens": 15781861.0, "reward": 0.822265625, "reward_std": 0.011779746040701866, "rewards//mean": 0.822265625, "rewards//std": 0.023540614172816277, "step": 1826 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3654, "grad_norm": 20.339641571044922, "kl": 6.685268387198448, "learning_rate": 7.146108847653641e-07, "loss": 0.2674, "num_tokens": 15790461.0, "reward": 0.74822998046875, "reward_std": 0.010201862081885338, "rewards//mean": 0.74822998046875, "rewards//std": 0.0224874597042799, "step": 1827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3656, "grad_norm": 4.9110612869262695, "kl": 2.461530778557062, "learning_rate": 7.143242270409037e-07, "loss": 0.0985, "num_tokens": 15799221.0, "reward": 0.75177001953125, "reward_std": 0.004146665334701538, "rewards//mean": 0.75177001953125, "rewards//std": 0.030233096331357956, "step": 1828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3658, "grad_norm": 5.22251033782959, "kl": 1.3586306106299162, "learning_rate": 7.140374829866702e-07, "loss": 0.0543, "num_tokens": 15807805.0, "reward": 0.748046875, "reward_std": 0.00377071974799037, "rewards//mean": 0.748046875, "rewards//std": 0.018727252259850502, "step": 1829 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.366, "grad_norm": 4.655337810516357, "kl": 4.365455897524953, "learning_rate": 7.137506527181643e-07, "loss": 0.1746, "num_tokens": 15816461.0, "reward": 0.74749755859375, "reward_std": 0.010998677462339401, "rewards//mean": 0.74749755859375, "rewards//std": 0.029333990067243576, "step": 1830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3662, "grad_norm": 11.291488647460938, "kl": 4.101958701387048, "learning_rate": 7.134637363509209e-07, "loss": 0.1641, "num_tokens": 15825077.0, "reward": 0.7510986328125, "reward_std": 0.00990404561161995, "rewards//mean": 0.7510986328125, "rewards//std": 0.026887930929660797, "step": 1831 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3664, "grad_norm": 4.310554027557373, "kl": 1.6261647623032331, "learning_rate": 7.131767340005101e-07, "loss": 0.065, "num_tokens": 15833709.0, "reward": 0.7259521484375, "reward_std": 0.004137130919843912, "rewards//mean": 0.7259521484375, "rewards//std": 0.025546055287122726, "step": 1832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3666, "grad_norm": 6.229635238647461, "kl": 3.4514841958880424, "learning_rate": 7.128896457825363e-07, "loss": 0.1381, "num_tokens": 15842405.0, "reward": 0.7718505859375, "reward_std": 0.00895756483078003, "rewards//mean": 0.7718505859375, "rewards//std": 0.026399319991469383, "step": 1833 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3668, "grad_norm": 11.113919258117676, "kl": 3.233485661447048, "learning_rate": 7.126024718126387e-07, "loss": 0.1293, "num_tokens": 15851117.0, "reward": 0.7513427734375, "reward_std": 0.009316753596067429, "rewards//mean": 0.7513427734375, "rewards//std": 0.024931885302066803, "step": 1834 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.367, "grad_norm": 6.431647300720215, "kl": 3.877834590151906, "learning_rate": 7.123152122064908e-07, "loss": 0.1551, "num_tokens": 15859685.0, "reward": 0.7598876953125, "reward_std": 0.008115262724459171, "rewards//mean": 0.7598876953125, "rewards//std": 0.02669356018304825, "step": 1835 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3672, "grad_norm": 5.051802635192871, "kl": 3.4883563816547394, "learning_rate": 7.120278670798009e-07, "loss": 0.1395, "num_tokens": 15868293.0, "reward": 0.7637939453125, "reward_std": 0.009110360406339169, "rewards//mean": 0.7637939453125, "rewards//std": 0.031970009207725525, "step": 1836 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 126.5625, "epoch": 0.3674, "grad_norm": 3.741506576538086, "kl": 2.80803806707263, "learning_rate": 7.117404365483115e-07, "loss": 0.1287, "num_tokens": 15876961.0, "reward": 0.79388427734375, "reward_std": 0.0073487600311636925, "rewards//mean": 0.79388427734375, "rewards//std": 0.02423756755888462, "step": 1837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3676, "grad_norm": 8.29547119140625, "kl": 2.961475620046258, "learning_rate": 7.114529207277995e-07, "loss": 0.1185, "num_tokens": 15885633.0, "reward": 0.76275634765625, "reward_std": 0.008384748362004757, "rewards//mean": 0.76275634765625, "rewards//std": 0.0255747027695179, "step": 1838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3678, "grad_norm": 3.3730926513671875, "kl": 3.096415415406227, "learning_rate": 7.111653197340764e-07, "loss": 0.1239, "num_tokens": 15894241.0, "reward": 0.73614501953125, "reward_std": 0.0058305514976382256, "rewards//mean": 0.73614501953125, "rewards//std": 0.031473420560359955, "step": 1839 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.368, "grad_norm": 5.94202184677124, "kl": 2.1518924087285995, "learning_rate": 7.108776336829876e-07, "loss": 0.0861, "num_tokens": 15902793.0, "reward": 0.76837158203125, "reward_std": 0.0043298485688865185, "rewards//mean": 0.76837158203125, "rewards//std": 0.017916640266776085, "step": 1840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3682, "grad_norm": 14.551260948181152, "kl": 4.650878993794322, "learning_rate": 7.105898626904134e-07, "loss": 0.186, "num_tokens": 15911425.0, "reward": 0.797119140625, "reward_std": 0.009744038805365562, "rewards//mean": 0.797119140625, "rewards//std": 0.0221941526979208, "step": 1841 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3684, "grad_norm": 3.400298833847046, "kl": 1.1997675318270922, "learning_rate": 7.103020068722674e-07, "loss": 0.048, "num_tokens": 15920129.0, "reward": 0.7310791015625, "reward_std": 0.00397300161421299, "rewards//mean": 0.7310791015625, "rewards//std": 0.028196033090353012, "step": 1842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3686, "grad_norm": 5.189095497131348, "kl": 2.531966695562005, "learning_rate": 7.100140663444984e-07, "loss": 0.1013, "num_tokens": 15928817.0, "reward": 0.7432861328125, "reward_std": 0.005302074830979109, "rewards//mean": 0.7432861328125, "rewards//std": 0.03379307687282562, "step": 1843 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3688, "grad_norm": 5.2451958656311035, "kl": 0.819163890555501, "learning_rate": 7.097260412230885e-07, "loss": 0.0328, "num_tokens": 15937457.0, "reward": 0.7691650390625, "reward_std": 0.0031269120518118143, "rewards//mean": 0.7691650390625, "rewards//std": 0.02102680318057537, "step": 1844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.369, "grad_norm": 1.9537146091461182, "kl": 2.4095728807151318, "learning_rate": 7.094379316240544e-07, "loss": 0.0964, "num_tokens": 15946089.0, "reward": 0.7491455078125, "reward_std": 0.007642756681889296, "rewards//mean": 0.7491455078125, "rewards//std": 0.028960904106497765, "step": 1845 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3692, "grad_norm": 9.979609489440918, "kl": 1.6186884567141533, "learning_rate": 7.091497376634463e-07, "loss": 0.0647, "num_tokens": 15954761.0, "reward": 0.7952880859375, "reward_std": 0.005419217981398106, "rewards//mean": 0.7952880859375, "rewards//std": 0.023681342601776123, "step": 1846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3694, "grad_norm": 12.618873596191406, "kl": 3.9344957154244184, "learning_rate": 7.088614594573491e-07, "loss": 0.1574, "num_tokens": 15963369.0, "reward": 0.76629638671875, "reward_std": 0.01030784659087658, "rewards//mean": 0.76629638671875, "rewards//std": 0.029120083898305893, "step": 1847 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3696, "grad_norm": 9.172502517700195, "kl": 2.6048205718398094, "learning_rate": 7.085730971218809e-07, "loss": 0.1042, "num_tokens": 15971985.0, "reward": 0.77740478515625, "reward_std": 0.006511168088763952, "rewards//mean": 0.77740478515625, "rewards//std": 0.019960464909672737, "step": 1848 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3698, "grad_norm": 7.123248100280762, "kl": 1.7873948123306036, "learning_rate": 7.082846507731941e-07, "loss": 0.0715, "num_tokens": 15980625.0, "reward": 0.73388671875, "reward_std": 0.005009832326322794, "rewards//mean": 0.73388671875, "rewards//std": 0.022771839052438736, "step": 1849 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.37, "grad_norm": 14.527019500732422, "kl": 3.1183022148907185, "learning_rate": 7.079961205274748e-07, "loss": 0.1247, "num_tokens": 15989297.0, "reward": 0.78668212890625, "reward_std": 0.006082155276089907, "rewards//mean": 0.78668212890625, "rewards//std": 0.017310744151473045, "step": 1850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3702, "grad_norm": 6.804238796234131, "kl": 4.943616297096014, "learning_rate": 7.077075065009433e-07, "loss": 0.1977, "num_tokens": 15997929.0, "reward": 0.79132080078125, "reward_std": 0.012570510618388653, "rewards//mean": 0.79132080078125, "rewards//std": 0.026836862787604332, "step": 1851 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3704, "grad_norm": 5.623232364654541, "kl": 1.2311439421027899, "learning_rate": 7.074188088098527e-07, "loss": 0.0492, "num_tokens": 16006505.0, "reward": 0.78045654296875, "reward_std": 0.005158598534762859, "rewards//mean": 0.78045654296875, "rewards//std": 0.025297973304986954, "step": 1852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3706, "grad_norm": 3.320368766784668, "kl": 3.7772607430815697, "learning_rate": 7.071300275704909e-07, "loss": 0.1511, "num_tokens": 16015257.0, "reward": 0.7457275390625, "reward_std": 0.013418788090348244, "rewards//mean": 0.7457275390625, "rewards//std": 0.02494645304977894, "step": 1853 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3708, "grad_norm": 7.783235549926758, "kl": 3.596936283633113, "learning_rate": 7.068411628991787e-07, "loss": 0.1439, "num_tokens": 16023985.0, "reward": 0.77288818359375, "reward_std": 0.0066768997348845005, "rewards//mean": 0.77288818359375, "rewards//std": 0.043218113481998444, "step": 1854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.371, "grad_norm": 5.301913738250732, "kl": 0.6076066605746746, "learning_rate": 7.065522149122709e-07, "loss": 0.0243, "num_tokens": 16032585.0, "reward": 0.73272705078125, "reward_std": 0.0008328249678015709, "rewards//mean": 0.73272705078125, "rewards//std": 0.022518403828144073, "step": 1855 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3712, "grad_norm": 2.3196868896484375, "kl": 0.8560703378170729, "learning_rate": 7.062631837261556e-07, "loss": 0.0342, "num_tokens": 16041097.0, "reward": 0.79913330078125, "reward_std": 0.002766229910776019, "rewards//mean": 0.79913330078125, "rewards//std": 0.016874125227332115, "step": 1856 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3714, "grad_norm": 3.6160199642181396, "kl": 2.6725729685276747, "learning_rate": 7.059740694572545e-07, "loss": 0.1069, "num_tokens": 16049721.0, "reward": 0.74658203125, "reward_std": 0.005142111796885729, "rewards//mean": 0.74658203125, "rewards//std": 0.020640617236495018, "step": 1857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3716, "grad_norm": 5.1055006980896, "kl": 3.198473073542118, "learning_rate": 7.056848722220228e-07, "loss": 0.1279, "num_tokens": 16058401.0, "reward": 0.7724609375, "reward_std": 0.011910544708371162, "rewards//mean": 0.7724609375, "rewards//std": 0.023765888065099716, "step": 1858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3718, "grad_norm": 11.609858512878418, "kl": 1.2559426296502352, "learning_rate": 7.053955921369493e-07, "loss": 0.0502, "num_tokens": 16067001.0, "reward": 0.77606201171875, "reward_std": 0.008136093616485596, "rewards//mean": 0.77606201171875, "rewards//std": 0.019930865615606308, "step": 1859 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.372, "grad_norm": 4.9913811683654785, "kl": 3.288194267079234, "learning_rate": 7.051062293185559e-07, "loss": 0.1315, "num_tokens": 16075641.0, "reward": 0.75323486328125, "reward_std": 0.008526476100087166, "rewards//mean": 0.75323486328125, "rewards//std": 0.026789437979459763, "step": 1860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3722, "grad_norm": 4.668308258056641, "kl": 0.9770767856389284, "learning_rate": 7.048167838833976e-07, "loss": 0.0391, "num_tokens": 16084201.0, "reward": 0.75799560546875, "reward_std": 0.0024019088596105576, "rewards//mean": 0.75799560546875, "rewards//std": 0.02114981971681118, "step": 1861 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3724, "grad_norm": 7.400339603424072, "kl": 2.7208936866372824, "learning_rate": 7.045272559480635e-07, "loss": 0.1088, "num_tokens": 16092793.0, "reward": 0.75408935546875, "reward_std": 0.007091997656971216, "rewards//mean": 0.75408935546875, "rewards//std": 0.02746685966849327, "step": 1862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3726, "grad_norm": 5.857992172241211, "kl": 2.533273993059993, "learning_rate": 7.042376456291751e-07, "loss": 0.1013, "num_tokens": 16101465.0, "reward": 0.735595703125, "reward_std": 0.00920802727341652, "rewards//mean": 0.735595703125, "rewards//std": 0.03148069605231285, "step": 1863 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3728, "grad_norm": 13.461249351501465, "kl": 3.350059609860182, "learning_rate": 7.039479530433874e-07, "loss": 0.134, "num_tokens": 16110065.0, "reward": 0.77325439453125, "reward_std": 0.001975295366719365, "rewards//mean": 0.77325439453125, "rewards//std": 0.022079868242144585, "step": 1864 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.373, "grad_norm": 25.732999801635742, "kl": 0.7749489434063435, "learning_rate": 7.036581783073887e-07, "loss": 0.031, "num_tokens": 16118681.0, "reward": 0.74041748046875, "reward_std": 0.0021365329157561064, "rewards//mean": 0.74041748046875, "rewards//std": 0.020433401688933372, "step": 1865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3732, "grad_norm": 4.1407470703125, "kl": 2.4588584788143635, "learning_rate": 7.033683215379002e-07, "loss": 0.0984, "num_tokens": 16127433.0, "reward": 0.77386474609375, "reward_std": 0.004057341255247593, "rewards//mean": 0.77386474609375, "rewards//std": 0.0292575154453516, "step": 1866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3734, "grad_norm": 10.789741516113281, "kl": 2.492692956700921, "learning_rate": 7.030783828516759e-07, "loss": 0.0997, "num_tokens": 16136129.0, "reward": 0.75921630859375, "reward_std": 0.0030138352885842323, "rewards//mean": 0.75921630859375, "rewards//std": 0.020414873957633972, "step": 1867 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3736, "grad_norm": 4.771057605743408, "kl": 1.283412892371416, "learning_rate": 7.027883623655034e-07, "loss": 0.0513, "num_tokens": 16144681.0, "reward": 0.7633056640625, "reward_std": 0.002828120719641447, "rewards//mean": 0.7633056640625, "rewards//std": 0.01930948905646801, "step": 1868 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3738, "grad_norm": 6.336138725280762, "kl": 2.7801638580858707, "learning_rate": 7.024982601962026e-07, "loss": 0.1112, "num_tokens": 16153377.0, "reward": 0.77020263671875, "reward_std": 0.012073297053575516, "rewards//mean": 0.77020263671875, "rewards//std": 0.025972329080104828, "step": 1869 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.374, "grad_norm": 4.534235954284668, "kl": 2.658373724669218, "learning_rate": 7.022080764606271e-07, "loss": 0.1063, "num_tokens": 16161937.0, "reward": 0.73101806640625, "reward_std": 0.006083108484745026, "rewards//mean": 0.73101806640625, "rewards//std": 0.03175640106201172, "step": 1870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3742, "grad_norm": 2.8037991523742676, "kl": 1.569754533469677, "learning_rate": 7.019178112756625e-07, "loss": 0.0628, "num_tokens": 16170625.0, "reward": 0.764404296875, "reward_std": 0.005482660606503487, "rewards//mean": 0.764404296875, "rewards//std": 0.028038017451763153, "step": 1871 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3744, "grad_norm": 2.921419382095337, "kl": 1.2974753342568874, "learning_rate": 7.016274647582276e-07, "loss": 0.0519, "num_tokens": 16179305.0, "reward": 0.779296875, "reward_std": 0.006271827034652233, "rewards//mean": 0.779296875, "rewards//std": 0.020505229011178017, "step": 1872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3746, "grad_norm": 7.523586273193359, "kl": 1.884477224200964, "learning_rate": 7.013370370252739e-07, "loss": 0.0754, "num_tokens": 16187841.0, "reward": 0.7667236328125, "reward_std": 0.0027007656171917915, "rewards//mean": 0.7667236328125, "rewards//std": 0.011676693335175514, "step": 1873 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3748, "grad_norm": 1.177193284034729, "kl": 0.48449935764074326, "learning_rate": 7.010465281937858e-07, "loss": 0.0194, "num_tokens": 16196409.0, "reward": 0.80242919921875, "reward_std": 0.0006246941047720611, "rewards//mean": 0.80242919921875, "rewards//std": 0.009664148092269897, "step": 1874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.375, "grad_norm": 2.5623443126678467, "kl": 0.8957772143185139, "learning_rate": 7.007559383807802e-07, "loss": 0.0358, "num_tokens": 16205089.0, "reward": 0.765380859375, "reward_std": 0.0035510831512510777, "rewards//mean": 0.765380859375, "rewards//std": 0.021291865035891533, "step": 1875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3752, "grad_norm": 2.6249661445617676, "kl": 1.313623446971178, "learning_rate": 7.004652677033068e-07, "loss": 0.0525, "num_tokens": 16213697.0, "reward": 0.77490234375, "reward_std": 0.0047293840907514095, "rewards//mean": 0.77490234375, "rewards//std": 0.01691276952624321, "step": 1876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3754, "grad_norm": 6.697422504425049, "kl": 2.700675642117858, "learning_rate": 7.001745162784475e-07, "loss": 0.108, "num_tokens": 16222273.0, "reward": 0.7401123046875, "reward_std": 0.004058271646499634, "rewards//mean": 0.7401123046875, "rewards//std": 0.022052858024835587, "step": 1877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3756, "grad_norm": 2.777811288833618, "kl": 1.570582689717412, "learning_rate": 6.998836842233169e-07, "loss": 0.0628, "num_tokens": 16230945.0, "reward": 0.7818603515625, "reward_std": 0.004180000629276037, "rewards//mean": 0.7818603515625, "rewards//std": 0.020997988060116768, "step": 1878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3758, "grad_norm": 4.907735824584961, "kl": 2.7225206568837166, "learning_rate": 6.995927716550622e-07, "loss": 0.1089, "num_tokens": 16239593.0, "reward": 0.773681640625, "reward_std": 0.006213858723640442, "rewards//mean": 0.773681640625, "rewards//std": 0.02637321501970291, "step": 1879 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.376, "grad_norm": 2.8154067993164062, "kl": 1.4785632863640785, "learning_rate": 6.99301778690863e-07, "loss": 0.0591, "num_tokens": 16248241.0, "reward": 0.76678466796875, "reward_std": 0.008459039963781834, "rewards//mean": 0.76678466796875, "rewards//std": 0.02695280872285366, "step": 1880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3762, "grad_norm": 0.7601634860038757, "kl": 0.5685290694236755, "learning_rate": 6.990107054479312e-07, "loss": 0.0227, "num_tokens": 16256897.0, "reward": 0.76593017578125, "reward_std": 0.00039866380393505096, "rewards//mean": 0.76593017578125, "rewards//std": 0.019578374922275543, "step": 1881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3764, "grad_norm": 1.846854329109192, "kl": 2.0518183298408985, "learning_rate": 6.987195520435109e-07, "loss": 0.0821, "num_tokens": 16265569.0, "reward": 0.73638916015625, "reward_std": 0.004954901058226824, "rewards//mean": 0.73638916015625, "rewards//std": 0.025382203981280327, "step": 1882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3766, "grad_norm": 3.0427513122558594, "kl": 1.4905531387776136, "learning_rate": 6.984283185948789e-07, "loss": 0.0596, "num_tokens": 16274201.0, "reward": 0.77337646484375, "reward_std": 0.0029347692616283894, "rewards//mean": 0.77337646484375, "rewards//std": 0.02367902360856533, "step": 1883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3768, "grad_norm": 4.615382194519043, "kl": 3.5652448013424873, "learning_rate": 6.981370052193439e-07, "loss": 0.1426, "num_tokens": 16283041.0, "reward": 0.7841796875, "reward_std": 0.0068917106837034225, "rewards//mean": 0.7841796875, "rewards//std": 0.029190242290496826, "step": 1884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.377, "grad_norm": 0.02688196487724781, "kl": 0.5113901514559984, "learning_rate": 6.978456120342469e-07, "loss": 0.0205, "num_tokens": 16291745.0, "reward": 0.77978515625, "reward_std": 0.0, "rewards//mean": 0.77978515625, "rewards//std": 0.017267076298594475, "step": 1885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3772, "grad_norm": 1.5688022375106812, "kl": 0.9871741626411676, "learning_rate": 6.975541391569609e-07, "loss": 0.0395, "num_tokens": 16300337.0, "reward": 0.76629638671875, "reward_std": 0.00266617932356894, "rewards//mean": 0.76629638671875, "rewards//std": 0.02031005173921585, "step": 1886 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3774, "grad_norm": 6.302677154541016, "kl": 3.641746509820223, "learning_rate": 6.972625867048914e-07, "loss": 0.1457, "num_tokens": 16308905.0, "reward": 0.7529296875, "reward_std": 0.008294638246297836, "rewards//mean": 0.7529296875, "rewards//std": 0.032053474336862564, "step": 1887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3776, "grad_norm": 1.8699313402175903, "kl": 0.9070111233741045, "learning_rate": 6.969709547954755e-07, "loss": 0.0363, "num_tokens": 16317513.0, "reward": 0.7874755859375, "reward_std": 0.0033482497092336416, "rewards//mean": 0.7874755859375, "rewards//std": 0.024416543543338776, "step": 1888 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3778, "grad_norm": 1.1291965246200562, "kl": 1.054303988814354, "learning_rate": 6.966792435461826e-07, "loss": 0.0422, "num_tokens": 16326153.0, "reward": 0.78106689453125, "reward_std": 0.003481745719909668, "rewards//mean": 0.78106689453125, "rewards//std": 0.023379866033792496, "step": 1889 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.378, "grad_norm": 4.352121829986572, "kl": 0.525041151791811, "learning_rate": 6.963874530745139e-07, "loss": 0.021, "num_tokens": 16334721.0, "reward": 0.74969482421875, "reward_std": 0.001633777399547398, "rewards//mean": 0.74969482421875, "rewards//std": 0.01619293913245201, "step": 1890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3782, "grad_norm": 2.1457080841064453, "kl": 0.546690184623003, "learning_rate": 6.960955834980027e-07, "loss": 0.0219, "num_tokens": 16343401.0, "reward": 0.74993896484375, "reward_std": 0.0012657219776883721, "rewards//mean": 0.74993896484375, "rewards//std": 0.03805427625775337, "step": 1891 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3784, "grad_norm": 2.121290683746338, "kl": 2.8007206730544567, "learning_rate": 6.958036349342139e-07, "loss": 0.112, "num_tokens": 16351961.0, "reward": 0.7564697265625, "reward_std": 0.0069049508310854435, "rewards//mean": 0.7564697265625, "rewards//std": 0.026071587577462196, "step": 1892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3786, "grad_norm": 1.729514479637146, "kl": 2.4328966289758682, "learning_rate": 6.955116075007442e-07, "loss": 0.0973, "num_tokens": 16360601.0, "reward": 0.76934814453125, "reward_std": 0.007147075608372688, "rewards//mean": 0.76934814453125, "rewards//std": 0.026119951158761978, "step": 1893 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3788, "grad_norm": 8.549952507019043, "kl": 5.081495724618435, "learning_rate": 6.952195013152225e-07, "loss": 0.2033, "num_tokens": 16369193.0, "reward": 0.7581787109375, "reward_std": 0.007141421549022198, "rewards//mean": 0.7581787109375, "rewards//std": 0.02632582001388073, "step": 1894 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.379, "grad_norm": 0.9424338340759277, "kl": 1.179397590458393, "learning_rate": 6.94927316495309e-07, "loss": 0.0472, "num_tokens": 16377833.0, "reward": 0.7740478515625, "reward_std": 0.0033621666952967644, "rewards//mean": 0.7740478515625, "rewards//std": 0.014947145245969296, "step": 1895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3792, "grad_norm": 1.572584867477417, "kl": 1.4091698415577412, "learning_rate": 6.946350531586957e-07, "loss": 0.0564, "num_tokens": 16386481.0, "reward": 0.75811767578125, "reward_std": 0.0030726620461791754, "rewards//mean": 0.75811767578125, "rewards//std": 0.022523781284689903, "step": 1896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3794, "grad_norm": 3.0030221939086914, "kl": 2.376695152372122, "learning_rate": 6.943427114231063e-07, "loss": 0.0951, "num_tokens": 16395105.0, "reward": 0.78314208984375, "reward_std": 0.006373615004122257, "rewards//mean": 0.78314208984375, "rewards//std": 0.014023732393980026, "step": 1897 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3796, "grad_norm": 3.121123790740967, "kl": 2.3021695595234632, "learning_rate": 6.94050291406296e-07, "loss": 0.0921, "num_tokens": 16403753.0, "reward": 0.76837158203125, "reward_std": 0.003544052829965949, "rewards//mean": 0.76837158203125, "rewards//std": 0.018528051674365997, "step": 1898 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3798, "grad_norm": 2.6215803623199463, "kl": 1.5299625340849161, "learning_rate": 6.937577932260514e-07, "loss": 0.0612, "num_tokens": 16412481.0, "reward": 0.77008056640625, "reward_std": 0.006633509416133165, "rewards//mean": 0.77008056640625, "rewards//std": 0.024188801646232605, "step": 1899 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.38, "grad_norm": 5.254921913146973, "kl": 5.1301261857151985, "learning_rate": 6.93465217000191e-07, "loss": 0.2052, "num_tokens": 16421089.0, "reward": 0.73919677734375, "reward_std": 0.011049970984458923, "rewards//mean": 0.73919677734375, "rewards//std": 0.03640343248844147, "step": 1900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3802, "grad_norm": 5.4445037841796875, "kl": 1.0033046416938305, "learning_rate": 6.931725628465642e-07, "loss": 0.0401, "num_tokens": 16429721.0, "reward": 0.76068115234375, "reward_std": 0.0011668069055303931, "rewards//mean": 0.76068115234375, "rewards//std": 0.019908828660845757, "step": 1901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3804, "grad_norm": 1.969037652015686, "kl": 1.7568380050361156, "learning_rate": 6.928798308830523e-07, "loss": 0.0703, "num_tokens": 16438369.0, "reward": 0.75250244140625, "reward_std": 0.006278123240917921, "rewards//mean": 0.75250244140625, "rewards//std": 0.02212301827967167, "step": 1902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3806, "grad_norm": 6.894189357757568, "kl": 1.9299946874380112, "learning_rate": 6.925870212275676e-07, "loss": 0.0772, "num_tokens": 16446977.0, "reward": 0.73553466796875, "reward_std": 0.002975771902129054, "rewards//mean": 0.73553466796875, "rewards//std": 0.03099067509174347, "step": 1903 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3808, "grad_norm": 0.13057000935077667, "kl": 0.5076147466897964, "learning_rate": 6.922941339980537e-07, "loss": 0.0203, "num_tokens": 16455593.0, "reward": 0.76409912109375, "reward_std": 0.0001726334885461256, "rewards//mean": 0.76409912109375, "rewards//std": 0.024701541289687157, "step": 1904 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.381, "grad_norm": 5.440893173217773, "kl": 1.5726684052497149, "learning_rate": 6.920011693124856e-07, "loss": 0.0629, "num_tokens": 16464273.0, "reward": 0.8004150390625, "reward_std": 0.004623282700777054, "rewards//mean": 0.8004150390625, "rewards//std": 0.023966003209352493, "step": 1905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3812, "grad_norm": 1.5737837553024292, "kl": 1.1342873200774193, "learning_rate": 6.917081272888696e-07, "loss": 0.0454, "num_tokens": 16472913.0, "reward": 0.7923583984375, "reward_std": 0.00479449238628149, "rewards//mean": 0.7923583984375, "rewards//std": 0.01912674866616726, "step": 1906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3814, "grad_norm": 0.18740078806877136, "kl": 0.5126616656780243, "learning_rate": 6.914150080452428e-07, "loss": 0.0205, "num_tokens": 16481553.0, "reward": 0.75439453125, "reward_std": 0.0003452669770922512, "rewards//mean": 0.75439453125, "rewards//std": 0.022994112223386765, "step": 1907 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3816, "grad_norm": 1.2281596660614014, "kl": 1.7362540028989315, "learning_rate": 6.911218116996736e-07, "loss": 0.0695, "num_tokens": 16490177.0, "reward": 0.77850341796875, "reward_std": 0.003961163107305765, "rewards//mean": 0.77850341796875, "rewards//std": 0.017353542149066925, "step": 1908 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3818, "grad_norm": 4.46489143371582, "kl": 4.049469344317913, "learning_rate": 6.908285383702616e-07, "loss": 0.162, "num_tokens": 16498809.0, "reward": 0.770751953125, "reward_std": 0.016258426010608673, "rewards//mean": 0.770751953125, "rewards//std": 0.028687022626399994, "step": 1909 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.382, "grad_norm": 0.3052246868610382, "kl": 0.5113873705267906, "learning_rate": 6.905351881751371e-07, "loss": 0.0205, "num_tokens": 16507385.0, "reward": 0.76611328125, "reward_std": 0.0003452669770922512, "rewards//mean": 0.76611328125, "rewards//std": 0.02711247280240059, "step": 1910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3822, "grad_norm": 10.212292671203613, "kl": 1.0953901186585426, "learning_rate": 6.902417612324615e-07, "loss": 0.0438, "num_tokens": 16516009.0, "reward": 0.72564697265625, "reward_std": 0.0009432435035705566, "rewards//mean": 0.72564697265625, "rewards//std": 0.021471600979566574, "step": 1911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3824, "grad_norm": 0.18088702857494354, "kl": 0.5196038149297237, "learning_rate": 6.899482576604274e-07, "loss": 0.0208, "num_tokens": 16524705.0, "reward": 0.72503662109375, "reward_std": 0.0001726334885461256, "rewards//mean": 0.72503662109375, "rewards//std": 0.026808079332113266, "step": 1912 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3826, "grad_norm": 3.121293544769287, "kl": 2.167650617659092, "learning_rate": 6.896546775772576e-07, "loss": 0.0867, "num_tokens": 16533297.0, "reward": 0.7655029296875, "reward_std": 0.0039255376905202866, "rewards//mean": 0.7655029296875, "rewards//std": 0.024744173511862755, "step": 1913 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3828, "grad_norm": 3.308300733566284, "kl": 3.256075270473957, "learning_rate": 6.893610211012066e-07, "loss": 0.1302, "num_tokens": 16541953.0, "reward": 0.75604248046875, "reward_std": 0.007841933518648148, "rewards//mean": 0.75604248046875, "rewards//std": 0.023744778707623482, "step": 1914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.383, "grad_norm": 2.5183308124542236, "kl": 3.267112335190177, "learning_rate": 6.890672883505588e-07, "loss": 0.1307, "num_tokens": 16550641.0, "reward": 0.8157958984375, "reward_std": 0.010520893149077892, "rewards//mean": 0.8157958984375, "rewards//std": 0.024912448599934578, "step": 1915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3832, "grad_norm": 5.343947887420654, "kl": 2.1912842132151127, "learning_rate": 6.887734794436299e-07, "loss": 0.0877, "num_tokens": 16559265.0, "reward": 0.80401611328125, "reward_std": 0.007197076454758644, "rewards//mean": 0.80401611328125, "rewards//std": 0.02031005173921585, "step": 1916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 126.578125, "epoch": 0.3834, "grad_norm": 2.964585781097412, "kl": 3.19575122743845, "learning_rate": 6.884795944987661e-07, "loss": 0.0998, "num_tokens": 16567846.0, "reward": 0.741943359375, "reward_std": 0.008112197741866112, "rewards//mean": 0.741943359375, "rewards//std": 0.02922237664461136, "step": 1917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3836, "grad_norm": 0.877312958240509, "kl": 0.5779435522854328, "learning_rate": 6.881856336343441e-07, "loss": 0.0231, "num_tokens": 16576414.0, "reward": 0.77777099609375, "reward_std": 0.0009345901780761778, "rewards//mean": 0.77777099609375, "rewards//std": 0.016054930165410042, "step": 1918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3838, "grad_norm": 2.432293176651001, "kl": 3.6531177069991827, "learning_rate": 6.878915969687714e-07, "loss": 0.1461, "num_tokens": 16585126.0, "reward": 0.7664794921875, "reward_std": 0.009412640705704689, "rewards//mean": 0.7664794921875, "rewards//std": 0.02713005430996418, "step": 1919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.384, "grad_norm": 4.7014665603637695, "kl": 3.147923842072487, "learning_rate": 6.875974846204858e-07, "loss": 0.1259, "num_tokens": 16593886.0, "reward": 0.7637939453125, "reward_std": 0.0037979367189109325, "rewards//mean": 0.7637939453125, "rewards//std": 0.018623093143105507, "step": 1920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3842, "grad_norm": 1.4284223318099976, "kl": 1.4464968033134937, "learning_rate": 6.87303296707956e-07, "loss": 0.0579, "num_tokens": 16602454.0, "reward": 0.782470703125, "reward_std": 0.006412533111870289, "rewards//mean": 0.782470703125, "rewards//std": 0.02188645675778389, "step": 1921 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3844, "grad_norm": 2.7460761070251465, "kl": 0.8830199912190437, "learning_rate": 6.870090333496806e-07, "loss": 0.0353, "num_tokens": 16611174.0, "reward": 0.758544921875, "reward_std": 0.0026580248959362507, "rewards//mean": 0.758544921875, "rewards//std": 0.01050638034939766, "step": 1922 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3846, "grad_norm": 3.0144944190979004, "kl": 0.7796740140765905, "learning_rate": 6.867146946641891e-07, "loss": 0.0312, "num_tokens": 16619886.0, "reward": 0.763427734375, "reward_std": 0.0036183581687510014, "rewards//mean": 0.763427734375, "rewards//std": 0.01431234460324049, "step": 1923 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3848, "grad_norm": 1.8614685535430908, "kl": 2.226287489756942, "learning_rate": 6.864202807700407e-07, "loss": 0.0891, "num_tokens": 16628510.0, "reward": 0.78741455078125, "reward_std": 0.006015172693878412, "rewards//mean": 0.78741455078125, "rewards//std": 0.020133385434746742, "step": 1924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.385, "grad_norm": 4.981801509857178, "kl": 1.6397059187293053, "learning_rate": 6.861257917858257e-07, "loss": 0.0656, "num_tokens": 16637070.0, "reward": 0.76141357421875, "reward_std": 0.006148449145257473, "rewards//mean": 0.76141357421875, "rewards//std": 0.02031899429857731, "step": 1925 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3852, "grad_norm": 0.3080269694328308, "kl": 0.5317921955138445, "learning_rate": 6.858312278301637e-07, "loss": 0.0213, "num_tokens": 16645686.0, "reward": 0.76513671875, "reward_std": 0.0003452669770922512, "rewards//mean": 0.76513671875, "rewards//std": 0.021693328395485878, "step": 1926 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3854, "grad_norm": 1.264594554901123, "kl": 0.5243890956044197, "learning_rate": 6.855365890217056e-07, "loss": 0.021, "num_tokens": 16654302.0, "reward": 0.7730712890625, "reward_std": 0.0006062642787583172, "rewards//mean": 0.7730712890625, "rewards//std": 0.020643183961510658, "step": 1927 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3856, "grad_norm": 1.9129774570465088, "kl": 2.7641488015651703, "learning_rate": 6.852418754791316e-07, "loss": 0.1106, "num_tokens": 16663014.0, "reward": 0.83514404296875, "reward_std": 0.008596815168857574, "rewards//mean": 0.83514404296875, "rewards//std": 0.01876833103597164, "step": 1928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3858, "grad_norm": 2.6161179542541504, "kl": 3.4495524652302265, "learning_rate": 6.849470873211522e-07, "loss": 0.138, "num_tokens": 16671678.0, "reward": 0.72967529296875, "reward_std": 0.0054890476167202, "rewards//mean": 0.72967529296875, "rewards//std": 0.030932005494832993, "step": 1929 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.386, "grad_norm": 1.5045418739318848, "kl": 1.0208391211926937, "learning_rate": 6.846522246665083e-07, "loss": 0.0408, "num_tokens": 16680270.0, "reward": 0.74993896484375, "reward_std": 0.0017160075949504972, "rewards//mean": 0.74993896484375, "rewards//std": 0.017364878207445145, "step": 1930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3862, "grad_norm": 3.578099489212036, "kl": 2.0742180831730366, "learning_rate": 6.843572876339704e-07, "loss": 0.083, "num_tokens": 16688878.0, "reward": 0.74609375, "reward_std": 0.007274179719388485, "rewards//mean": 0.74609375, "rewards//std": 0.02097237855195999, "step": 1931 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3864, "grad_norm": 4.720656394958496, "kl": 1.0430541932582855, "learning_rate": 6.840622763423391e-07, "loss": 0.0417, "num_tokens": 16697486.0, "reward": 0.7464599609375, "reward_std": 0.002421728568151593, "rewards//mean": 0.7464599609375, "rewards//std": 0.028507843613624573, "step": 1932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3866, "grad_norm": 0.20217135548591614, "kl": 0.5369813703000546, "learning_rate": 6.837671909104447e-07, "loss": 0.0215, "num_tokens": 16706150.0, "reward": 0.75970458984375, "reward_std": 0.0001726334885461256, "rewards//mean": 0.75970458984375, "rewards//std": 0.02921505831182003, "step": 1933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3868, "grad_norm": 11.511653900146484, "kl": 3.0753328688442707, "learning_rate": 6.834720314571479e-07, "loss": 0.123, "num_tokens": 16714870.0, "reward": 0.7637939453125, "reward_std": 0.006273890379816294, "rewards//mean": 0.7637939453125, "rewards//std": 0.027673648670315742, "step": 1934 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.387, "grad_norm": 10.808451652526855, "kl": 3.7302815541625023, "learning_rate": 6.831767981013388e-07, "loss": 0.1492, "num_tokens": 16723566.0, "reward": 0.7432861328125, "reward_std": 0.005238348618149757, "rewards//mean": 0.7432861328125, "rewards//std": 0.02662542089819908, "step": 1935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3872, "grad_norm": 1.1473380327224731, "kl": 1.6026993170380592, "learning_rate": 6.828814909619372e-07, "loss": 0.0641, "num_tokens": 16732230.0, "reward": 0.78302001953125, "reward_std": 0.0037447321228682995, "rewards//mean": 0.78302001953125, "rewards//std": 0.020266776904463768, "step": 1936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3874, "grad_norm": 3.704252004623413, "kl": 1.4108627662062645, "learning_rate": 6.82586110157893e-07, "loss": 0.0564, "num_tokens": 16740894.0, "reward": 0.74346923828125, "reward_std": 0.004684296436607838, "rewards//mean": 0.74346923828125, "rewards//std": 0.019930865615606308, "step": 1937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3876, "grad_norm": 2.702566623687744, "kl": 2.1260617580264807, "learning_rate": 6.822906558081856e-07, "loss": 0.085, "num_tokens": 16749590.0, "reward": 0.7769775390625, "reward_std": 0.004325139801949263, "rewards//mean": 0.7769775390625, "rewards//std": 0.022463634610176086, "step": 1938 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3878, "grad_norm": 3.545187473297119, "kl": 1.5988430865108967, "learning_rate": 6.819951280318236e-07, "loss": 0.064, "num_tokens": 16758214.0, "reward": 0.75970458984375, "reward_std": 0.002550494857132435, "rewards//mean": 0.75970458984375, "rewards//std": 0.0240293201059103, "step": 1939 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.388, "grad_norm": 0.3236708343029022, "kl": 0.5156267713755369, "learning_rate": 6.816995269478459e-07, "loss": 0.0206, "num_tokens": 16766902.0, "reward": 0.7744140625, "reward_std": 0.0009165642550215125, "rewards//mean": 0.7744140625, "rewards//std": 0.021497054025530815, "step": 1940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3882, "grad_norm": 8.211658477783203, "kl": 3.5581136774271727, "learning_rate": 6.814038526753204e-07, "loss": 0.1423, "num_tokens": 16775582.0, "reward": 0.7431640625, "reward_std": 0.005529571324586868, "rewards//mean": 0.7431640625, "rewards//std": 0.02930617518723011, "step": 1941 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3884, "grad_norm": 6.93541145324707, "kl": 3.7149961553514004, "learning_rate": 6.811081053333449e-07, "loss": 0.1486, "num_tokens": 16784198.0, "reward": 0.72314453125, "reward_std": 0.004664306528866291, "rewards//mean": 0.72314453125, "rewards//std": 0.035662416368722916, "step": 1942 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3886, "grad_norm": 4.992636203765869, "kl": 4.106484729796648, "learning_rate": 6.80812285041046e-07, "loss": 0.1643, "num_tokens": 16792822.0, "reward": 0.78521728515625, "reward_std": 0.007567324209958315, "rewards//mean": 0.78521728515625, "rewards//std": 0.02045191451907158, "step": 1943 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3888, "grad_norm": 1.497548222541809, "kl": 1.7024446092545986, "learning_rate": 6.805163919175806e-07, "loss": 0.0681, "num_tokens": 16801446.0, "reward": 0.7347412109375, "reward_std": 0.003278088755905628, "rewards//mean": 0.7347412109375, "rewards//std": 0.03186187148094177, "step": 1944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.328125, "epoch": 0.389, "grad_norm": 6.377845764160156, "kl": 2.6406592093408108, "learning_rate": 6.80220426082134e-07, "loss": 0.1172, "num_tokens": 16810115.0, "reward": 0.770263671875, "reward_std": 0.0065625980496406555, "rewards//mean": 0.770263671875, "rewards//std": 0.024977076798677444, "step": 1945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3892, "grad_norm": 1.7635467052459717, "kl": 1.741395279765129, "learning_rate": 6.799243876539213e-07, "loss": 0.0697, "num_tokens": 16818747.0, "reward": 0.7537841796875, "reward_std": 0.006043876521289349, "rewards//mean": 0.7537841796875, "rewards//std": 0.02696438878774643, "step": 1946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 127.84375, "epoch": 0.3894, "grad_norm": 13.894864082336426, "kl": 2.9758454263210297, "learning_rate": 6.796282767521869e-07, "loss": 0.1216, "num_tokens": 16827361.0, "reward": 0.73358154296875, "reward_std": 0.0029574199579656124, "rewards//mean": 0.73358154296875, "rewards//std": 0.029241472482681274, "step": 1947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3896, "grad_norm": 10.494483947753906, "kl": 1.0923346541821957, "learning_rate": 6.793320934962038e-07, "loss": 0.0437, "num_tokens": 16835993.0, "reward": 0.77471923828125, "reward_std": 0.003066399833187461, "rewards//mean": 0.77471923828125, "rewards//std": 0.020076163113117218, "step": 1948 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3898, "grad_norm": 3.937959909439087, "kl": 2.0723252799361944, "learning_rate": 6.790358380052751e-07, "loss": 0.0829, "num_tokens": 16844625.0, "reward": 0.73541259765625, "reward_std": 0.0021803861018270254, "rewards//mean": 0.73541259765625, "rewards//std": 0.02030632458627224, "step": 1949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.39, "grad_norm": 5.640705585479736, "kl": 3.2054260168224573, "learning_rate": 6.787395103987322e-07, "loss": 0.1282, "num_tokens": 16853249.0, "reward": 0.73291015625, "reward_std": 0.006277929991483688, "rewards//mean": 0.73291015625, "rewards//std": 0.021760214120149612, "step": 1950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3902, "grad_norm": 9.933618545532227, "kl": 5.4257692992687225, "learning_rate": 6.784431107959358e-07, "loss": 0.217, "num_tokens": 16861929.0, "reward": 0.7498779296875, "reward_std": 0.0079146483913064, "rewards//mean": 0.7498779296875, "rewards//std": 0.04172457382082939, "step": 1951 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 126.953125, "epoch": 0.3904, "grad_norm": 4.315380096435547, "kl": 2.1314827129244804, "learning_rate": 6.781466393162761e-07, "loss": 0.0654, "num_tokens": 16870414.0, "reward": 0.7310791015625, "reward_std": 0.007672573905438185, "rewards//mean": 0.7310791015625, "rewards//std": 0.04683477804064751, "step": 1952 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3906, "grad_norm": 6.360894680023193, "kl": 2.370349396020174, "learning_rate": 6.778500960791708e-07, "loss": 0.0948, "num_tokens": 16879062.0, "reward": 0.76434326171875, "reward_std": 0.007206708192825317, "rewards//mean": 0.76434326171875, "rewards//std": 0.023272138088941574, "step": 1953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3908, "grad_norm": 5.728469371795654, "kl": 1.5349312275648117, "learning_rate": 6.775534812040686e-07, "loss": 0.0614, "num_tokens": 16887750.0, "reward": 0.76910400390625, "reward_std": 0.006011077668517828, "rewards//mean": 0.76910400390625, "rewards//std": 0.01769132725894451, "step": 1954 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.391, "grad_norm": 9.019811630249023, "kl": 4.6899121925234795, "learning_rate": 6.772567948104452e-07, "loss": 0.1876, "num_tokens": 16896334.0, "reward": 0.771728515625, "reward_std": 0.023606345057487488, "rewards//mean": 0.771728515625, "rewards//std": 0.033049583435058594, "step": 1955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3912, "grad_norm": 4.124979019165039, "kl": 1.4003071710467339, "learning_rate": 6.769600370178059e-07, "loss": 0.056, "num_tokens": 16905014.0, "reward": 0.7510986328125, "reward_std": 0.005611318163573742, "rewards//mean": 0.7510986328125, "rewards//std": 0.03201921656727791, "step": 1956 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3914, "grad_norm": 6.446751594543457, "kl": 3.793982319533825, "learning_rate": 6.766632079456851e-07, "loss": 0.1518, "num_tokens": 16913686.0, "reward": 0.75048828125, "reward_std": 0.009841522201895714, "rewards//mean": 0.75048828125, "rewards//std": 0.026259321719408035, "step": 1957 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3916, "grad_norm": 4.390337944030762, "kl": 2.421869423240423, "learning_rate": 6.76366307713645e-07, "loss": 0.0969, "num_tokens": 16922270.0, "reward": 0.71356201171875, "reward_std": 0.0046535031870007515, "rewards//mean": 0.71356201171875, "rewards//std": 0.03769335895776749, "step": 1958 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3918, "grad_norm": 7.774868965148926, "kl": 2.7957709059119225, "learning_rate": 6.760693364412775e-07, "loss": 0.1118, "num_tokens": 16930942.0, "reward": 0.77545166015625, "reward_std": 0.008291560225188732, "rewards//mean": 0.77545166015625, "rewards//std": 0.028433604165911674, "step": 1959 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.392, "grad_norm": 1.5700663328170776, "kl": 1.0542664527893066, "learning_rate": 6.757722942482022e-07, "loss": 0.0422, "num_tokens": 16939646.0, "reward": 0.762451171875, "reward_std": 0.0034526698291301727, "rewards//mean": 0.762451171875, "rewards//std": 0.02265854924917221, "step": 1960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3922, "grad_norm": 4.8175458908081055, "kl": 3.195584464818239, "learning_rate": 6.754751812540679e-07, "loss": 0.1278, "num_tokens": 16948310.0, "reward": 0.77020263671875, "reward_std": 0.009702223353087902, "rewards//mean": 0.77020263671875, "rewards//std": 0.027835838496685028, "step": 1961 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3924, "grad_norm": 7.265767574310303, "kl": 3.2545111440122128, "learning_rate": 6.751779975785514e-07, "loss": 0.1302, "num_tokens": 16956958.0, "reward": 0.74847412109375, "reward_std": 0.00839682575315237, "rewards//mean": 0.74847412109375, "rewards//std": 0.03243369236588478, "step": 1962 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3926, "grad_norm": 6.825344562530518, "kl": 2.0388757158070803, "learning_rate": 6.748807433413586e-07, "loss": 0.0816, "num_tokens": 16965574.0, "reward": 0.7994384765625, "reward_std": 0.008380787447094917, "rewards//mean": 0.7994384765625, "rewards//std": 0.022832583636045456, "step": 1963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3928, "grad_norm": 5.2122721672058105, "kl": 3.83143163472414, "learning_rate": 6.745834186622231e-07, "loss": 0.1533, "num_tokens": 16974414.0, "reward": 0.7384033203125, "reward_std": 0.00824748631566763, "rewards//mean": 0.7384033203125, "rewards//std": 0.026874415576457977, "step": 1964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.393, "grad_norm": 6.726317405700684, "kl": 1.003250416368246, "learning_rate": 6.742860236609076e-07, "loss": 0.0401, "num_tokens": 16983070.0, "reward": 0.75146484375, "reward_std": 0.0035906690172851086, "rewards//mean": 0.75146484375, "rewards//std": 0.01692708395421505, "step": 1965 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3932, "grad_norm": 4.655847072601318, "kl": 1.132183164358139, "learning_rate": 6.739885584572025e-07, "loss": 0.0453, "num_tokens": 16991726.0, "reward": 0.77154541015625, "reward_std": 0.003970570396631956, "rewards//mean": 0.77154541015625, "rewards//std": 0.01645536907017231, "step": 1966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3934, "grad_norm": 8.565656661987305, "kl": 2.140916470438242, "learning_rate": 6.73691023170927e-07, "loss": 0.0856, "num_tokens": 17000334.0, "reward": 0.741943359375, "reward_std": 0.003573625348508358, "rewards//mean": 0.741943359375, "rewards//std": 0.03848705068230629, "step": 1967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3936, "grad_norm": 4.9662275314331055, "kl": 0.9332614950835705, "learning_rate": 6.733934179219281e-07, "loss": 0.0373, "num_tokens": 17008974.0, "reward": 0.77008056640625, "reward_std": 0.0043439012952148914, "rewards//mean": 0.77008056640625, "rewards//std": 0.018790094181895256, "step": 1968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3938, "grad_norm": 3.461106777191162, "kl": 0.5400516279041767, "learning_rate": 6.730957428300811e-07, "loss": 0.0216, "num_tokens": 17017486.0, "reward": 0.76617431640625, "reward_std": 0.0008589738281443715, "rewards//mean": 0.76617431640625, "rewards//std": 0.02537146583199501, "step": 1969 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.394, "grad_norm": 5.700676918029785, "kl": 1.6092449203133583, "learning_rate": 6.727979980152898e-07, "loss": 0.0644, "num_tokens": 17026086.0, "reward": 0.7872314453125, "reward_std": 0.006560072768479586, "rewards//mean": 0.7872314453125, "rewards//std": 0.023114599287509918, "step": 1970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3942, "grad_norm": 4.2400312423706055, "kl": 1.0226588770747185, "learning_rate": 6.725001835974852e-07, "loss": 0.0409, "num_tokens": 17034678.0, "reward": 0.7716064453125, "reward_std": 0.001833128510043025, "rewards//mean": 0.7716064453125, "rewards//std": 0.020774759352207184, "step": 1971 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3944, "grad_norm": 2.9719595909118652, "kl": 1.8176025450229645, "learning_rate": 6.722022996966277e-07, "loss": 0.0727, "num_tokens": 17043286.0, "reward": 0.73687744140625, "reward_std": 0.00567342946305871, "rewards//mean": 0.73687744140625, "rewards//std": 0.02576459012925625, "step": 1972 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3946, "grad_norm": 2.695848226547241, "kl": 2.118714366108179, "learning_rate": 6.719043464327042e-07, "loss": 0.0847, "num_tokens": 17051862.0, "reward": 0.76287841796875, "reward_std": 0.0035351249389350414, "rewards//mean": 0.76287841796875, "rewards//std": 0.021459612995386124, "step": 1973 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3948, "grad_norm": 2.256169319152832, "kl": 2.1824787370860577, "learning_rate": 6.716063239257306e-07, "loss": 0.0873, "num_tokens": 17060526.0, "reward": 0.755859375, "reward_std": 0.001942233182489872, "rewards//mean": 0.755859375, "rewards//std": 0.035297200083732605, "step": 1974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.395, "grad_norm": 2.6194136142730713, "kl": 0.9225284308195114, "learning_rate": 6.713082322957502e-07, "loss": 0.0369, "num_tokens": 17069118.0, "reward": 0.77252197265625, "reward_std": 0.0016100658103823662, "rewards//mean": 0.77252197265625, "rewards//std": 0.020419321954250336, "step": 1975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3952, "grad_norm": 5.163909435272217, "kl": 3.525698345154524, "learning_rate": 6.710100716628344e-07, "loss": 0.141, "num_tokens": 17077790.0, "reward": 0.76458740234375, "reward_std": 0.006846260279417038, "rewards//mean": 0.76458740234375, "rewards//std": 0.03611956164240837, "step": 1976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3954, "grad_norm": 1.7235068082809448, "kl": 0.5247271433472633, "learning_rate": 6.70711842147082e-07, "loss": 0.021, "num_tokens": 17086342.0, "reward": 0.7650146484375, "reward_std": 0.0003452669770922512, "rewards//mean": 0.7650146484375, "rewards//std": 0.019139407202601433, "step": 1977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3956, "grad_norm": 4.16033411026001, "kl": 0.6964804530143738, "learning_rate": 6.704135438686203e-07, "loss": 0.0279, "num_tokens": 17094966.0, "reward": 0.77239990234375, "reward_std": 0.0036463579162955284, "rewards//mean": 0.77239990234375, "rewards//std": 0.02306501939892769, "step": 1978 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3958, "grad_norm": 2.280411720275879, "kl": 1.7261448204517365, "learning_rate": 6.701151769476032e-07, "loss": 0.069, "num_tokens": 17103566.0, "reward": 0.77471923828125, "reward_std": 0.004031552001833916, "rewards//mean": 0.77471923828125, "rewards//std": 0.02696516178548336, "step": 1979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.396, "grad_norm": 7.489660739898682, "kl": 1.3034958392381668, "learning_rate": 6.698167415042134e-07, "loss": 0.0521, "num_tokens": 17112270.0, "reward": 0.7337646484375, "reward_std": 0.001598489936441183, "rewards//mean": 0.7337646484375, "rewards//std": 0.02923351153731346, "step": 1980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3962, "grad_norm": 8.335091590881348, "kl": 3.803154118359089, "learning_rate": 6.695182376586602e-07, "loss": 0.1521, "num_tokens": 17121046.0, "reward": 0.76043701171875, "reward_std": 0.00808590929955244, "rewards//mean": 0.76043701171875, "rewards//std": 0.03781525045633316, "step": 1981 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3964, "grad_norm": 4.71075439453125, "kl": 2.199516534805298, "learning_rate": 6.692196655311814e-07, "loss": 0.088, "num_tokens": 17129606.0, "reward": 0.7464599609375, "reward_std": 0.005811984650790691, "rewards//mean": 0.7464599609375, "rewards//std": 0.027372246608138084, "step": 1982 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3966, "grad_norm": 20.35379981994629, "kl": 6.530514575541019, "learning_rate": 6.689210252420415e-07, "loss": 0.2612, "num_tokens": 17138302.0, "reward": 0.73248291015625, "reward_std": 0.008087247610092163, "rewards//mean": 0.73248291015625, "rewards//std": 0.035943109542131424, "step": 1983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3968, "grad_norm": 41.484493255615234, "kl": 9.794805243611336, "learning_rate": 6.686223169115327e-07, "loss": 0.3918, "num_tokens": 17147094.0, "reward": 0.73590087890625, "reward_std": 0.0081851901486516, "rewards//mean": 0.73590087890625, "rewards//std": 0.03236641362309456, "step": 1984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.397, "grad_norm": 2.531568765640259, "kl": 1.0259710513055325, "learning_rate": 6.683235406599749e-07, "loss": 0.041, "num_tokens": 17155750.0, "reward": 0.74365234375, "reward_std": 0.004134616814553738, "rewards//mean": 0.74365234375, "rewards//std": 0.02501220442354679, "step": 1985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3972, "grad_norm": 21.211650848388672, "kl": 4.83174666762352, "learning_rate": 6.68024696607715e-07, "loss": 0.1933, "num_tokens": 17164486.0, "reward": 0.78106689453125, "reward_std": 0.00467984564602375, "rewards//mean": 0.78106689453125, "rewards//std": 0.022792354226112366, "step": 1986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3974, "grad_norm": 0.18116259574890137, "kl": 0.5025690570473671, "learning_rate": 6.677257848751276e-07, "loss": 0.0201, "num_tokens": 17173118.0, "reward": 0.80072021484375, "reward_std": 0.0001726334885461256, "rewards//mean": 0.80072021484375, "rewards//std": 0.016840901225805283, "step": 1987 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3976, "grad_norm": 2.183115243911743, "kl": 2.132648713886738, "learning_rate": 6.674268055826138e-07, "loss": 0.0853, "num_tokens": 17181830.0, "reward": 0.77362060546875, "reward_std": 0.006493935827165842, "rewards//mean": 0.77362060546875, "rewards//std": 0.02627711370587349, "step": 1988 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3978, "grad_norm": 1.2820903062820435, "kl": 1.1089227385818958, "learning_rate": 6.671277588506029e-07, "loss": 0.0444, "num_tokens": 17190510.0, "reward": 0.7415771484375, "reward_std": 0.0029527952428907156, "rewards//mean": 0.7415771484375, "rewards//std": 0.03049067221581936, "step": 1989 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.398, "grad_norm": 4.301590442657471, "kl": 3.203248431906104, "learning_rate": 6.668286447995507e-07, "loss": 0.1281, "num_tokens": 17199142.0, "reward": 0.7623291015625, "reward_std": 0.00420347461476922, "rewards//mean": 0.7623291015625, "rewards//std": 0.02261943183839321, "step": 1990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3982, "grad_norm": 1.2312685251235962, "kl": 1.366452895104885, "learning_rate": 6.665294635499403e-07, "loss": 0.0547, "num_tokens": 17207790.0, "reward": 0.77581787109375, "reward_std": 0.0036111832596361637, "rewards//mean": 0.77581787109375, "rewards//std": 0.018422355875372887, "step": 1991 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3984, "grad_norm": 2.7833945751190186, "kl": 2.5449508111923933, "learning_rate": 6.66230215222282e-07, "loss": 0.1018, "num_tokens": 17216406.0, "reward": 0.77215576171875, "reward_std": 0.007916904985904694, "rewards//mean": 0.77215576171875, "rewards//std": 0.031095029786229134, "step": 1992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3986, "grad_norm": 2.739675283432007, "kl": 2.807829365134239, "learning_rate": 6.659308999371129e-07, "loss": 0.1123, "num_tokens": 17225070.0, "reward": 0.7308349609375, "reward_std": 0.005479984916746616, "rewards//mean": 0.7308349609375, "rewards//std": 0.02748703584074974, "step": 1993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3988, "grad_norm": 3.9408509731292725, "kl": 2.529978759586811, "learning_rate": 6.65631517814997e-07, "loss": 0.1012, "num_tokens": 17233710.0, "reward": 0.7501220703125, "reward_std": 0.005697350949048996, "rewards//mean": 0.7501220703125, "rewards//std": 0.02913184091448784, "step": 1994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.399, "grad_norm": 0.9195917844772339, "kl": 1.3602860979735851, "learning_rate": 6.653320689765256e-07, "loss": 0.0544, "num_tokens": 17242334.0, "reward": 0.7642822265625, "reward_std": 0.0020170093048363924, "rewards//mean": 0.7642822265625, "rewards//std": 0.017206918448209763, "step": 1995 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3992, "grad_norm": 3.3152499198913574, "kl": 2.117432627826929, "learning_rate": 6.650325535423166e-07, "loss": 0.0847, "num_tokens": 17250950.0, "reward": 0.74365234375, "reward_std": 0.002731853164732456, "rewards//mean": 0.74365234375, "rewards//std": 0.019054194912314415, "step": 1996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3994, "grad_norm": 12.34266471862793, "kl": 4.6731117479503155, "learning_rate": 6.647329716330147e-07, "loss": 0.1869, "num_tokens": 17259566.0, "reward": 0.7325439453125, "reward_std": 0.0050992234610021114, "rewards//mean": 0.7325439453125, "rewards//std": 0.03254812955856323, "step": 1997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3996, "grad_norm": 3.4830985069274902, "kl": 3.159088408574462, "learning_rate": 6.644333233692916e-07, "loss": 0.1264, "num_tokens": 17268366.0, "reward": 0.77423095703125, "reward_std": 0.009905059821903706, "rewards//mean": 0.77423095703125, "rewards//std": 0.02363487333059311, "step": 1998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.3998, "grad_norm": 2.4505362510681152, "kl": 0.5356312431395054, "learning_rate": 6.641336088718456e-07, "loss": 0.0214, "num_tokens": 17277014.0, "reward": 0.7716064453125, "reward_std": 0.0003452669770922512, "rewards//mean": 0.7716064453125, "rewards//std": 0.02353000082075596, "step": 1999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 128.0, "epoch": 0.4, "grad_norm": 2.995344638824463, "kl": 1.5457897055894136, "learning_rate": 6.638338282614014e-07, "loss": 0.0618, "num_tokens": 17285654.0, "reward": 0.73614501953125, "reward_std": 0.005006371531635523, "rewards//mean": 0.73614501953125, "rewards//std": 0.03228633850812912, "step": 2000 } ], "logging_steps": 1, "max_steps": 5000, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 1000, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }