| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.0, | |
| "eval_steps": 500, | |
| "global_step": 500, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1502.0476379394531, | |
| "epoch": 0.004, | |
| "grad_norm": 0.17851164937019348, | |
| "kl": 0.0, | |
| "learning_rate": 2e-08, | |
| "loss": -0.0206, | |
| "reward": -0.23460307717323303, | |
| "reward_std": 0.13429159671068192, | |
| "rewards/cosine_scaled_reward": -0.11730154044926167, | |
| "rewards/format_reward": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1483.3928833007812, | |
| "epoch": 0.008, | |
| "grad_norm": 0.20364968478679657, | |
| "kl": 0.0, | |
| "learning_rate": 4e-08, | |
| "loss": -0.0247, | |
| "reward": -0.24404804036021233, | |
| "reward_std": 0.15527689084410667, | |
| "rewards/cosine_scaled_reward": -0.12202401272952557, | |
| "rewards/format_reward": 0.0, | |
| "step": 2 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.7857360839844, | |
| "epoch": 0.012, | |
| "grad_norm": 0.2454664260149002, | |
| "kl": -2.008676528930664e-05, | |
| "learning_rate": 6e-08, | |
| "loss": -0.024, | |
| "reward": -0.2683428265154362, | |
| "reward_std": 0.1479046531021595, | |
| "rewards/cosine_scaled_reward": -0.13417141698300838, | |
| "rewards/format_reward": 0.0, | |
| "step": 3 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1531.5595397949219, | |
| "epoch": 0.016, | |
| "grad_norm": 0.26290684938430786, | |
| "kl": -1.3932585716247559e-06, | |
| "learning_rate": 8e-08, | |
| "loss": -0.0063, | |
| "reward": -0.22999461740255356, | |
| "reward_std": 0.13789904117584229, | |
| "rewards/cosine_scaled_reward": -0.11499731056392193, | |
| "rewards/format_reward": 0.0, | |
| "step": 4 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1496.1845703125, | |
| "epoch": 0.02, | |
| "grad_norm": 0.25596341490745544, | |
| "kl": -5.0514936447143555e-06, | |
| "learning_rate": 1e-07, | |
| "loss": -0.0425, | |
| "reward": -0.23508117347955704, | |
| "reward_std": 0.15481781959533691, | |
| "rewards/cosine_scaled_reward": -0.11754059046506882, | |
| "rewards/format_reward": 0.0, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.8690795898438, | |
| "epoch": 0.024, | |
| "grad_norm": 0.24606633186340332, | |
| "kl": -1.0028481483459473e-05, | |
| "learning_rate": 1.2e-07, | |
| "loss": -0.0007, | |
| "reward": -0.25243763625621796, | |
| "reward_std": 0.13587487116456032, | |
| "rewards/cosine_scaled_reward": -0.12621882185339928, | |
| "rewards/format_reward": 0.0, | |
| "step": 6 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1527.5238037109375, | |
| "epoch": 0.028, | |
| "grad_norm": 0.30039116740226746, | |
| "kl": 3.2372772693634033e-06, | |
| "learning_rate": 1.4e-07, | |
| "loss": 0.0092, | |
| "reward": -0.221938356757164, | |
| "reward_std": 0.13823154009878635, | |
| "rewards/cosine_scaled_reward": -0.11096917279064655, | |
| "rewards/format_reward": 0.0, | |
| "step": 7 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1509.2143249511719, | |
| "epoch": 0.032, | |
| "grad_norm": 0.2639496624469757, | |
| "kl": -3.641587682068348e-06, | |
| "learning_rate": 1.6e-07, | |
| "loss": -0.035, | |
| "reward": -0.27191318944096565, | |
| "reward_std": 0.15618360042572021, | |
| "rewards/cosine_scaled_reward": -0.13595658540725708, | |
| "rewards/format_reward": 0.0, | |
| "step": 8 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.0535888671875, | |
| "epoch": 0.036, | |
| "grad_norm": 0.21626578271389008, | |
| "kl": 7.82310962677002e-08, | |
| "learning_rate": 1.8e-07, | |
| "loss": -0.0261, | |
| "reward": -0.24237940087914467, | |
| "reward_std": 0.15605646930634975, | |
| "rewards/cosine_scaled_reward": -0.12118970789015293, | |
| "rewards/format_reward": 0.0, | |
| "step": 9 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.0357360839844, | |
| "epoch": 0.04, | |
| "grad_norm": 0.26584091782569885, | |
| "kl": -6.256159394979477e-06, | |
| "learning_rate": 2e-07, | |
| "loss": -0.0261, | |
| "reward": -0.24382107332348824, | |
| "reward_std": 0.147519638761878, | |
| "rewards/cosine_scaled_reward": -0.12191054411232471, | |
| "rewards/format_reward": 0.0, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.6190795898438, | |
| "epoch": 0.044, | |
| "grad_norm": 0.26533427834510803, | |
| "kl": -5.21540641784668e-06, | |
| "learning_rate": 2.1999999999999998e-07, | |
| "loss": -0.0136, | |
| "reward": -0.26315416768193245, | |
| "reward_std": 0.14432235062122345, | |
| "rewards/cosine_scaled_reward": -0.13157708384096622, | |
| "rewards/format_reward": 0.0, | |
| "step": 11 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.6785888671875, | |
| "epoch": 0.048, | |
| "grad_norm": 0.23109059035778046, | |
| "kl": 4.7460198402404785e-06, | |
| "learning_rate": 2.4e-07, | |
| "loss": -0.0225, | |
| "reward": -0.2342548444867134, | |
| "reward_std": 0.1461905539035797, | |
| "rewards/cosine_scaled_reward": -0.117127425968647, | |
| "rewards/format_reward": 0.0, | |
| "step": 12 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1486.6726379394531, | |
| "epoch": 0.052, | |
| "grad_norm": 0.18697038292884827, | |
| "kl": 3.1348317861557007e-06, | |
| "learning_rate": 2.6e-07, | |
| "loss": -0.0475, | |
| "reward": -0.23411722108721733, | |
| "reward_std": 0.16247618943452835, | |
| "rewards/cosine_scaled_reward": -0.11705861054360867, | |
| "rewards/format_reward": 0.0, | |
| "step": 13 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1493.9166870117188, | |
| "epoch": 0.056, | |
| "grad_norm": 0.26729440689086914, | |
| "kl": -6.791204214096069e-06, | |
| "learning_rate": 2.8e-07, | |
| "loss": -0.0147, | |
| "reward": -0.2375863455235958, | |
| "reward_std": 0.1451248899102211, | |
| "rewards/cosine_scaled_reward": -0.1187931690365076, | |
| "rewards/format_reward": 0.0, | |
| "step": 14 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1493.3392944335938, | |
| "epoch": 0.06, | |
| "grad_norm": 0.24526046216487885, | |
| "kl": 1.4476478099822998e-05, | |
| "learning_rate": 3e-07, | |
| "loss": -0.0512, | |
| "reward": -0.2283475622534752, | |
| "reward_std": 0.1646866388618946, | |
| "rewards/cosine_scaled_reward": -0.11417377926409245, | |
| "rewards/format_reward": 0.0, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1495.8809814453125, | |
| "epoch": 0.064, | |
| "grad_norm": 0.21982906758785248, | |
| "kl": 7.815659046173096e-06, | |
| "learning_rate": 3.2e-07, | |
| "loss": -0.033, | |
| "reward": -0.229737039655447, | |
| "reward_std": 0.1594039984047413, | |
| "rewards/cosine_scaled_reward": -0.1148685198277235, | |
| "rewards/format_reward": 0.0, | |
| "step": 16 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1471.8869018554688, | |
| "epoch": 0.068, | |
| "grad_norm": 0.32592612504959106, | |
| "kl": 9.292736649513245e-06, | |
| "learning_rate": 3.4000000000000003e-07, | |
| "loss": -0.0491, | |
| "reward": -0.22187525033950806, | |
| "reward_std": 0.15859584510326385, | |
| "rewards/cosine_scaled_reward": -0.11093762516975403, | |
| "rewards/format_reward": 0.0, | |
| "step": 17 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1505.4702453613281, | |
| "epoch": 0.072, | |
| "grad_norm": 0.27544769644737244, | |
| "kl": 6.070360541343689e-06, | |
| "learning_rate": 3.6e-07, | |
| "loss": -0.0094, | |
| "reward": -0.2488204501569271, | |
| "reward_std": 0.14258970320224762, | |
| "rewards/cosine_scaled_reward": -0.1244102232158184, | |
| "rewards/format_reward": 0.0, | |
| "step": 18 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.1428833007812, | |
| "epoch": 0.076, | |
| "grad_norm": 0.2813272774219513, | |
| "kl": 4.976987838745117e-06, | |
| "learning_rate": 3.7999999999999996e-07, | |
| "loss": -0.0174, | |
| "reward": -0.20330505073070526, | |
| "reward_std": 0.12616467103362083, | |
| "rewards/cosine_scaled_reward": -0.10165252350270748, | |
| "rewards/format_reward": 0.0, | |
| "step": 19 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1512.0595397949219, | |
| "epoch": 0.08, | |
| "grad_norm": 0.24715571105480194, | |
| "kl": 1.5944242477416992e-06, | |
| "learning_rate": 4e-07, | |
| "loss": -0.0147, | |
| "reward": -0.22946816682815552, | |
| "reward_std": 0.12867936864495277, | |
| "rewards/cosine_scaled_reward": -0.11473408341407776, | |
| "rewards/format_reward": 0.0, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.6428833007812, | |
| "epoch": 0.084, | |
| "grad_norm": 0.2901928424835205, | |
| "kl": 6.8731606006622314e-06, | |
| "learning_rate": 4.1999999999999995e-07, | |
| "loss": -0.0034, | |
| "reward": -0.25839560478925705, | |
| "reward_std": 0.13489549793303013, | |
| "rewards/cosine_scaled_reward": -0.12919779494404793, | |
| "rewards/format_reward": 0.0, | |
| "step": 21 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1488.4642944335938, | |
| "epoch": 0.088, | |
| "grad_norm": 0.2510276436805725, | |
| "kl": 8.165836334228516e-06, | |
| "learning_rate": 4.3999999999999997e-07, | |
| "loss": -0.0203, | |
| "reward": -0.26031654328107834, | |
| "reward_std": 0.148418840020895, | |
| "rewards/cosine_scaled_reward": -0.13015827164053917, | |
| "rewards/format_reward": 0.0, | |
| "step": 22 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1509.4761962890625, | |
| "epoch": 0.092, | |
| "grad_norm": 0.21568480134010315, | |
| "kl": 3.085937350988388e-06, | |
| "learning_rate": 4.6e-07, | |
| "loss": -0.029, | |
| "reward": -0.25878410786390305, | |
| "reward_std": 0.1478017084300518, | |
| "rewards/cosine_scaled_reward": -0.12939205765724182, | |
| "rewards/format_reward": 0.0, | |
| "step": 23 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1522.6667175292969, | |
| "epoch": 0.096, | |
| "grad_norm": 0.20768144726753235, | |
| "kl": 5.729496479034424e-06, | |
| "learning_rate": 4.8e-07, | |
| "loss": -0.0056, | |
| "reward": -0.23294677585363388, | |
| "reward_std": 0.12670473381876945, | |
| "rewards/cosine_scaled_reward": -0.11647338047623634, | |
| "rewards/format_reward": 0.0, | |
| "step": 24 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1505.5059814453125, | |
| "epoch": 0.1, | |
| "grad_norm": 0.20974963903427124, | |
| "kl": 4.060566425323486e-06, | |
| "learning_rate": 5e-07, | |
| "loss": -0.0228, | |
| "reward": -0.23598218336701393, | |
| "reward_std": 0.13139526918530464, | |
| "rewards/cosine_scaled_reward": -0.11799109354615211, | |
| "rewards/format_reward": 0.0, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1503.90478515625, | |
| "epoch": 0.104, | |
| "grad_norm": 0.31668928265571594, | |
| "kl": 5.900859832763672e-06, | |
| "learning_rate": 5.2e-07, | |
| "loss": -0.0143, | |
| "reward": -0.2467576116323471, | |
| "reward_std": 0.15891429036855698, | |
| "rewards/cosine_scaled_reward": -0.12337880209088326, | |
| "rewards/format_reward": 0.0, | |
| "step": 26 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1499.4166870117188, | |
| "epoch": 0.108, | |
| "grad_norm": 0.2661290764808655, | |
| "kl": 1.9058585166931152e-05, | |
| "learning_rate": 5.4e-07, | |
| "loss": -0.0059, | |
| "reward": -0.2198324091732502, | |
| "reward_std": 0.13804786279797554, | |
| "rewards/cosine_scaled_reward": -0.10991620272397995, | |
| "rewards/format_reward": 0.0, | |
| "step": 27 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1494.3929138183594, | |
| "epoch": 0.112, | |
| "grad_norm": 0.24837514758110046, | |
| "kl": 1.190975308418274e-05, | |
| "learning_rate": 5.6e-07, | |
| "loss": -0.0358, | |
| "reward": -0.2488894909620285, | |
| "reward_std": 0.14470278844237328, | |
| "rewards/cosine_scaled_reward": -0.12444474548101425, | |
| "rewards/format_reward": 0.0, | |
| "step": 28 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1495.5893249511719, | |
| "epoch": 0.116, | |
| "grad_norm": 0.2956860065460205, | |
| "kl": 1.84476375579834e-05, | |
| "learning_rate": 5.8e-07, | |
| "loss": -0.0464, | |
| "reward": -0.258271723985672, | |
| "reward_std": 0.17091093584895134, | |
| "rewards/cosine_scaled_reward": -0.129135861992836, | |
| "rewards/format_reward": 0.0, | |
| "step": 29 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1491.5833740234375, | |
| "epoch": 0.12, | |
| "grad_norm": 0.21843063831329346, | |
| "kl": 3.403425216674805e-05, | |
| "learning_rate": 6e-07, | |
| "loss": -0.025, | |
| "reward": -0.22956868633627892, | |
| "reward_std": 0.13988509960472584, | |
| "rewards/cosine_scaled_reward": -0.11478434316813946, | |
| "rewards/format_reward": 0.0, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1494.8691101074219, | |
| "epoch": 0.124, | |
| "grad_norm": 0.27265459299087524, | |
| "kl": 3.37064266204834e-05, | |
| "learning_rate": 6.2e-07, | |
| "loss": -0.0028, | |
| "reward": -0.22422148287296295, | |
| "reward_std": 0.13563549891114235, | |
| "rewards/cosine_scaled_reward": -0.11211073212325573, | |
| "rewards/format_reward": 0.0, | |
| "step": 31 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1500.4226379394531, | |
| "epoch": 0.128, | |
| "grad_norm": 0.2947193682193756, | |
| "kl": 4.464387893676758e-05, | |
| "learning_rate": 6.4e-07, | |
| "loss": -0.0253, | |
| "reward": -0.2335178479552269, | |
| "reward_std": 0.15123932622373104, | |
| "rewards/cosine_scaled_reward": -0.11675892770290375, | |
| "rewards/format_reward": 0.0, | |
| "step": 32 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1512.3988342285156, | |
| "epoch": 0.132, | |
| "grad_norm": 0.27640435099601746, | |
| "kl": 3.385916352272034e-05, | |
| "learning_rate": 6.6e-07, | |
| "loss": 0.0004, | |
| "reward": -0.22090798616409302, | |
| "reward_std": 0.1239312905818224, | |
| "rewards/cosine_scaled_reward": -0.11045399680733681, | |
| "rewards/format_reward": 0.0, | |
| "step": 33 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.6190795898438, | |
| "epoch": 0.136, | |
| "grad_norm": 0.24311278760433197, | |
| "kl": 2.8401613235473633e-05, | |
| "learning_rate": 6.800000000000001e-07, | |
| "loss": -0.0182, | |
| "reward": -0.24944494664669037, | |
| "reward_std": 0.1345935631543398, | |
| "rewards/cosine_scaled_reward": -0.12472246773540974, | |
| "rewards/format_reward": 0.0, | |
| "step": 34 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1497.9107666015625, | |
| "epoch": 0.14, | |
| "grad_norm": 0.21990132331848145, | |
| "kl": 4.646182060241699e-05, | |
| "learning_rate": 7e-07, | |
| "loss": -0.0132, | |
| "reward": -0.22735398262739182, | |
| "reward_std": 0.14252249151468277, | |
| "rewards/cosine_scaled_reward": -0.11367699131369591, | |
| "rewards/format_reward": 0.0, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.0357360839844, | |
| "epoch": 0.144, | |
| "grad_norm": 0.25016918778419495, | |
| "kl": 5.340576171875e-05, | |
| "learning_rate": 7.2e-07, | |
| "loss": -0.0145, | |
| "reward": -0.22054903954267502, | |
| "reward_std": 0.14707811176776886, | |
| "rewards/cosine_scaled_reward": -0.11027451977133751, | |
| "rewards/format_reward": 0.0, | |
| "step": 36 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.1250305175781, | |
| "epoch": 0.148, | |
| "grad_norm": 0.27119770646095276, | |
| "kl": 6.008148193359375e-05, | |
| "learning_rate": 7.4e-07, | |
| "loss": -0.0074, | |
| "reward": -0.21688038110733032, | |
| "reward_std": 0.14238713681697845, | |
| "rewards/cosine_scaled_reward": -0.10844019241631031, | |
| "rewards/format_reward": 0.0, | |
| "step": 37 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.2083740234375, | |
| "epoch": 0.152, | |
| "grad_norm": 0.22954951226711273, | |
| "kl": 4.4226646423339844e-05, | |
| "learning_rate": 7.599999999999999e-07, | |
| "loss": -0.025, | |
| "reward": -0.2533186711370945, | |
| "reward_std": 0.13519956730306149, | |
| "rewards/cosine_scaled_reward": -0.12665932811796665, | |
| "rewards/format_reward": 0.0, | |
| "step": 38 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1505.2440490722656, | |
| "epoch": 0.156, | |
| "grad_norm": 0.20884250104427338, | |
| "kl": 3.9130449295043945e-05, | |
| "learning_rate": 7.799999999999999e-07, | |
| "loss": -0.0186, | |
| "reward": -0.2563895806670189, | |
| "reward_std": 0.1503661349415779, | |
| "rewards/cosine_scaled_reward": -0.12819479033350945, | |
| "rewards/format_reward": 0.0, | |
| "step": 39 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1518.8809509277344, | |
| "epoch": 0.16, | |
| "grad_norm": 0.31034034490585327, | |
| "kl": 4.661083221435547e-05, | |
| "learning_rate": 8e-07, | |
| "loss": -0.014, | |
| "reward": -0.24387329444289207, | |
| "reward_std": 0.14256866462528706, | |
| "rewards/cosine_scaled_reward": -0.12193664722144604, | |
| "rewards/format_reward": 0.0, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1482.1964416503906, | |
| "epoch": 0.164, | |
| "grad_norm": 0.2809925973415375, | |
| "kl": 0.00013720989227294922, | |
| "learning_rate": 8.199999999999999e-07, | |
| "loss": -0.0377, | |
| "reward": -0.23471787199378014, | |
| "reward_std": 0.1464288830757141, | |
| "rewards/cosine_scaled_reward": -0.11735892854630947, | |
| "rewards/format_reward": 0.0, | |
| "step": 41 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.0416870117188, | |
| "epoch": 0.168, | |
| "grad_norm": 0.22887176275253296, | |
| "kl": 0.00016939640045166016, | |
| "learning_rate": 8.399999999999999e-07, | |
| "loss": -0.0143, | |
| "reward": -0.2401226907968521, | |
| "reward_std": 0.12950069829821587, | |
| "rewards/cosine_scaled_reward": -0.1200613472610712, | |
| "rewards/format_reward": 0.0, | |
| "step": 42 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1471.5536193847656, | |
| "epoch": 0.172, | |
| "grad_norm": 0.2556413412094116, | |
| "kl": 0.0002295970916748047, | |
| "learning_rate": 8.599999999999999e-07, | |
| "loss": -0.0741, | |
| "reward": -0.2366926297545433, | |
| "reward_std": 0.17133169993758202, | |
| "rewards/cosine_scaled_reward": -0.1183463241904974, | |
| "rewards/format_reward": 0.0, | |
| "step": 43 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1511.7678527832031, | |
| "epoch": 0.176, | |
| "grad_norm": 0.2524208724498749, | |
| "kl": 0.0002219676971435547, | |
| "learning_rate": 8.799999999999999e-07, | |
| "loss": -0.0275, | |
| "reward": -0.25312257930636406, | |
| "reward_std": 0.16574446111917496, | |
| "rewards/cosine_scaled_reward": -0.12656129337847233, | |
| "rewards/format_reward": 0.0, | |
| "step": 44 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1494.7321472167969, | |
| "epoch": 0.18, | |
| "grad_norm": 0.28623661398887634, | |
| "kl": 0.0002727508544921875, | |
| "learning_rate": 9e-07, | |
| "loss": -0.0223, | |
| "reward": -0.24078572914004326, | |
| "reward_std": 0.14357317984104156, | |
| "rewards/cosine_scaled_reward": -0.12039286643266678, | |
| "rewards/format_reward": 0.0, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1493.5238342285156, | |
| "epoch": 0.184, | |
| "grad_norm": 0.27407318353652954, | |
| "kl": 0.000263214111328125, | |
| "learning_rate": 9.2e-07, | |
| "loss": -0.0368, | |
| "reward": -0.23090793937444687, | |
| "reward_std": 0.15557732805609703, | |
| "rewards/cosine_scaled_reward": -0.11545397154986858, | |
| "rewards/format_reward": 0.0, | |
| "step": 46 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1501.1012268066406, | |
| "epoch": 0.188, | |
| "grad_norm": 0.26540830731391907, | |
| "kl": 0.000286102294921875, | |
| "learning_rate": 9.399999999999999e-07, | |
| "loss": -0.0079, | |
| "reward": -0.21494316309690475, | |
| "reward_std": 0.14235420525074005, | |
| "rewards/cosine_scaled_reward": -0.10747158527374268, | |
| "rewards/format_reward": 0.0, | |
| "step": 47 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1493.4285583496094, | |
| "epoch": 0.192, | |
| "grad_norm": 0.20362912118434906, | |
| "kl": 0.0003046989440917969, | |
| "learning_rate": 9.6e-07, | |
| "loss": -0.0465, | |
| "reward": -0.24233370646834373, | |
| "reward_std": 0.16549209877848625, | |
| "rewards/cosine_scaled_reward": -0.12116685323417187, | |
| "rewards/format_reward": 0.0, | |
| "step": 48 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1496.9107360839844, | |
| "epoch": 0.196, | |
| "grad_norm": 0.23884467780590057, | |
| "kl": 0.00034809112548828125, | |
| "learning_rate": 9.8e-07, | |
| "loss": -0.0294, | |
| "reward": -0.235232163220644, | |
| "reward_std": 0.14046380668878555, | |
| "rewards/cosine_scaled_reward": -0.1176160853356123, | |
| "rewards/format_reward": 0.0, | |
| "step": 49 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1512.2083740234375, | |
| "epoch": 0.2, | |
| "grad_norm": 0.2550745904445648, | |
| "kl": 0.0003681182861328125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0009, | |
| "reward": -0.14228077605366707, | |
| "reward_std": 0.13498482666909695, | |
| "rewards/cosine_scaled_reward": -0.07114038616418839, | |
| "rewards/format_reward": 0.0, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.3154602050781, | |
| "epoch": 0.204, | |
| "grad_norm": 0.26381194591522217, | |
| "kl": 0.0003437995910644531, | |
| "learning_rate": 9.999890338174275e-07, | |
| "loss": -0.0287, | |
| "reward": -0.24466010928153992, | |
| "reward_std": 0.16256770864129066, | |
| "rewards/cosine_scaled_reward": -0.12233005836606026, | |
| "rewards/format_reward": 0.0, | |
| "step": 51 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1503.1488647460938, | |
| "epoch": 0.208, | |
| "grad_norm": 0.1665869802236557, | |
| "kl": 0.00042724609375, | |
| "learning_rate": 9.999561358041868e-07, | |
| "loss": -0.0102, | |
| "reward": -0.2257002554833889, | |
| "reward_std": 0.1360796671360731, | |
| "rewards/cosine_scaled_reward": -0.11285012774169445, | |
| "rewards/format_reward": 0.0, | |
| "step": 52 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1486.7381286621094, | |
| "epoch": 0.212, | |
| "grad_norm": 0.2398059219121933, | |
| "kl": 0.0004673004150390625, | |
| "learning_rate": 9.999013075636804e-07, | |
| "loss": -0.0184, | |
| "reward": -0.23267855867743492, | |
| "reward_std": 0.14873512834310532, | |
| "rewards/cosine_scaled_reward": -0.11633927933871746, | |
| "rewards/format_reward": 0.0, | |
| "step": 53 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1509.2083435058594, | |
| "epoch": 0.216, | |
| "grad_norm": 0.2826734781265259, | |
| "kl": 0.0006093978881835938, | |
| "learning_rate": 9.998245517681593e-07, | |
| "loss": -0.0234, | |
| "reward": -0.2084299884736538, | |
| "reward_std": 0.1463002786040306, | |
| "rewards/cosine_scaled_reward": -0.1042149942368269, | |
| "rewards/format_reward": 0.0, | |
| "step": 54 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1501.0476379394531, | |
| "epoch": 0.22, | |
| "grad_norm": 0.29020005464553833, | |
| "kl": 0.0004963874816894531, | |
| "learning_rate": 9.997258721585931e-07, | |
| "loss": -0.0355, | |
| "reward": -0.2434540018439293, | |
| "reward_std": 0.17717645689845085, | |
| "rewards/cosine_scaled_reward": -0.12172700092196465, | |
| "rewards/format_reward": 0.0, | |
| "step": 55 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.0833435058594, | |
| "epoch": 0.224, | |
| "grad_norm": 0.26805394887924194, | |
| "kl": 0.0004773139953613281, | |
| "learning_rate": 9.996052735444862e-07, | |
| "loss": 0.0009, | |
| "reward": -0.2181985266506672, | |
| "reward_std": 0.13354611210525036, | |
| "rewards/cosine_scaled_reward": -0.10909926891326904, | |
| "rewards/format_reward": 0.0, | |
| "step": 56 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.6488342285156, | |
| "epoch": 0.228, | |
| "grad_norm": 0.27495479583740234, | |
| "kl": 0.0012969970703125, | |
| "learning_rate": 9.994627618036452e-07, | |
| "loss": -0.0226, | |
| "reward": -0.2011367231607437, | |
| "reward_std": 0.14377126656472683, | |
| "rewards/cosine_scaled_reward": -0.10056836158037186, | |
| "rewards/format_reward": 0.0, | |
| "step": 57 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1522.52978515625, | |
| "epoch": 0.232, | |
| "grad_norm": 0.22906683385372162, | |
| "kl": 0.002330780029296875, | |
| "learning_rate": 9.992983438818915e-07, | |
| "loss": -0.0165, | |
| "reward": -0.23109900206327438, | |
| "reward_std": 0.13826126232743263, | |
| "rewards/cosine_scaled_reward": -0.11554950661957264, | |
| "rewards/format_reward": 0.0, | |
| "step": 58 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1461.1904907226562, | |
| "epoch": 0.236, | |
| "grad_norm": 0.21256300806999207, | |
| "kl": 0.004283905029296875, | |
| "learning_rate": 9.991120277927223e-07, | |
| "loss": -0.0729, | |
| "reward": -0.19971829652786255, | |
| "reward_std": 0.18633990362286568, | |
| "rewards/cosine_scaled_reward": -0.09985914640128613, | |
| "rewards/format_reward": 0.0, | |
| "step": 59 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1518.7380981445312, | |
| "epoch": 0.24, | |
| "grad_norm": 0.2587541341781616, | |
| "kl": 0.00316619873046875, | |
| "learning_rate": 9.989038226169207e-07, | |
| "loss": -0.0134, | |
| "reward": -0.2089497372508049, | |
| "reward_std": 0.14539672806859016, | |
| "rewards/cosine_scaled_reward": -0.1044748667627573, | |
| "rewards/format_reward": 0.0, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1492.4881286621094, | |
| "epoch": 0.244, | |
| "grad_norm": 0.2516862154006958, | |
| "kl": 0.0037384033203125, | |
| "learning_rate": 9.98673738502114e-07, | |
| "loss": -0.0497, | |
| "reward": -0.22766747325658798, | |
| "reward_std": 0.17161306738853455, | |
| "rewards/cosine_scaled_reward": -0.11383373104035854, | |
| "rewards/format_reward": 0.0, | |
| "step": 61 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1479.232177734375, | |
| "epoch": 0.248, | |
| "grad_norm": 0.26911845803260803, | |
| "kl": 0.004444122314453125, | |
| "learning_rate": 9.98421786662277e-07, | |
| "loss": -0.0545, | |
| "reward": -0.22953158989548683, | |
| "reward_std": 0.1646084077656269, | |
| "rewards/cosine_scaled_reward": -0.11476579494774342, | |
| "rewards/format_reward": 0.0, | |
| "step": 62 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1500.9821472167969, | |
| "epoch": 0.252, | |
| "grad_norm": 0.30081605911254883, | |
| "kl": 0.0038433074951171875, | |
| "learning_rate": 9.981479793771866e-07, | |
| "loss": -0.0306, | |
| "reward": -0.2425815463066101, | |
| "reward_std": 0.15784814581274986, | |
| "rewards/cosine_scaled_reward": -0.1212907712906599, | |
| "rewards/format_reward": 0.0, | |
| "step": 63 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1474.2262268066406, | |
| "epoch": 0.256, | |
| "grad_norm": 0.11899983882904053, | |
| "kl": 0.00433349609375, | |
| "learning_rate": 9.97852329991824e-07, | |
| "loss": -0.028, | |
| "reward": -0.1484425999224186, | |
| "reward_std": 0.13878681510686874, | |
| "rewards/cosine_scaled_reward": -0.07422130089253187, | |
| "rewards/format_reward": 0.0, | |
| "step": 64 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1469.2440490722656, | |
| "epoch": 0.26, | |
| "grad_norm": 0.21076174080371857, | |
| "kl": 0.0048980712890625, | |
| "learning_rate": 9.975348529157229e-07, | |
| "loss": -0.063, | |
| "reward": -0.23214704915881157, | |
| "reward_std": 0.15723764523863792, | |
| "rewards/cosine_scaled_reward": -0.11607352085411549, | |
| "rewards/format_reward": 0.0, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1479.2738342285156, | |
| "epoch": 0.264, | |
| "grad_norm": 0.25172120332717896, | |
| "kl": 0.0048065185546875, | |
| "learning_rate": 9.971955636222684e-07, | |
| "loss": -0.0575, | |
| "reward": -0.24701237678527832, | |
| "reward_std": 0.176135566085577, | |
| "rewards/cosine_scaled_reward": -0.12350618466734886, | |
| "rewards/format_reward": 0.0, | |
| "step": 66 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1460.0178833007812, | |
| "epoch": 0.268, | |
| "grad_norm": 0.2059454619884491, | |
| "kl": 0.006103515625, | |
| "learning_rate": 9.968344786479415e-07, | |
| "loss": -0.0581, | |
| "reward": -0.21051475405693054, | |
| "reward_std": 0.1539991032332182, | |
| "rewards/cosine_scaled_reward": -0.10525736771523952, | |
| "rewards/format_reward": 0.0, | |
| "step": 67 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.8035888671875, | |
| "epoch": 0.272, | |
| "grad_norm": 0.2547614276409149, | |
| "kl": 0.0057525634765625, | |
| "learning_rate": 9.964516155915151e-07, | |
| "loss": -0.017, | |
| "reward": -0.18334244936704636, | |
| "reward_std": 0.151863232254982, | |
| "rewards/cosine_scaled_reward": -0.09167122654616833, | |
| "rewards/format_reward": 0.0, | |
| "step": 68 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1481.8392944335938, | |
| "epoch": 0.276, | |
| "grad_norm": 0.17602519690990448, | |
| "kl": 0.00487518310546875, | |
| "learning_rate": 9.960469931131936e-07, | |
| "loss": -0.0368, | |
| "reward": -0.15826850943267345, | |
| "reward_std": 0.1565675064921379, | |
| "rewards/cosine_scaled_reward": -0.07913425378501415, | |
| "rewards/format_reward": 0.0, | |
| "step": 69 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1488.6726379394531, | |
| "epoch": 0.28, | |
| "grad_norm": 0.2315431833267212, | |
| "kl": 0.00534820556640625, | |
| "learning_rate": 9.956206309337066e-07, | |
| "loss": -0.0383, | |
| "reward": -0.21064428612589836, | |
| "reward_std": 0.14298444241285324, | |
| "rewards/cosine_scaled_reward": -0.10532214120030403, | |
| "rewards/format_reward": 0.0, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1521.3095397949219, | |
| "epoch": 0.284, | |
| "grad_norm": 0.25028038024902344, | |
| "kl": 0.00449371337890625, | |
| "learning_rate": 9.951725498333448e-07, | |
| "loss": -0.018, | |
| "reward": -0.2252160757780075, | |
| "reward_std": 0.1335316188633442, | |
| "rewards/cosine_scaled_reward": -0.11260804533958435, | |
| "rewards/format_reward": 0.0, | |
| "step": 71 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1487.1666870117188, | |
| "epoch": 0.288, | |
| "grad_norm": 0.3014598488807678, | |
| "kl": 0.0053863525390625, | |
| "learning_rate": 9.947027716509488e-07, | |
| "loss": -0.0296, | |
| "reward": -0.22651539742946625, | |
| "reward_std": 0.15895461291074753, | |
| "rewards/cosine_scaled_reward": -0.11325769871473312, | |
| "rewards/format_reward": 0.0, | |
| "step": 72 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1500.0417175292969, | |
| "epoch": 0.292, | |
| "grad_norm": 0.19289681315422058, | |
| "kl": 0.00534820556640625, | |
| "learning_rate": 9.942113192828444e-07, | |
| "loss": -0.0204, | |
| "reward": -0.22585123777389526, | |
| "reward_std": 0.1563442163169384, | |
| "rewards/cosine_scaled_reward": -0.11292561888694763, | |
| "rewards/format_reward": 0.0, | |
| "step": 73 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1481.1845703125, | |
| "epoch": 0.296, | |
| "grad_norm": 0.21433287858963013, | |
| "kl": 0.00696563720703125, | |
| "learning_rate": 9.93698216681727e-07, | |
| "loss": -0.0248, | |
| "reward": -0.21346117928624153, | |
| "reward_std": 0.16779018752276897, | |
| "rewards/cosine_scaled_reward": -0.10673058778047562, | |
| "rewards/format_reward": 0.0, | |
| "step": 74 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1492.0595397949219, | |
| "epoch": 0.3, | |
| "grad_norm": 0.20086415112018585, | |
| "kl": 0.0060577392578125, | |
| "learning_rate": 9.931634888554935e-07, | |
| "loss": -0.0315, | |
| "reward": -0.24520759657025337, | |
| "reward_std": 0.16547074727714062, | |
| "rewards/cosine_scaled_reward": -0.12260380387306213, | |
| "rewards/format_reward": 0.0, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1509.7440795898438, | |
| "epoch": 0.304, | |
| "grad_norm": 0.2499692440032959, | |
| "kl": 0.00594329833984375, | |
| "learning_rate": 9.926071618660237e-07, | |
| "loss": -0.029, | |
| "reward": -0.22075266018509865, | |
| "reward_std": 0.1520039215683937, | |
| "rewards/cosine_scaled_reward": -0.11037633195519447, | |
| "rewards/format_reward": 0.0, | |
| "step": 76 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1493.0655212402344, | |
| "epoch": 0.308, | |
| "grad_norm": 0.1303885579109192, | |
| "kl": 0.0078887939453125, | |
| "learning_rate": 9.9202926282791e-07, | |
| "loss": 0.0068, | |
| "reward": -0.20533370971679688, | |
| "reward_std": 0.1422851476818323, | |
| "rewards/cosine_scaled_reward": -0.10266684927046299, | |
| "rewards/format_reward": 0.0, | |
| "step": 77 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1501.3035888671875, | |
| "epoch": 0.312, | |
| "grad_norm": 0.23692218959331512, | |
| "kl": 0.0076141357421875, | |
| "learning_rate": 9.91429819907136e-07, | |
| "loss": -0.0294, | |
| "reward": -0.1995321549475193, | |
| "reward_std": 0.15328680351376534, | |
| "rewards/cosine_scaled_reward": -0.0997660793364048, | |
| "rewards/format_reward": 0.0, | |
| "step": 78 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.3512268066406, | |
| "epoch": 0.316, | |
| "grad_norm": 0.2173466831445694, | |
| "kl": 0.0062408447265625, | |
| "learning_rate": 9.908088623197048e-07, | |
| "loss": -0.0243, | |
| "reward": -0.22904419153928757, | |
| "reward_std": 0.1448185909539461, | |
| "rewards/cosine_scaled_reward": -0.11452210135757923, | |
| "rewards/format_reward": 0.0, | |
| "step": 79 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1484.3869323730469, | |
| "epoch": 0.32, | |
| "grad_norm": 0.16200865805149078, | |
| "kl": 0.00707244873046875, | |
| "learning_rate": 9.901664203302124e-07, | |
| "loss": -0.0239, | |
| "reward": -0.22017718479037285, | |
| "reward_std": 0.15285737439990044, | |
| "rewards/cosine_scaled_reward": -0.11008859053254128, | |
| "rewards/format_reward": 0.0, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1484.7261962890625, | |
| "epoch": 0.324, | |
| "grad_norm": 0.1811896711587906, | |
| "kl": 0.00650787353515625, | |
| "learning_rate": 9.895025252503755e-07, | |
| "loss": -0.0262, | |
| "reward": -0.24382955580949783, | |
| "reward_std": 0.1623360477387905, | |
| "rewards/cosine_scaled_reward": -0.12191477790474892, | |
| "rewards/format_reward": 0.0, | |
| "step": 81 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1456.0357360839844, | |
| "epoch": 0.328, | |
| "grad_norm": 0.1585092842578888, | |
| "kl": 0.010345458984375, | |
| "learning_rate": 9.888172094375033e-07, | |
| "loss": -0.066, | |
| "reward": -0.23999103158712387, | |
| "reward_std": 0.1869661882519722, | |
| "rewards/cosine_scaled_reward": -0.11999551579356194, | |
| "rewards/format_reward": 0.0, | |
| "step": 82 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1485.02978515625, | |
| "epoch": 0.332, | |
| "grad_norm": 0.2301841825246811, | |
| "kl": 0.0067596435546875, | |
| "learning_rate": 9.881105062929221e-07, | |
| "loss": -0.0321, | |
| "reward": -0.2552091106772423, | |
| "reward_std": 0.1685757040977478, | |
| "rewards/cosine_scaled_reward": -0.12760455161333084, | |
| "rewards/format_reward": 0.0, | |
| "step": 83 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1486.7024230957031, | |
| "epoch": 0.336, | |
| "grad_norm": 0.24452205002307892, | |
| "kl": 0.0095062255859375, | |
| "learning_rate": 9.873824502603459e-07, | |
| "loss": -0.0399, | |
| "reward": -0.2177020013332367, | |
| "reward_std": 0.1546536386013031, | |
| "rewards/cosine_scaled_reward": -0.1088510025292635, | |
| "rewards/format_reward": 0.0, | |
| "step": 84 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1472.2083740234375, | |
| "epoch": 0.34, | |
| "grad_norm": 0.2162124067544937, | |
| "kl": 0.00879669189453125, | |
| "learning_rate": 9.866330768241983e-07, | |
| "loss": -0.0363, | |
| "reward": -0.2306637428700924, | |
| "reward_std": 0.15445036813616753, | |
| "rewards/cosine_scaled_reward": -0.11533187702298164, | |
| "rewards/format_reward": 0.0, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.90478515625, | |
| "epoch": 0.344, | |
| "grad_norm": 0.29467546939849854, | |
| "kl": 0.010467529296875, | |
| "learning_rate": 9.85862422507884e-07, | |
| "loss": -0.0225, | |
| "reward": -0.21584435179829597, | |
| "reward_std": 0.15913846716284752, | |
| "rewards/cosine_scaled_reward": -0.10792217776179314, | |
| "rewards/format_reward": 0.0, | |
| "step": 86 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1493.9166870117188, | |
| "epoch": 0.348, | |
| "grad_norm": 0.2552238404750824, | |
| "kl": 0.0130157470703125, | |
| "learning_rate": 9.850705248720068e-07, | |
| "loss": -0.0419, | |
| "reward": -0.22282668948173523, | |
| "reward_std": 0.1558297798037529, | |
| "rewards/cosine_scaled_reward": -0.11141334660351276, | |
| "rewards/format_reward": 0.0, | |
| "step": 87 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.4166870117188, | |
| "epoch": 0.352, | |
| "grad_norm": 0.18905065953731537, | |
| "kl": 0.0161590576171875, | |
| "learning_rate": 9.8425742251254e-07, | |
| "loss": -0.0124, | |
| "reward": -0.2435382977128029, | |
| "reward_std": 0.1510351374745369, | |
| "rewards/cosine_scaled_reward": -0.1217691469937563, | |
| "rewards/format_reward": 0.0, | |
| "step": 88 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1456.6131286621094, | |
| "epoch": 0.356, | |
| "grad_norm": 0.18983080983161926, | |
| "kl": 0.0308837890625, | |
| "learning_rate": 9.83423155058946e-07, | |
| "loss": -0.0716, | |
| "reward": -0.23182464018464088, | |
| "reward_std": 0.17492059245705605, | |
| "rewards/cosine_scaled_reward": -0.11591232009232044, | |
| "rewards/format_reward": 0.0, | |
| "step": 89 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1462.7916870117188, | |
| "epoch": 0.36, | |
| "grad_norm": 0.18697425723075867, | |
| "kl": 0.036468505859375, | |
| "learning_rate": 9.825677631722435e-07, | |
| "loss": -0.0794, | |
| "reward": -0.25727425515651703, | |
| "reward_std": 0.18599339574575424, | |
| "rewards/cosine_scaled_reward": -0.12863712757825851, | |
| "rewards/format_reward": 0.0, | |
| "step": 90 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1452.9166870117188, | |
| "epoch": 0.364, | |
| "grad_norm": 0.11306176334619522, | |
| "kl": 0.05767822265625, | |
| "learning_rate": 9.816912885430258e-07, | |
| "loss": -0.074, | |
| "reward": -0.2355377934873104, | |
| "reward_std": 0.1886419989168644, | |
| "rewards/cosine_scaled_reward": -0.11776889488101006, | |
| "rewards/format_reward": 0.0, | |
| "step": 91 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1423.2381286621094, | |
| "epoch": 0.368, | |
| "grad_norm": 0.12384199351072311, | |
| "kl": 0.0667724609375, | |
| "learning_rate": 9.807937738894303e-07, | |
| "loss": -0.094, | |
| "reward": -0.28002386912703514, | |
| "reward_std": 0.22308171913027763, | |
| "rewards/cosine_scaled_reward": -0.14001193456351757, | |
| "rewards/format_reward": 0.0, | |
| "step": 92 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1463.5298156738281, | |
| "epoch": 0.372, | |
| "grad_norm": 0.1097106859087944, | |
| "kl": 0.0745849609375, | |
| "learning_rate": 9.798752629550546e-07, | |
| "loss": -0.0497, | |
| "reward": -0.260006383061409, | |
| "reward_std": 0.20789402723312378, | |
| "rewards/cosine_scaled_reward": -0.1300031915307045, | |
| "rewards/format_reward": 0.0, | |
| "step": 93 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1423.0416870117188, | |
| "epoch": 0.376, | |
| "grad_norm": 0.13807816803455353, | |
| "kl": 0.0921630859375, | |
| "learning_rate": 9.78935800506826e-07, | |
| "loss": -0.0664, | |
| "reward": -0.2304685339331627, | |
| "reward_std": 0.20511355623602867, | |
| "rewards/cosine_scaled_reward": -0.11523427255451679, | |
| "rewards/format_reward": 0.0, | |
| "step": 94 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1439.827392578125, | |
| "epoch": 0.38, | |
| "grad_norm": 0.1847662776708603, | |
| "kl": 0.080810546875, | |
| "learning_rate": 9.779754323328192e-07, | |
| "loss": -0.0574, | |
| "reward": -0.22580492869019508, | |
| "reward_std": 0.19689049944281578, | |
| "rewards/cosine_scaled_reward": -0.11290246807038784, | |
| "rewards/format_reward": 0.0, | |
| "step": 95 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1450.6666870117188, | |
| "epoch": 0.384, | |
| "grad_norm": 0.14935868978500366, | |
| "kl": 0.0830078125, | |
| "learning_rate": 9.769942052400235e-07, | |
| "loss": -0.0721, | |
| "reward": -0.28848847001791, | |
| "reward_std": 0.22373779118061066, | |
| "rewards/cosine_scaled_reward": -0.144244235008955, | |
| "rewards/format_reward": 0.0, | |
| "step": 96 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1433.7916870117188, | |
| "epoch": 0.388, | |
| "grad_norm": 0.14467309415340424, | |
| "kl": 0.0953369140625, | |
| "learning_rate": 9.759921670520634e-07, | |
| "loss": -0.0565, | |
| "reward": -0.2675531320273876, | |
| "reward_std": 0.19667528942227364, | |
| "rewards/cosine_scaled_reward": -0.1337765622884035, | |
| "rewards/format_reward": 0.0, | |
| "step": 97 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1444.4345703125, | |
| "epoch": 0.392, | |
| "grad_norm": 0.14709749817848206, | |
| "kl": 0.092529296875, | |
| "learning_rate": 9.749693666068663e-07, | |
| "loss": -0.0794, | |
| "reward": -0.26461150124669075, | |
| "reward_std": 0.2228638045489788, | |
| "rewards/cosine_scaled_reward": -0.13230575062334538, | |
| "rewards/format_reward": 0.0, | |
| "step": 98 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1438.6369018554688, | |
| "epoch": 0.396, | |
| "grad_norm": 0.1645117700099945, | |
| "kl": 0.090576171875, | |
| "learning_rate": 9.739258537542835e-07, | |
| "loss": -0.0804, | |
| "reward": -0.2615456096827984, | |
| "reward_std": 0.22437894716858864, | |
| "rewards/cosine_scaled_reward": -0.13077280297875404, | |
| "rewards/format_reward": 0.0, | |
| "step": 99 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1447.9404907226562, | |
| "epoch": 0.4, | |
| "grad_norm": 0.12536393105983734, | |
| "kl": 0.0902099609375, | |
| "learning_rate": 9.728616793536587e-07, | |
| "loss": -0.0565, | |
| "reward": -0.23606609553098679, | |
| "reward_std": 0.20211446657776833, | |
| "rewards/cosine_scaled_reward": -0.11803305521607399, | |
| "rewards/format_reward": 0.0, | |
| "step": 100 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1461.0238647460938, | |
| "epoch": 0.404, | |
| "grad_norm": 0.1715729534626007, | |
| "kl": 0.0977783203125, | |
| "learning_rate": 9.717768952713511e-07, | |
| "loss": -0.0419, | |
| "reward": -0.25642454996705055, | |
| "reward_std": 0.20571942254900932, | |
| "rewards/cosine_scaled_reward": -0.12821227870881557, | |
| "rewards/format_reward": 0.0, | |
| "step": 101 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1441.0595397949219, | |
| "epoch": 0.408, | |
| "grad_norm": 0.14349091053009033, | |
| "kl": 0.07373046875, | |
| "learning_rate": 9.706715543782064e-07, | |
| "loss": -0.0755, | |
| "reward": -0.25798120722174644, | |
| "reward_std": 0.18921422585844994, | |
| "rewards/cosine_scaled_reward": -0.12899060919880867, | |
| "rewards/format_reward": 0.0, | |
| "step": 102 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1429.2559814453125, | |
| "epoch": 0.412, | |
| "grad_norm": 0.12457617372274399, | |
| "kl": 0.08154296875, | |
| "learning_rate": 9.695457105469804e-07, | |
| "loss": -0.0635, | |
| "reward": -0.2596958056092262, | |
| "reward_std": 0.21758576482534409, | |
| "rewards/cosine_scaled_reward": -0.12984789907932281, | |
| "rewards/format_reward": 0.0, | |
| "step": 103 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1455.0000610351562, | |
| "epoch": 0.416, | |
| "grad_norm": 0.1320362240076065, | |
| "kl": 0.0777587890625, | |
| "learning_rate": 9.683994186497132e-07, | |
| "loss": -0.0587, | |
| "reward": -0.23436888307332993, | |
| "reward_std": 0.18900957331061363, | |
| "rewards/cosine_scaled_reward": -0.11718444526195526, | |
| "rewards/format_reward": 0.0, | |
| "step": 104 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1421.0952758789062, | |
| "epoch": 0.42, | |
| "grad_norm": 0.13309422135353088, | |
| "kl": 0.0897216796875, | |
| "learning_rate": 9.672327345550543e-07, | |
| "loss": -0.0934, | |
| "reward": -0.25493229925632477, | |
| "reward_std": 0.22001174464821815, | |
| "rewards/cosine_scaled_reward": -0.12746614776551723, | |
| "rewards/format_reward": 0.0, | |
| "step": 105 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1490.9940795898438, | |
| "epoch": 0.424, | |
| "grad_norm": 0.1491984874010086, | |
| "kl": 0.0736083984375, | |
| "learning_rate": 9.66045715125541e-07, | |
| "loss": -0.0299, | |
| "reward": -0.2312908135354519, | |
| "reward_std": 0.16199098154902458, | |
| "rewards/cosine_scaled_reward": -0.11564541421830654, | |
| "rewards/format_reward": 0.0, | |
| "step": 106 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1421.0357666015625, | |
| "epoch": 0.428, | |
| "grad_norm": 0.16547471284866333, | |
| "kl": 0.0780029296875, | |
| "learning_rate": 9.648384182148252e-07, | |
| "loss": -0.067, | |
| "reward": -0.27354947850108147, | |
| "reward_std": 0.1951713114976883, | |
| "rewards/cosine_scaled_reward": -0.13677473925054073, | |
| "rewards/format_reward": 0.0, | |
| "step": 107 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1469.0833740234375, | |
| "epoch": 0.432, | |
| "grad_norm": 0.14035455882549286, | |
| "kl": 0.0938720703125, | |
| "learning_rate": 9.636109026648554e-07, | |
| "loss": -0.0248, | |
| "reward": -0.22176991775631905, | |
| "reward_std": 0.1815592534840107, | |
| "rewards/cosine_scaled_reward": -0.11088495329022408, | |
| "rewards/format_reward": 0.0, | |
| "step": 108 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1440.5238647460938, | |
| "epoch": 0.436, | |
| "grad_norm": 0.11456211656332016, | |
| "kl": 0.077880859375, | |
| "learning_rate": 9.623632283030077e-07, | |
| "loss": -0.0604, | |
| "reward": -0.2620925232768059, | |
| "reward_std": 0.21131489053368568, | |
| "rewards/cosine_scaled_reward": -0.13104625791311264, | |
| "rewards/format_reward": 0.0, | |
| "step": 109 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1479.4583740234375, | |
| "epoch": 0.44, | |
| "grad_norm": 0.11461742222309113, | |
| "kl": 0.07177734375, | |
| "learning_rate": 9.610954559391704e-07, | |
| "loss": -0.0354, | |
| "reward": -0.21340973302721977, | |
| "reward_std": 0.18217052891850471, | |
| "rewards/cosine_scaled_reward": -0.10670486651360989, | |
| "rewards/format_reward": 0.0, | |
| "step": 110 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1417.2976379394531, | |
| "epoch": 0.444, | |
| "grad_norm": 0.1527003049850464, | |
| "kl": 0.07318115234375, | |
| "learning_rate": 9.598076473627796e-07, | |
| "loss": -0.1136, | |
| "reward": -0.24674446135759354, | |
| "reward_std": 0.19690455496311188, | |
| "rewards/cosine_scaled_reward": -0.12337223440408707, | |
| "rewards/format_reward": 0.0, | |
| "step": 111 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1433.4464721679688, | |
| "epoch": 0.448, | |
| "grad_norm": 0.1355064958333969, | |
| "kl": 0.0743408203125, | |
| "learning_rate": 9.58499865339809e-07, | |
| "loss": -0.0673, | |
| "reward": -0.2527715191245079, | |
| "reward_std": 0.19281791523098946, | |
| "rewards/cosine_scaled_reward": -0.12638575583696365, | |
| "rewards/format_reward": 0.0, | |
| "step": 112 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.5714721679688, | |
| "epoch": 0.452, | |
| "grad_norm": 0.11734712868928909, | |
| "kl": 0.0765380859375, | |
| "learning_rate": 9.571721736097088e-07, | |
| "loss": -0.0251, | |
| "reward": -0.22142696008086205, | |
| "reward_std": 0.18005133792757988, | |
| "rewards/cosine_scaled_reward": -0.11071347445249557, | |
| "rewards/format_reward": 0.0, | |
| "step": 113 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1462.7084045410156, | |
| "epoch": 0.456, | |
| "grad_norm": 0.1974153220653534, | |
| "kl": 0.0665283203125, | |
| "learning_rate": 9.55824636882301e-07, | |
| "loss": -0.0632, | |
| "reward": -0.1928116953931749, | |
| "reward_std": 0.19508182629942894, | |
| "rewards/cosine_scaled_reward": -0.09640584839507937, | |
| "rewards/format_reward": 0.0, | |
| "step": 114 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1444.9524230957031, | |
| "epoch": 0.46, | |
| "grad_norm": 0.13885585963726044, | |
| "kl": 0.0650634765625, | |
| "learning_rate": 9.54457320834625e-07, | |
| "loss": -0.0436, | |
| "reward": -0.2363814003765583, | |
| "reward_std": 0.19080934301018715, | |
| "rewards/cosine_scaled_reward": -0.11819070391356945, | |
| "rewards/format_reward": 0.0, | |
| "step": 115 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1462.9881286621094, | |
| "epoch": 0.464, | |
| "grad_norm": 0.14188507199287415, | |
| "kl": 0.0745849609375, | |
| "learning_rate": 9.530702921077358e-07, | |
| "loss": -0.0326, | |
| "reward": -0.21813786774873734, | |
| "reward_std": 0.17841872572898865, | |
| "rewards/cosine_scaled_reward": -0.10906893201172352, | |
| "rewards/format_reward": 0.0, | |
| "step": 116 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1423.232177734375, | |
| "epoch": 0.468, | |
| "grad_norm": 0.14759542047977448, | |
| "kl": 0.07354736328125, | |
| "learning_rate": 9.516636183034564e-07, | |
| "loss": -0.0955, | |
| "reward": -0.24884852021932602, | |
| "reward_std": 0.20949439704418182, | |
| "rewards/cosine_scaled_reward": -0.12442425638437271, | |
| "rewards/format_reward": 0.0, | |
| "step": 117 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1433.3631286621094, | |
| "epoch": 0.472, | |
| "grad_norm": 0.12356512248516083, | |
| "kl": 0.0736083984375, | |
| "learning_rate": 9.502373679810839e-07, | |
| "loss": -0.0907, | |
| "reward": -0.2389114946126938, | |
| "reward_std": 0.1962103582918644, | |
| "rewards/cosine_scaled_reward": -0.11945574544370174, | |
| "rewards/format_reward": 0.0, | |
| "step": 118 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1425.5476684570312, | |
| "epoch": 0.476, | |
| "grad_norm": 0.13675737380981445, | |
| "kl": 0.080322265625, | |
| "learning_rate": 9.487916106540465e-07, | |
| "loss": -0.0548, | |
| "reward": -0.22732584923505783, | |
| "reward_std": 0.17597166821360588, | |
| "rewards/cosine_scaled_reward": -0.11366293206810951, | |
| "rewards/format_reward": 0.0, | |
| "step": 119 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1407.125, | |
| "epoch": 0.48, | |
| "grad_norm": 0.1944396197795868, | |
| "kl": 0.0782470703125, | |
| "learning_rate": 9.473264167865171e-07, | |
| "loss": -0.1104, | |
| "reward": -0.23876191675662994, | |
| "reward_std": 0.21116216480731964, | |
| "rewards/cosine_scaled_reward": -0.11938095837831497, | |
| "rewards/format_reward": 0.0, | |
| "step": 120 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1460.9940795898438, | |
| "epoch": 0.484, | |
| "grad_norm": 0.13976925611495972, | |
| "kl": 0.07110595703125, | |
| "learning_rate": 9.458418577899774e-07, | |
| "loss": -0.0623, | |
| "reward": -0.22172370925545692, | |
| "reward_std": 0.17737172171473503, | |
| "rewards/cosine_scaled_reward": -0.11086185090243816, | |
| "rewards/format_reward": 0.0, | |
| "step": 121 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1445.0536193847656, | |
| "epoch": 0.488, | |
| "grad_norm": 0.1271459460258484, | |
| "kl": 0.0771484375, | |
| "learning_rate": 9.443380060197385e-07, | |
| "loss": -0.09, | |
| "reward": -0.234033714979887, | |
| "reward_std": 0.21107907965779305, | |
| "rewards/cosine_scaled_reward": -0.1170168574899435, | |
| "rewards/format_reward": 0.0, | |
| "step": 122 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1461.5833740234375, | |
| "epoch": 0.492, | |
| "grad_norm": 0.1251479536294937, | |
| "kl": 0.07427978515625, | |
| "learning_rate": 9.428149347714143e-07, | |
| "loss": -0.0677, | |
| "reward": -0.21826723590493202, | |
| "reward_std": 0.19678263366222382, | |
| "rewards/cosine_scaled_reward": -0.10913361981511116, | |
| "rewards/format_reward": 0.0, | |
| "step": 123 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1441.7857360839844, | |
| "epoch": 0.496, | |
| "grad_norm": 0.16092143952846527, | |
| "kl": 0.060302734375, | |
| "learning_rate": 9.412727182773486e-07, | |
| "loss": -0.0921, | |
| "reward": -0.22351692616939545, | |
| "reward_std": 0.2029583677649498, | |
| "rewards/cosine_scaled_reward": -0.11175846680998802, | |
| "rewards/format_reward": 0.0, | |
| "step": 124 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1426.7262268066406, | |
| "epoch": 0.5, | |
| "grad_norm": 0.1188863217830658, | |
| "kl": 0.0787353515625, | |
| "learning_rate": 9.397114317029974e-07, | |
| "loss": -0.0431, | |
| "reward": -0.2245512492954731, | |
| "reward_std": 0.17906467244029045, | |
| "rewards/cosine_scaled_reward": -0.11227562837302685, | |
| "rewards/format_reward": 0.0, | |
| "step": 125 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1443.7917175292969, | |
| "epoch": 0.504, | |
| "grad_norm": 0.11914021521806717, | |
| "kl": 0.0745849609375, | |
| "learning_rate": 9.381311511432658e-07, | |
| "loss": -0.0781, | |
| "reward": -0.19801979139447212, | |
| "reward_std": 0.1959025263786316, | |
| "rewards/cosine_scaled_reward": -0.09900989942252636, | |
| "rewards/format_reward": 0.0, | |
| "step": 126 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1415.8512268066406, | |
| "epoch": 0.508, | |
| "grad_norm": 0.1357516646385193, | |
| "kl": 0.07177734375, | |
| "learning_rate": 9.36531953618799e-07, | |
| "loss": -0.0865, | |
| "reward": -0.21713878214359283, | |
| "reward_std": 0.18351096659898758, | |
| "rewards/cosine_scaled_reward": -0.10856938920915127, | |
| "rewards/format_reward": 0.0, | |
| "step": 127 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1472.5000305175781, | |
| "epoch": 0.512, | |
| "grad_norm": 0.1369076818227768, | |
| "kl": 0.05865478515625, | |
| "learning_rate": 9.34913917072228e-07, | |
| "loss": -0.0478, | |
| "reward": -0.21694474667310715, | |
| "reward_std": 0.17777032032608986, | |
| "rewards/cosine_scaled_reward": -0.10847238078713417, | |
| "rewards/format_reward": 0.0, | |
| "step": 128 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1477.2559509277344, | |
| "epoch": 0.516, | |
| "grad_norm": 0.17333942651748657, | |
| "kl": 0.0572509765625, | |
| "learning_rate": 9.332771203643714e-07, | |
| "loss": -0.0249, | |
| "reward": -0.21337437257170677, | |
| "reward_std": 0.18009737133979797, | |
| "rewards/cosine_scaled_reward": -0.10668718256056309, | |
| "rewards/format_reward": 0.0, | |
| "step": 129 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1473.7381286621094, | |
| "epoch": 0.52, | |
| "grad_norm": 0.10983088612556458, | |
| "kl": 0.0703125, | |
| "learning_rate": 9.316216432703916e-07, | |
| "loss": -0.0583, | |
| "reward": -0.19089093804359436, | |
| "reward_std": 0.18549956753849983, | |
| "rewards/cosine_scaled_reward": -0.09544547088444233, | |
| "rewards/format_reward": 0.0, | |
| "step": 130 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1432.7559814453125, | |
| "epoch": 0.524, | |
| "grad_norm": 0.16586177051067352, | |
| "kl": 0.0791015625, | |
| "learning_rate": 9.299475664759068e-07, | |
| "loss": -0.0994, | |
| "reward": -0.23504262417554855, | |
| "reward_std": 0.21443113684654236, | |
| "rewards/cosine_scaled_reward": -0.11752131581306458, | |
| "rewards/format_reward": 0.0, | |
| "step": 131 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1481.9762268066406, | |
| "epoch": 0.528, | |
| "grad_norm": 0.17732541263103485, | |
| "kl": 0.06622314453125, | |
| "learning_rate": 9.282549715730579e-07, | |
| "loss": -0.0324, | |
| "reward": -0.17852769792079926, | |
| "reward_std": 0.1502333115786314, | |
| "rewards/cosine_scaled_reward": -0.08926384896039963, | |
| "rewards/format_reward": 0.0, | |
| "step": 132 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1476.4048156738281, | |
| "epoch": 0.532, | |
| "grad_norm": 0.15965314209461212, | |
| "kl": 0.060546875, | |
| "learning_rate": 9.265439410565328e-07, | |
| "loss": -0.058, | |
| "reward": -0.22374625876545906, | |
| "reward_std": 0.18331025168299675, | |
| "rewards/cosine_scaled_reward": -0.11187312379479408, | |
| "rewards/format_reward": 0.0, | |
| "step": 133 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1490.8452453613281, | |
| "epoch": 0.536, | |
| "grad_norm": 0.11577272415161133, | |
| "kl": 0.0740966796875, | |
| "learning_rate": 9.248145583195447e-07, | |
| "loss": -0.0366, | |
| "reward": -0.1072634905576706, | |
| "reward_std": 0.15717832930386066, | |
| "rewards/cosine_scaled_reward": -0.05363174341619015, | |
| "rewards/format_reward": 0.0, | |
| "step": 134 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1490.982177734375, | |
| "epoch": 0.54, | |
| "grad_norm": 0.14535216987133026, | |
| "kl": 0.05950927734375, | |
| "learning_rate": 9.230669076497687e-07, | |
| "loss": -0.0312, | |
| "reward": -0.21274039149284363, | |
| "reward_std": 0.15816222876310349, | |
| "rewards/cosine_scaled_reward": -0.10637019760906696, | |
| "rewards/format_reward": 0.0, | |
| "step": 135 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1475.7679138183594, | |
| "epoch": 0.544, | |
| "grad_norm": 0.14940175414085388, | |
| "kl": 0.072265625, | |
| "learning_rate": 9.213010742252327e-07, | |
| "loss": -0.0302, | |
| "reward": -0.20727308467030525, | |
| "reward_std": 0.18025333806872368, | |
| "rewards/cosine_scaled_reward": -0.10363654233515263, | |
| "rewards/format_reward": 0.0, | |
| "step": 136 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1497.27978515625, | |
| "epoch": 0.548, | |
| "grad_norm": 0.11683686077594757, | |
| "kl": 0.0675048828125, | |
| "learning_rate": 9.195171441101668e-07, | |
| "loss": -0.0354, | |
| "reward": -0.19653399288654327, | |
| "reward_std": 0.17340604588389397, | |
| "rewards/cosine_scaled_reward": -0.09826699271798134, | |
| "rewards/format_reward": 0.0, | |
| "step": 137 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1472.4524230957031, | |
| "epoch": 0.552, | |
| "grad_norm": 0.19184042513370514, | |
| "kl": 0.0703125, | |
| "learning_rate": 9.177152042508077e-07, | |
| "loss": -0.0365, | |
| "reward": -0.20156748220324516, | |
| "reward_std": 0.16703343763947487, | |
| "rewards/cosine_scaled_reward": -0.10078373923897743, | |
| "rewards/format_reward": 0.0, | |
| "step": 138 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1441.5238342285156, | |
| "epoch": 0.556, | |
| "grad_norm": 0.21460425853729248, | |
| "kl": 0.0751953125, | |
| "learning_rate": 9.158953424711624e-07, | |
| "loss": -0.0713, | |
| "reward": -0.1934008002281189, | |
| "reward_std": 0.1858229860663414, | |
| "rewards/cosine_scaled_reward": -0.096700394526124, | |
| "rewards/format_reward": 0.0, | |
| "step": 139 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1477.5536193847656, | |
| "epoch": 0.56, | |
| "grad_norm": 0.1624855250120163, | |
| "kl": 0.07452392578125, | |
| "learning_rate": 9.140576474687263e-07, | |
| "loss": -0.0644, | |
| "reward": -0.20967105776071548, | |
| "reward_std": 0.17497684434056282, | |
| "rewards/cosine_scaled_reward": -0.1048355270177126, | |
| "rewards/format_reward": 0.0, | |
| "step": 140 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1488.3928833007812, | |
| "epoch": 0.564, | |
| "grad_norm": 0.13148972392082214, | |
| "kl": 0.0745849609375, | |
| "learning_rate": 9.122022088101613e-07, | |
| "loss": -0.0365, | |
| "reward": -0.12153960764408112, | |
| "reward_std": 0.15567267499864101, | |
| "rewards/cosine_scaled_reward": -0.06076979637145996, | |
| "rewards/format_reward": 0.0, | |
| "step": 141 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1439.5536193847656, | |
| "epoch": 0.568, | |
| "grad_norm": 0.18604105710983276, | |
| "kl": 0.0751953125, | |
| "learning_rate": 9.103291169269299e-07, | |
| "loss": -0.0679, | |
| "reward": -0.21637247875332832, | |
| "reward_std": 0.19511258974671364, | |
| "rewards/cosine_scaled_reward": -0.10818623751401901, | |
| "rewards/format_reward": 0.0, | |
| "step": 142 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1486.5774230957031, | |
| "epoch": 0.572, | |
| "grad_norm": 0.17263971269130707, | |
| "kl": 0.07757568359375, | |
| "learning_rate": 9.084384631108882e-07, | |
| "loss": -0.0389, | |
| "reward": -0.2009837031364441, | |
| "reward_std": 0.16309702023863792, | |
| "rewards/cosine_scaled_reward": -0.10049185156822205, | |
| "rewards/format_reward": 0.0, | |
| "step": 143 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1478.3988647460938, | |
| "epoch": 0.576, | |
| "grad_norm": 0.14938153326511383, | |
| "kl": 0.079833984375, | |
| "learning_rate": 9.065303395098358e-07, | |
| "loss": -0.0585, | |
| "reward": -0.1797693967819214, | |
| "reward_std": 0.16727757826447487, | |
| "rewards/cosine_scaled_reward": -0.08988469652831554, | |
| "rewards/format_reward": 0.0, | |
| "step": 144 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1460.5417175292969, | |
| "epoch": 0.58, | |
| "grad_norm": 0.18892593681812286, | |
| "kl": 0.0838623046875, | |
| "learning_rate": 9.046048391230247e-07, | |
| "loss": -0.061, | |
| "reward": -0.1960761584341526, | |
| "reward_std": 0.16931083425879478, | |
| "rewards/cosine_scaled_reward": -0.0980380792170763, | |
| "rewards/format_reward": 0.0, | |
| "step": 145 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1495.2916870117188, | |
| "epoch": 0.584, | |
| "grad_norm": 0.20080548524856567, | |
| "kl": 0.0863037109375, | |
| "learning_rate": 9.026620557966279e-07, | |
| "loss": -0.0189, | |
| "reward": -0.18559397384524345, | |
| "reward_std": 0.16799203678965569, | |
| "rewards/cosine_scaled_reward": -0.09279698692262173, | |
| "rewards/format_reward": 0.0, | |
| "step": 146 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1502.8809814453125, | |
| "epoch": 0.588, | |
| "grad_norm": 0.15992848575115204, | |
| "kl": 0.0850830078125, | |
| "learning_rate": 9.007020842191634e-07, | |
| "loss": -0.0201, | |
| "reward": -0.19452324509620667, | |
| "reward_std": 0.1573326252400875, | |
| "rewards/cosine_scaled_reward": -0.09726162627339363, | |
| "rewards/format_reward": 0.0, | |
| "step": 147 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1463.65478515625, | |
| "epoch": 0.592, | |
| "grad_norm": 0.12942340970039368, | |
| "kl": 0.08453369140625, | |
| "learning_rate": 8.987250199168808e-07, | |
| "loss": -0.0368, | |
| "reward": -0.20581213757395744, | |
| "reward_std": 0.1570763811469078, | |
| "rewards/cosine_scaled_reward": -0.10290606319904327, | |
| "rewards/format_reward": 0.0, | |
| "step": 148 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1493.7619323730469, | |
| "epoch": 0.596, | |
| "grad_norm": 0.18132224678993225, | |
| "kl": 0.0960693359375, | |
| "learning_rate": 8.967309592491052e-07, | |
| "loss": -0.023, | |
| "reward": -0.19545432925224304, | |
| "reward_std": 0.1527048945426941, | |
| "rewards/cosine_scaled_reward": -0.09772716276347637, | |
| "rewards/format_reward": 0.0, | |
| "step": 149 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1462.8452758789062, | |
| "epoch": 0.6, | |
| "grad_norm": 0.17014774680137634, | |
| "kl": 0.1109619140625, | |
| "learning_rate": 8.9471999940354e-07, | |
| "loss": -0.0427, | |
| "reward": -0.203117735683918, | |
| "reward_std": 0.18610898405313492, | |
| "rewards/cosine_scaled_reward": -0.10155886970460415, | |
| "rewards/format_reward": 0.0, | |
| "step": 150 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1477.3393249511719, | |
| "epoch": 0.604, | |
| "grad_norm": 0.15196801722049713, | |
| "kl": 0.1031494140625, | |
| "learning_rate": 8.926922383915315e-07, | |
| "loss": -0.0465, | |
| "reward": -0.20808908715844154, | |
| "reward_std": 0.18062585964798927, | |
| "rewards/cosine_scaled_reward": -0.10404454357922077, | |
| "rewards/format_reward": 0.0, | |
| "step": 151 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1479.3333740234375, | |
| "epoch": 0.608, | |
| "grad_norm": 0.13162937760353088, | |
| "kl": 0.0885009765625, | |
| "learning_rate": 8.906477750432903e-07, | |
| "loss": -0.0501, | |
| "reward": -0.21273208782076836, | |
| "reward_std": 0.16958871111273766, | |
| "rewards/cosine_scaled_reward": -0.10636604763567448, | |
| "rewards/format_reward": 0.0, | |
| "step": 152 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.1904907226562, | |
| "epoch": 0.612, | |
| "grad_norm": 0.15801787376403809, | |
| "kl": 0.0894775390625, | |
| "learning_rate": 8.88586709003076e-07, | |
| "loss": -0.024, | |
| "reward": -0.18563585355877876, | |
| "reward_std": 0.14705245569348335, | |
| "rewards/cosine_scaled_reward": -0.09281792864203453, | |
| "rewards/format_reward": 0.0, | |
| "step": 153 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1487.75, | |
| "epoch": 0.616, | |
| "grad_norm": 0.19425953924655914, | |
| "kl": 0.1038818359375, | |
| "learning_rate": 8.865091407243394e-07, | |
| "loss": -0.0361, | |
| "reward": -0.18547611683607101, | |
| "reward_std": 0.1538998931646347, | |
| "rewards/cosine_scaled_reward": -0.09273805841803551, | |
| "rewards/format_reward": 0.0, | |
| "step": 154 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1479.5595703125, | |
| "epoch": 0.62, | |
| "grad_norm": 0.13653838634490967, | |
| "kl": 0.1011962890625, | |
| "learning_rate": 8.844151714648274e-07, | |
| "loss": -0.0666, | |
| "reward": -0.20050010830163956, | |
| "reward_std": 0.15926911309361458, | |
| "rewards/cosine_scaled_reward": -0.10025005042552948, | |
| "rewards/format_reward": 0.0, | |
| "step": 155 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.6071472167969, | |
| "epoch": 0.624, | |
| "grad_norm": 0.1796942949295044, | |
| "kl": 0.1038818359375, | |
| "learning_rate": 8.823049032816478e-07, | |
| "loss": -0.0216, | |
| "reward": -0.1935092769563198, | |
| "reward_std": 0.17067047394812107, | |
| "rewards/cosine_scaled_reward": -0.09675464034080505, | |
| "rewards/format_reward": 0.0, | |
| "step": 156 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1490.2679138183594, | |
| "epoch": 0.628, | |
| "grad_norm": 0.15652166306972504, | |
| "kl": 0.0980224609375, | |
| "learning_rate": 8.801784390262943e-07, | |
| "loss": -0.0199, | |
| "reward": -0.10874435119330883, | |
| "reward_std": 0.15503624081611633, | |
| "rewards/cosine_scaled_reward": -0.05437217652797699, | |
| "rewards/format_reward": 0.0, | |
| "step": 157 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1478.327392578125, | |
| "epoch": 0.632, | |
| "grad_norm": 0.10631345212459564, | |
| "kl": 0.10205078125, | |
| "learning_rate": 8.780358823396352e-07, | |
| "loss": -0.0558, | |
| "reward": -0.19594154134392738, | |
| "reward_std": 0.18128735944628716, | |
| "rewards/cosine_scaled_reward": -0.09797077253460884, | |
| "rewards/format_reward": 0.0, | |
| "step": 158 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1492.0059814453125, | |
| "epoch": 0.636, | |
| "grad_norm": 0.1802942007780075, | |
| "kl": 0.1143798828125, | |
| "learning_rate": 8.758773376468604e-07, | |
| "loss": -0.045, | |
| "reward": -0.17765655368566513, | |
| "reward_std": 0.16990270093083382, | |
| "rewards/cosine_scaled_reward": -0.08882827498018742, | |
| "rewards/format_reward": 0.0, | |
| "step": 159 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1490.7798156738281, | |
| "epoch": 0.64, | |
| "grad_norm": 0.09826915711164474, | |
| "kl": 0.1187744140625, | |
| "learning_rate": 8.737029101523929e-07, | |
| "loss": -0.0474, | |
| "reward": -0.15192299336194992, | |
| "reward_std": 0.1335773952305317, | |
| "rewards/cosine_scaled_reward": -0.07596149481832981, | |
| "rewards/format_reward": 0.0, | |
| "step": 160 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1518.3035888671875, | |
| "epoch": 0.644, | |
| "grad_norm": 0.14415128529071808, | |
| "kl": 0.1134033203125, | |
| "learning_rate": 8.715127058347614e-07, | |
| "loss": -0.0181, | |
| "reward": -0.16958895698189735, | |
| "reward_std": 0.14540697447955608, | |
| "rewards/cosine_scaled_reward": -0.08479447849094868, | |
| "rewards/format_reward": 0.0, | |
| "step": 161 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1478.0714416503906, | |
| "epoch": 0.648, | |
| "grad_norm": 0.13143935799598694, | |
| "kl": 0.1153564453125, | |
| "learning_rate": 8.693068314414344e-07, | |
| "loss": -0.0602, | |
| "reward": -0.17831872776150703, | |
| "reward_std": 0.16582145914435387, | |
| "rewards/cosine_scaled_reward": -0.08915936388075352, | |
| "rewards/format_reward": 0.0, | |
| "step": 162 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1500.261962890625, | |
| "epoch": 0.652, | |
| "grad_norm": 0.2650466561317444, | |
| "kl": 0.1025390625, | |
| "learning_rate": 8.670853944836176e-07, | |
| "loss": -0.0444, | |
| "reward": -0.19617021456360817, | |
| "reward_std": 0.16974329948425293, | |
| "rewards/cosine_scaled_reward": -0.09808510728180408, | |
| "rewards/format_reward": 0.0, | |
| "step": 163 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1476.3630981445312, | |
| "epoch": 0.656, | |
| "grad_norm": 0.1531592160463333, | |
| "kl": 0.1103515625, | |
| "learning_rate": 8.648485032310144e-07, | |
| "loss": -0.07, | |
| "reward": -0.17511003464460373, | |
| "reward_std": 0.16592327691614628, | |
| "rewards/cosine_scaled_reward": -0.08755501732230186, | |
| "rewards/format_reward": 0.0, | |
| "step": 164 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1491.1845703125, | |
| "epoch": 0.66, | |
| "grad_norm": 0.11288218945264816, | |
| "kl": 0.117919921875, | |
| "learning_rate": 8.625962667065487e-07, | |
| "loss": -0.0371, | |
| "reward": -0.19623659178614616, | |
| "reward_std": 0.15025305189192295, | |
| "rewards/cosine_scaled_reward": -0.09811829589307308, | |
| "rewards/format_reward": 0.0, | |
| "step": 165 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1496.6607055664062, | |
| "epoch": 0.664, | |
| "grad_norm": 0.1232253834605217, | |
| "kl": 0.115234375, | |
| "learning_rate": 8.603287946810513e-07, | |
| "loss": -0.0475, | |
| "reward": -0.19438259676098824, | |
| "reward_std": 0.1660812869668007, | |
| "rewards/cosine_scaled_reward": -0.09719130024313927, | |
| "rewards/format_reward": 0.0, | |
| "step": 166 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.577392578125, | |
| "epoch": 0.668, | |
| "grad_norm": 0.11001460999250412, | |
| "kl": 0.1339111328125, | |
| "learning_rate": 8.580461976679099e-07, | |
| "loss": -0.0191, | |
| "reward": -0.07667672634124756, | |
| "reward_std": 0.14517304301261902, | |
| "rewards/cosine_scaled_reward": -0.03833836503326893, | |
| "rewards/format_reward": 0.0, | |
| "step": 167 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.4345397949219, | |
| "epoch": 0.672, | |
| "grad_norm": 0.12665332853794098, | |
| "kl": 0.12939453125, | |
| "learning_rate": 8.557485869176825e-07, | |
| "loss": -0.0262, | |
| "reward": -0.18039095029234886, | |
| "reward_std": 0.14454744383692741, | |
| "rewards/cosine_scaled_reward": -0.09019547514617443, | |
| "rewards/format_reward": 0.0, | |
| "step": 168 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1505.3809814453125, | |
| "epoch": 0.676, | |
| "grad_norm": 0.10181339830160141, | |
| "kl": 0.1201171875, | |
| "learning_rate": 8.534360744126753e-07, | |
| "loss": -0.0296, | |
| "reward": -0.15367550402879715, | |
| "reward_std": 0.154752716422081, | |
| "rewards/cosine_scaled_reward": -0.07683775387704372, | |
| "rewards/format_reward": 0.0, | |
| "step": 169 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1502.2678833007812, | |
| "epoch": 0.68, | |
| "grad_norm": 0.08636263757944107, | |
| "kl": 0.140380859375, | |
| "learning_rate": 8.511087728614862e-07, | |
| "loss": -0.0405, | |
| "reward": -0.14581535942852497, | |
| "reward_std": 0.1418076604604721, | |
| "rewards/cosine_scaled_reward": -0.07290767971426249, | |
| "rewards/format_reward": 0.0, | |
| "step": 170 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1503.3154907226562, | |
| "epoch": 0.684, | |
| "grad_norm": 0.1574849933385849, | |
| "kl": 0.1326904296875, | |
| "learning_rate": 8.487667956935087e-07, | |
| "loss": -0.0257, | |
| "reward": -0.14666364155709743, | |
| "reward_std": 0.14365333877503872, | |
| "rewards/cosine_scaled_reward": -0.07333182357251644, | |
| "rewards/format_reward": 0.0, | |
| "step": 171 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1501.3690490722656, | |
| "epoch": 0.688, | |
| "grad_norm": 0.11679176241159439, | |
| "kl": 0.1322021484375, | |
| "learning_rate": 8.464102570534061e-07, | |
| "loss": -0.0394, | |
| "reward": -0.1798371747136116, | |
| "reward_std": 0.1620156615972519, | |
| "rewards/cosine_scaled_reward": -0.0899185873568058, | |
| "rewards/format_reward": 0.0, | |
| "step": 172 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1489.6845703125, | |
| "epoch": 0.692, | |
| "grad_norm": 0.143167644739151, | |
| "kl": 0.1221923828125, | |
| "learning_rate": 8.440392717955475e-07, | |
| "loss": -0.0454, | |
| "reward": -0.11209908872842789, | |
| "reward_std": 0.15652650594711304, | |
| "rewards/cosine_scaled_reward": -0.05604954622685909, | |
| "rewards/format_reward": 0.0, | |
| "step": 173 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.1607360839844, | |
| "epoch": 0.696, | |
| "grad_norm": 0.13327138125896454, | |
| "kl": 0.1173095703125, | |
| "learning_rate": 8.416539554784089e-07, | |
| "loss": -0.0349, | |
| "reward": -0.18605408817529678, | |
| "reward_std": 0.17399531230330467, | |
| "rewards/cosine_scaled_reward": -0.09302704595029354, | |
| "rewards/format_reward": 0.0, | |
| "step": 174 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1500.1309509277344, | |
| "epoch": 0.7, | |
| "grad_norm": 0.12825773656368256, | |
| "kl": 0.1453857421875, | |
| "learning_rate": 8.392544243589427e-07, | |
| "loss": -0.0389, | |
| "reward": -0.18417096138000488, | |
| "reward_std": 0.15830742567777634, | |
| "rewards/cosine_scaled_reward": -0.09208548441529274, | |
| "rewards/format_reward": 0.0, | |
| "step": 175 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1512.1488342285156, | |
| "epoch": 0.704, | |
| "grad_norm": 0.19698651134967804, | |
| "kl": 0.12744140625, | |
| "learning_rate": 8.368407953869103e-07, | |
| "loss": -0.0278, | |
| "reward": -0.1590665504336357, | |
| "reward_std": 0.1343939360231161, | |
| "rewards/cosine_scaled_reward": -0.0795332733541727, | |
| "rewards/format_reward": 0.0, | |
| "step": 176 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.4583435058594, | |
| "epoch": 0.708, | |
| "grad_norm": 0.11850512772798538, | |
| "kl": 0.12353515625, | |
| "learning_rate": 8.344131861991828e-07, | |
| "loss": -0.0209, | |
| "reward": -0.17914509028196335, | |
| "reward_std": 0.14407609589397907, | |
| "rewards/cosine_scaled_reward": -0.08957254886627197, | |
| "rewards/format_reward": 0.0, | |
| "step": 177 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1494.6785888671875, | |
| "epoch": 0.712, | |
| "grad_norm": 0.12900114059448242, | |
| "kl": 0.1517333984375, | |
| "learning_rate": 8.319717151140072e-07, | |
| "loss": -0.0476, | |
| "reward": -0.1623889021575451, | |
| "reward_std": 0.1595832072198391, | |
| "rewards/cosine_scaled_reward": -0.08119445107877254, | |
| "rewards/format_reward": 0.0, | |
| "step": 178 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.1309509277344, | |
| "epoch": 0.716, | |
| "grad_norm": 0.14418767392635345, | |
| "kl": 0.1248779296875, | |
| "learning_rate": 8.295165011252396e-07, | |
| "loss": -0.0145, | |
| "reward": -0.15135440602898598, | |
| "reward_std": 0.12371071800589561, | |
| "rewards/cosine_scaled_reward": -0.07567720301449299, | |
| "rewards/format_reward": 0.0, | |
| "step": 179 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.607177734375, | |
| "epoch": 0.72, | |
| "grad_norm": 0.11457547545433044, | |
| "kl": 0.1297607421875, | |
| "learning_rate": 8.270476638965461e-07, | |
| "loss": -0.0282, | |
| "reward": -0.1590440645813942, | |
| "reward_std": 0.13810313865542412, | |
| "rewards/cosine_scaled_reward": -0.07952203415334225, | |
| "rewards/format_reward": 0.0, | |
| "step": 180 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.702392578125, | |
| "epoch": 0.724, | |
| "grad_norm": 0.16079466044902802, | |
| "kl": 0.1300048828125, | |
| "learning_rate": 8.245653237555705e-07, | |
| "loss": -0.0259, | |
| "reward": -0.14657551050186157, | |
| "reward_std": 0.12384105287492275, | |
| "rewards/cosine_scaled_reward": -0.07328775525093079, | |
| "rewards/format_reward": 0.0, | |
| "step": 181 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.0595703125, | |
| "epoch": 0.728, | |
| "grad_norm": 0.12367437779903412, | |
| "kl": 0.13720703125, | |
| "learning_rate": 8.220696016880687e-07, | |
| "loss": -0.0248, | |
| "reward": -0.15343056619167328, | |
| "reward_std": 0.14851927012205124, | |
| "rewards/cosine_scaled_reward": -0.07671528309583664, | |
| "rewards/format_reward": 0.0, | |
| "step": 182 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1522.5297546386719, | |
| "epoch": 0.732, | |
| "grad_norm": 0.10700845718383789, | |
| "kl": 0.128662109375, | |
| "learning_rate": 8.195606193320136e-07, | |
| "loss": -0.0107, | |
| "reward": -0.14938092976808548, | |
| "reward_std": 0.11783361062407494, | |
| "rewards/cosine_scaled_reward": -0.07469046581536531, | |
| "rewards/format_reward": 0.0, | |
| "step": 183 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.9226379394531, | |
| "epoch": 0.736, | |
| "grad_norm": 0.13011117279529572, | |
| "kl": 0.1298828125, | |
| "learning_rate": 8.170384989716657e-07, | |
| "loss": -0.0148, | |
| "reward": -0.15839479491114616, | |
| "reward_std": 0.12780537828803062, | |
| "rewards/cosine_scaled_reward": -0.07919739931821823, | |
| "rewards/format_reward": 0.0, | |
| "step": 184 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.7797546386719, | |
| "epoch": 0.74, | |
| "grad_norm": 0.08933281898498535, | |
| "kl": 0.152099609375, | |
| "learning_rate": 8.145033635316128e-07, | |
| "loss": -0.0141, | |
| "reward": -0.1524551585316658, | |
| "reward_std": 0.12427524663507938, | |
| "rewards/cosine_scaled_reward": -0.07622758112847805, | |
| "rewards/format_reward": 0.0, | |
| "step": 185 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.3809509277344, | |
| "epoch": 0.744, | |
| "grad_norm": 0.12463698536157608, | |
| "kl": 0.1455078125, | |
| "learning_rate": 8.119553365707802e-07, | |
| "loss": -0.0308, | |
| "reward": -0.1026168204843998, | |
| "reward_std": 0.13808677345514297, | |
| "rewards/cosine_scaled_reward": -0.051308413967490196, | |
| "rewards/format_reward": 0.0, | |
| "step": 186 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1466.6845397949219, | |
| "epoch": 0.748, | |
| "grad_norm": 0.11249163746833801, | |
| "kl": 0.13720703125, | |
| "learning_rate": 8.093945422764069e-07, | |
| "loss": -0.0544, | |
| "reward": -0.16347405686974525, | |
| "reward_std": 0.14502743259072304, | |
| "rewards/cosine_scaled_reward": -0.08173702843487263, | |
| "rewards/format_reward": 0.0, | |
| "step": 187 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1492.2976684570312, | |
| "epoch": 0.752, | |
| "grad_norm": 0.08640649914741516, | |
| "kl": 0.151123046875, | |
| "learning_rate": 8.068211054579943e-07, | |
| "loss": -0.0329, | |
| "reward": -0.16077740490436554, | |
| "reward_std": 0.15286827459931374, | |
| "rewards/cosine_scaled_reward": -0.08038870431482792, | |
| "rewards/format_reward": 0.0, | |
| "step": 188 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.9643249511719, | |
| "epoch": 0.756, | |
| "grad_norm": 0.10966922342777252, | |
| "kl": 0.1417236328125, | |
| "learning_rate": 8.04235151541222e-07, | |
| "loss": -0.0235, | |
| "reward": -0.1736162230372429, | |
| "reward_std": 0.1491672247648239, | |
| "rewards/cosine_scaled_reward": -0.08680811524391174, | |
| "rewards/format_reward": 0.0, | |
| "step": 189 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1518.5833740234375, | |
| "epoch": 0.76, | |
| "grad_norm": 0.12140454351902008, | |
| "kl": 0.1397705078125, | |
| "learning_rate": 8.01636806561836e-07, | |
| "loss": -0.0182, | |
| "reward": -0.15859121643006802, | |
| "reward_std": 0.13637281768023968, | |
| "rewards/cosine_scaled_reward": -0.07929560728371143, | |
| "rewards/format_reward": 0.0, | |
| "step": 190 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1518.4167175292969, | |
| "epoch": 0.764, | |
| "grad_norm": 0.12270302325487137, | |
| "kl": 0.159423828125, | |
| "learning_rate": 7.990261971595048e-07, | |
| "loss": -0.0061, | |
| "reward": -0.1657138504087925, | |
| "reward_std": 0.1241717990487814, | |
| "rewards/cosine_scaled_reward": -0.08285692892968655, | |
| "rewards/format_reward": 0.0, | |
| "step": 191 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.9345397949219, | |
| "epoch": 0.768, | |
| "grad_norm": 0.12375301867723465, | |
| "kl": 0.162353515625, | |
| "learning_rate": 7.964034505716476e-07, | |
| "loss": -0.0398, | |
| "reward": -0.17186777852475643, | |
| "reward_std": 0.16331871785223484, | |
| "rewards/cosine_scaled_reward": -0.08593388926237822, | |
| "rewards/format_reward": 0.0, | |
| "step": 192 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1497.3452758789062, | |
| "epoch": 0.772, | |
| "grad_norm": 0.08711956441402435, | |
| "kl": 0.152587890625, | |
| "learning_rate": 7.93768694627233e-07, | |
| "loss": -0.0287, | |
| "reward": -0.15793642029166222, | |
| "reward_std": 0.1320202425122261, | |
| "rewards/cosine_scaled_reward": -0.07896821573376656, | |
| "rewards/format_reward": 0.0, | |
| "step": 193 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1517.7976379394531, | |
| "epoch": 0.776, | |
| "grad_norm": 0.0980345830321312, | |
| "kl": 0.14990234375, | |
| "learning_rate": 7.911220577405484e-07, | |
| "loss": -0.015, | |
| "reward": -0.15146314911544323, | |
| "reward_std": 0.13276733830571175, | |
| "rewards/cosine_scaled_reward": -0.07573157269507647, | |
| "rewards/format_reward": 0.0, | |
| "step": 194 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1509.5595397949219, | |
| "epoch": 0.78, | |
| "grad_norm": 0.19111458957195282, | |
| "kl": 0.14404296875, | |
| "learning_rate": 7.884636689049422e-07, | |
| "loss": -0.027, | |
| "reward": -0.1567194815725088, | |
| "reward_std": 0.14454844780266285, | |
| "rewards/cosine_scaled_reward": -0.07835974264889956, | |
| "rewards/format_reward": 0.0, | |
| "step": 195 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1503.6785888671875, | |
| "epoch": 0.784, | |
| "grad_norm": 0.10002721846103668, | |
| "kl": 0.150634765625, | |
| "learning_rate": 7.857936576865356e-07, | |
| "loss": -0.0234, | |
| "reward": -0.17957428470253944, | |
| "reward_std": 0.14822976663708687, | |
| "rewards/cosine_scaled_reward": -0.08978714607656002, | |
| "rewards/format_reward": 0.0, | |
| "step": 196 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1529.0, | |
| "epoch": 0.788, | |
| "grad_norm": 0.1074904203414917, | |
| "kl": 0.15283203125, | |
| "learning_rate": 7.831121542179086e-07, | |
| "loss": -0.0043, | |
| "reward": -0.135637816041708, | |
| "reward_std": 0.10331238061189651, | |
| "rewards/cosine_scaled_reward": -0.06781890522688627, | |
| "rewards/format_reward": 0.0, | |
| "step": 197 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1485.7738342285156, | |
| "epoch": 0.792, | |
| "grad_norm": 0.09117424488067627, | |
| "kl": 0.1513671875, | |
| "learning_rate": 7.804192891917571e-07, | |
| "loss": -0.0363, | |
| "reward": -0.17118510231375694, | |
| "reward_std": 0.16877446696162224, | |
| "rewards/cosine_scaled_reward": -0.08559254929423332, | |
| "rewards/format_reward": 0.0, | |
| "step": 198 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.6607055664062, | |
| "epoch": 0.796, | |
| "grad_norm": 0.10829849541187286, | |
| "kl": 0.14306640625, | |
| "learning_rate": 7.777151938545235e-07, | |
| "loss": -0.0227, | |
| "reward": -0.15547415241599083, | |
| "reward_std": 0.12111644446849823, | |
| "rewards/cosine_scaled_reward": -0.07773707807064056, | |
| "rewards/format_reward": 0.0, | |
| "step": 199 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.5476379394531, | |
| "epoch": 0.8, | |
| "grad_norm": 0.17340432107448578, | |
| "kl": 0.1441650390625, | |
| "learning_rate": 7.75e-07, | |
| "loss": -0.0141, | |
| "reward": -0.15766149759292603, | |
| "reward_std": 0.13325241580605507, | |
| "rewards/cosine_scaled_reward": -0.07883074693381786, | |
| "rewards/format_reward": 0.0, | |
| "step": 200 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1517.1785888671875, | |
| "epoch": 0.804, | |
| "grad_norm": 0.12855444848537445, | |
| "kl": 0.158447265625, | |
| "learning_rate": 7.72273839962904e-07, | |
| "loss": -0.0166, | |
| "reward": -0.14288493990898132, | |
| "reward_std": 0.1141566876322031, | |
| "rewards/cosine_scaled_reward": -0.07144246716052294, | |
| "rewards/format_reward": 0.0, | |
| "step": 201 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1492.6488342285156, | |
| "epoch": 0.808, | |
| "grad_norm": 0.09727565199136734, | |
| "kl": 0.163818359375, | |
| "learning_rate": 7.695368466124296e-07, | |
| "loss": -0.046, | |
| "reward": -0.1400277316570282, | |
| "reward_std": 0.1363294217735529, | |
| "rewards/cosine_scaled_reward": -0.07001386396586895, | |
| "rewards/format_reward": 0.0, | |
| "step": 202 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1502.5416564941406, | |
| "epoch": 0.812, | |
| "grad_norm": 0.1032966673374176, | |
| "kl": 0.166015625, | |
| "learning_rate": 7.667891533457718e-07, | |
| "loss": -0.0224, | |
| "reward": -0.1764848232269287, | |
| "reward_std": 0.15339024364948273, | |
| "rewards/cosine_scaled_reward": -0.08824240230023861, | |
| "rewards/format_reward": 0.0, | |
| "step": 203 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.7202453613281, | |
| "epoch": 0.816, | |
| "grad_norm": 0.15399117767810822, | |
| "kl": 0.175537109375, | |
| "learning_rate": 7.640308940816239e-07, | |
| "loss": -0.0163, | |
| "reward": -0.14551853574812412, | |
| "reward_std": 0.14542756974697113, | |
| "rewards/cosine_scaled_reward": -0.07275926228612661, | |
| "rewards/format_reward": 0.0, | |
| "step": 204 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1509.761962890625, | |
| "epoch": 0.82, | |
| "grad_norm": 0.0828375294804573, | |
| "kl": 0.165283203125, | |
| "learning_rate": 7.612622032536507e-07, | |
| "loss": -0.0178, | |
| "reward": -0.15090281143784523, | |
| "reward_std": 0.13103190064430237, | |
| "rewards/cosine_scaled_reward": -0.07545140571892262, | |
| "rewards/format_reward": 0.0, | |
| "step": 205 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1530.4583435058594, | |
| "epoch": 0.824, | |
| "grad_norm": 0.09723107516765594, | |
| "kl": 0.156005859375, | |
| "learning_rate": 7.584832158039378e-07, | |
| "loss": 0.0019, | |
| "reward": -0.13943813741207123, | |
| "reward_std": 0.12253955751657486, | |
| "rewards/cosine_scaled_reward": -0.06971907056868076, | |
| "rewards/format_reward": 0.0, | |
| "step": 206 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.8928833007812, | |
| "epoch": 0.828, | |
| "grad_norm": 0.10988614708185196, | |
| "kl": 0.16162109375, | |
| "learning_rate": 7.556940671764124e-07, | |
| "loss": -0.0272, | |
| "reward": -0.15701762214303017, | |
| "reward_std": 0.1170128583908081, | |
| "rewards/cosine_scaled_reward": -0.07850880734622478, | |
| "rewards/format_reward": 0.0, | |
| "step": 207 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1525.5952453613281, | |
| "epoch": 0.832, | |
| "grad_norm": 0.11009800434112549, | |
| "kl": 0.171142578125, | |
| "learning_rate": 7.528948933102438e-07, | |
| "loss": 0.0156, | |
| "reward": -0.14172504469752312, | |
| "reward_std": 0.11965266987681389, | |
| "rewards/cosine_scaled_reward": -0.07086252421140671, | |
| "rewards/format_reward": 0.0, | |
| "step": 208 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1520.1011962890625, | |
| "epoch": 0.836, | |
| "grad_norm": 0.15401668846607208, | |
| "kl": 0.16455078125, | |
| "learning_rate": 7.500858306332172e-07, | |
| "loss": -0.0106, | |
| "reward": -0.06414606049656868, | |
| "reward_std": 0.12946532107889652, | |
| "rewards/cosine_scaled_reward": -0.032073031179606915, | |
| "rewards/format_reward": 0.0, | |
| "step": 209 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1522.232177734375, | |
| "epoch": 0.84, | |
| "grad_norm": 0.12647868692874908, | |
| "kl": 0.160400390625, | |
| "learning_rate": 7.472670160550848e-07, | |
| "loss": 0.0009, | |
| "reward": -0.12890829890966415, | |
| "reward_std": 0.1113775297999382, | |
| "rewards/cosine_scaled_reward": -0.06445414572954178, | |
| "rewards/format_reward": 0.0, | |
| "step": 210 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1495.9405212402344, | |
| "epoch": 0.844, | |
| "grad_norm": 0.09751134365797043, | |
| "kl": 0.1451416015625, | |
| "learning_rate": 7.444385869608921e-07, | |
| "loss": -0.0263, | |
| "reward": -0.15089180506765842, | |
| "reward_std": 0.12495366670191288, | |
| "rewards/cosine_scaled_reward": -0.07544590625911951, | |
| "rewards/format_reward": 0.0, | |
| "step": 211 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1517.6845397949219, | |
| "epoch": 0.848, | |
| "grad_norm": 0.10632960498332977, | |
| "kl": 0.172607421875, | |
| "learning_rate": 7.416006812042827e-07, | |
| "loss": 0.0114, | |
| "reward": -0.12592120468616486, | |
| "reward_std": 0.10658309236168861, | |
| "rewards/cosine_scaled_reward": -0.06296060606837273, | |
| "rewards/format_reward": 0.0, | |
| "step": 212 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.1726379394531, | |
| "epoch": 0.852, | |
| "grad_norm": 0.08701854944229126, | |
| "kl": 0.173095703125, | |
| "learning_rate": 7.387534371007797e-07, | |
| "loss": -0.0094, | |
| "reward": -0.04494331777095795, | |
| "reward_std": 0.10984733514487743, | |
| "rewards/cosine_scaled_reward": -0.022471658885478973, | |
| "rewards/format_reward": 0.0, | |
| "step": 213 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.8392944335938, | |
| "epoch": 0.856, | |
| "grad_norm": 0.10350044071674347, | |
| "kl": 0.1650390625, | |
| "learning_rate": 7.358969934210438e-07, | |
| "loss": -0.0193, | |
| "reward": -0.15005925297737122, | |
| "reward_std": 0.1357622630894184, | |
| "rewards/cosine_scaled_reward": -0.07502962276339531, | |
| "rewards/format_reward": 0.0, | |
| "step": 214 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1500.7738342285156, | |
| "epoch": 0.86, | |
| "grad_norm": 0.08610279858112335, | |
| "kl": 0.17724609375, | |
| "learning_rate": 7.330314893841101e-07, | |
| "loss": -0.0337, | |
| "reward": -0.06414718553423882, | |
| "reward_std": 0.1431224588304758, | |
| "rewards/cosine_scaled_reward": -0.03207359462976456, | |
| "rewards/format_reward": 0.0, | |
| "step": 215 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1495.6667175292969, | |
| "epoch": 0.864, | |
| "grad_norm": 0.0906975269317627, | |
| "kl": 0.17041015625, | |
| "learning_rate": 7.301570646506027e-07, | |
| "loss": -0.0444, | |
| "reward": -0.14620519801974297, | |
| "reward_std": 0.1489107757806778, | |
| "rewards/cosine_scaled_reward": -0.07310259714722633, | |
| "rewards/format_reward": 0.0, | |
| "step": 216 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1501.3810119628906, | |
| "epoch": 0.868, | |
| "grad_norm": 0.08579116314649582, | |
| "kl": 0.16357421875, | |
| "learning_rate": 7.27273859315928e-07, | |
| "loss": -0.0184, | |
| "reward": -0.1731514111161232, | |
| "reward_std": 0.16837183013558388, | |
| "rewards/cosine_scaled_reward": -0.0865757018327713, | |
| "rewards/format_reward": 0.0, | |
| "step": 217 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1524.6488342285156, | |
| "epoch": 0.872, | |
| "grad_norm": 0.0806507021188736, | |
| "kl": 0.1669921875, | |
| "learning_rate": 7.243820139034464e-07, | |
| "loss": -0.0038, | |
| "reward": -0.12116867862641811, | |
| "reward_std": 0.10355196706950665, | |
| "rewards/cosine_scaled_reward": -0.060584343038499355, | |
| "rewards/format_reward": 0.0, | |
| "step": 218 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1489.3809814453125, | |
| "epoch": 0.876, | |
| "grad_norm": 0.07953932136297226, | |
| "kl": 0.18017578125, | |
| "learning_rate": 7.214816693576234e-07, | |
| "loss": -0.037, | |
| "reward": -0.15147988684475422, | |
| "reward_std": 0.13851544074714184, | |
| "rewards/cosine_scaled_reward": -0.07573994528502226, | |
| "rewards/format_reward": 0.0, | |
| "step": 219 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.6071472167969, | |
| "epoch": 0.88, | |
| "grad_norm": 0.09334682673215866, | |
| "kl": 0.161376953125, | |
| "learning_rate": 7.185729670371604e-07, | |
| "loss": -0.0124, | |
| "reward": -0.14643241465091705, | |
| "reward_std": 0.12924424931406975, | |
| "rewards/cosine_scaled_reward": -0.07321620732545853, | |
| "rewards/format_reward": 0.0, | |
| "step": 220 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1530.5357360839844, | |
| "epoch": 0.884, | |
| "grad_norm": 0.15875208377838135, | |
| "kl": 0.1796875, | |
| "learning_rate": 7.156560487081051e-07, | |
| "loss": -0.0004, | |
| "reward": -0.12105057947337627, | |
| "reward_std": 0.11511022225022316, | |
| "rewards/cosine_scaled_reward": -0.060525291599333286, | |
| "rewards/format_reward": 0.0, | |
| "step": 221 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1471.8214721679688, | |
| "epoch": 0.888, | |
| "grad_norm": 0.09343700110912323, | |
| "kl": 0.166015625, | |
| "learning_rate": 7.127310565369415e-07, | |
| "loss": -0.0241, | |
| "reward": -0.1482422910630703, | |
| "reward_std": 0.13543636724352837, | |
| "rewards/cosine_scaled_reward": -0.07412114553153515, | |
| "rewards/format_reward": 0.0, | |
| "step": 222 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.8214416503906, | |
| "epoch": 0.892, | |
| "grad_norm": 0.12307551503181458, | |
| "kl": 0.165771484375, | |
| "learning_rate": 7.097981330836616e-07, | |
| "loss": -0.0248, | |
| "reward": -0.12069513462483883, | |
| "reward_std": 0.13596191070973873, | |
| "rewards/cosine_scaled_reward": -0.060347567312419415, | |
| "rewards/format_reward": 0.0, | |
| "step": 223 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1536.0, | |
| "epoch": 0.896, | |
| "grad_norm": 0.1183975487947464, | |
| "kl": 0.153564453125, | |
| "learning_rate": 7.068574212948169e-07, | |
| "loss": 0.0062, | |
| "reward": -0.14491389319300652, | |
| "reward_std": 0.1196140144020319, | |
| "rewards/cosine_scaled_reward": -0.07245695032179356, | |
| "rewards/format_reward": 0.0, | |
| "step": 224 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1497.4583435058594, | |
| "epoch": 0.9, | |
| "grad_norm": 0.0833095982670784, | |
| "kl": 0.1728515625, | |
| "learning_rate": 7.039090644965509e-07, | |
| "loss": -0.0371, | |
| "reward": -0.13499305956065655, | |
| "reward_std": 0.12412836588919163, | |
| "rewards/cosine_scaled_reward": -0.06749652978032827, | |
| "rewards/format_reward": 0.0, | |
| "step": 225 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1496.6845397949219, | |
| "epoch": 0.904, | |
| "grad_norm": 0.08985927700996399, | |
| "kl": 0.16650390625, | |
| "learning_rate": 7.009532063876148e-07, | |
| "loss": -0.036, | |
| "reward": -0.13741241209208965, | |
| "reward_std": 0.13803981989622116, | |
| "rewards/cosine_scaled_reward": -0.06870621163398027, | |
| "rewards/format_reward": 0.0, | |
| "step": 226 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1524.4047546386719, | |
| "epoch": 0.908, | |
| "grad_norm": 0.0919218361377716, | |
| "kl": 0.16455078125, | |
| "learning_rate": 6.979899910323624e-07, | |
| "loss": -0.0092, | |
| "reward": -0.1429591029882431, | |
| "reward_std": 0.12002728693187237, | |
| "rewards/cosine_scaled_reward": -0.0714795533567667, | |
| "rewards/format_reward": 0.0, | |
| "step": 227 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1503.6607360839844, | |
| "epoch": 0.912, | |
| "grad_norm": 0.11030226945877075, | |
| "kl": 0.1650390625, | |
| "learning_rate": 6.950195628537299e-07, | |
| "loss": -0.0386, | |
| "reward": -0.13499593548476696, | |
| "reward_std": 0.13771596364676952, | |
| "rewards/cosine_scaled_reward": -0.06749796774238348, | |
| "rewards/format_reward": 0.0, | |
| "step": 228 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1488.8214416503906, | |
| "epoch": 0.916, | |
| "grad_norm": 0.09756813943386078, | |
| "kl": 0.1806640625, | |
| "learning_rate": 6.920420666261961e-07, | |
| "loss": -0.0537, | |
| "reward": -0.13239642046391964, | |
| "reward_std": 0.14560103975236416, | |
| "rewards/cosine_scaled_reward": -0.06619821395725012, | |
| "rewards/format_reward": 0.0, | |
| "step": 229 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1520.875, | |
| "epoch": 0.92, | |
| "grad_norm": 0.13375338912010193, | |
| "kl": 0.183349609375, | |
| "learning_rate": 6.890576474687263e-07, | |
| "loss": -0.0143, | |
| "reward": -0.11103706806898117, | |
| "reward_std": 0.11034407652914524, | |
| "rewards/cosine_scaled_reward": -0.05551853682845831, | |
| "rewards/format_reward": 0.0, | |
| "step": 230 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.6369323730469, | |
| "epoch": 0.924, | |
| "grad_norm": 0.09202142059803009, | |
| "kl": 0.14892578125, | |
| "learning_rate": 6.860664508377001e-07, | |
| "loss": -0.0339, | |
| "reward": -0.14039192162454128, | |
| "reward_std": 0.13506866246461868, | |
| "rewards/cosine_scaled_reward": -0.07019595894962549, | |
| "rewards/format_reward": 0.0, | |
| "step": 231 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1520.5476379394531, | |
| "epoch": 0.928, | |
| "grad_norm": 0.098211869597435, | |
| "kl": 0.1650390625, | |
| "learning_rate": 6.83068622519821e-07, | |
| "loss": -0.0127, | |
| "reward": -0.12785408459603786, | |
| "reward_std": 0.12025357596576214, | |
| "rewards/cosine_scaled_reward": -0.06392704229801893, | |
| "rewards/format_reward": 0.0, | |
| "step": 232 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.5000305175781, | |
| "epoch": 0.932, | |
| "grad_norm": 0.1020435318350792, | |
| "kl": 0.172119140625, | |
| "learning_rate": 6.800643086250121e-07, | |
| "loss": -0.0108, | |
| "reward": -0.11733199469745159, | |
| "reward_std": 0.09876838512718678, | |
| "rewards/cosine_scaled_reward": -0.058665999211370945, | |
| "rewards/format_reward": 0.0, | |
| "step": 233 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.2500305175781, | |
| "epoch": 0.936, | |
| "grad_norm": 0.09378773719072342, | |
| "kl": 0.154541015625, | |
| "learning_rate": 6.770536555792944e-07, | |
| "loss": -0.005, | |
| "reward": -0.12837751768529415, | |
| "reward_std": 0.12461850047111511, | |
| "rewards/cosine_scaled_reward": -0.06418875977396965, | |
| "rewards/format_reward": 0.0, | |
| "step": 234 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.8750305175781, | |
| "epoch": 0.94, | |
| "grad_norm": 0.1036522388458252, | |
| "kl": 0.1611328125, | |
| "learning_rate": 6.740368101176495e-07, | |
| "loss": -0.0073, | |
| "reward": -0.12727641500532627, | |
| "reward_std": 0.11483397521078587, | |
| "rewards/cosine_scaled_reward": -0.06363820657134056, | |
| "rewards/format_reward": 0.0, | |
| "step": 235 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1522.732177734375, | |
| "epoch": 0.944, | |
| "grad_norm": 0.12251052260398865, | |
| "kl": 0.1640625, | |
| "learning_rate": 6.710139192768694e-07, | |
| "loss": -0.0118, | |
| "reward": -0.11951213330030441, | |
| "reward_std": 0.1262509860098362, | |
| "rewards/cosine_scaled_reward": -0.059756068512797356, | |
| "rewards/format_reward": 0.0, | |
| "step": 236 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1509.75, | |
| "epoch": 0.948, | |
| "grad_norm": 0.17989245057106018, | |
| "kl": 0.156982421875, | |
| "learning_rate": 6.679851303883891e-07, | |
| "loss": -0.0289, | |
| "reward": -0.13234319165349007, | |
| "reward_std": 0.13814006373286247, | |
| "rewards/cosine_scaled_reward": -0.06617159210145473, | |
| "rewards/format_reward": 0.0, | |
| "step": 237 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1512.8333435058594, | |
| "epoch": 0.952, | |
| "grad_norm": 0.12887263298034668, | |
| "kl": 0.1552734375, | |
| "learning_rate": 6.649505910711058e-07, | |
| "loss": -0.0163, | |
| "reward": -0.13513639941811562, | |
| "reward_std": 0.1341167613863945, | |
| "rewards/cosine_scaled_reward": -0.06756819784641266, | |
| "rewards/format_reward": 0.0, | |
| "step": 238 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.7202453613281, | |
| "epoch": 0.956, | |
| "grad_norm": 0.10920631885528564, | |
| "kl": 0.163330078125, | |
| "learning_rate": 6.619104492241847e-07, | |
| "loss": -0.0208, | |
| "reward": -0.11296023428440094, | |
| "reward_std": 0.10206396505236626, | |
| "rewards/cosine_scaled_reward": -0.056480118073523045, | |
| "rewards/format_reward": 0.0, | |
| "step": 239 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1501.2917175292969, | |
| "epoch": 0.96, | |
| "grad_norm": 0.10691919177770615, | |
| "kl": 0.173828125, | |
| "learning_rate": 6.588648530198504e-07, | |
| "loss": -0.0251, | |
| "reward": -0.12773141264915466, | |
| "reward_std": 0.12681871838867664, | |
| "rewards/cosine_scaled_reward": -0.06386570446193218, | |
| "rewards/format_reward": 0.0, | |
| "step": 240 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.9940795898438, | |
| "epoch": 0.964, | |
| "grad_norm": 0.12696890532970428, | |
| "kl": 0.1640625, | |
| "learning_rate": 6.558139508961654e-07, | |
| "loss": -0.0235, | |
| "reward": -0.12828433699905872, | |
| "reward_std": 0.12480376102030277, | |
| "rewards/cosine_scaled_reward": -0.06414216570556164, | |
| "rewards/format_reward": 0.0, | |
| "step": 241 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1528.40478515625, | |
| "epoch": 0.968, | |
| "grad_norm": 0.19038116931915283, | |
| "kl": 0.166015625, | |
| "learning_rate": 6.527578915497951e-07, | |
| "loss": -0.0025, | |
| "reward": -0.12644540891051292, | |
| "reward_std": 0.11273947730660439, | |
| "rewards/cosine_scaled_reward": -0.06322270352393389, | |
| "rewards/format_reward": 0.0, | |
| "step": 242 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1502.4404907226562, | |
| "epoch": 0.972, | |
| "grad_norm": 0.1264588087797165, | |
| "kl": 0.1728515625, | |
| "learning_rate": 6.496968239287603e-07, | |
| "loss": -0.0168, | |
| "reward": -0.13079960085451603, | |
| "reward_std": 0.10997281037271023, | |
| "rewards/cosine_scaled_reward": -0.06539979483932257, | |
| "rewards/format_reward": 0.0, | |
| "step": 243 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.0119323730469, | |
| "epoch": 0.976, | |
| "grad_norm": 0.08865034580230713, | |
| "kl": 0.177734375, | |
| "learning_rate": 6.466308972251785e-07, | |
| "loss": -0.0154, | |
| "reward": -0.13137424178421497, | |
| "reward_std": 0.1251276545226574, | |
| "rewards/cosine_scaled_reward": -0.06568712089210749, | |
| "rewards/format_reward": 0.0, | |
| "step": 244 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1482.8036193847656, | |
| "epoch": 0.98, | |
| "grad_norm": 0.11429865658283234, | |
| "kl": 0.1796875, | |
| "learning_rate": 6.435602608679916e-07, | |
| "loss": -0.0297, | |
| "reward": -0.13589460216462612, | |
| "reward_std": 0.1405064184218645, | |
| "rewards/cosine_scaled_reward": -0.06794730294495821, | |
| "rewards/format_reward": 0.0, | |
| "step": 245 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1502.5178833007812, | |
| "epoch": 0.984, | |
| "grad_norm": 0.06874290853738785, | |
| "kl": 0.1602783203125, | |
| "learning_rate": 6.404850645156841e-07, | |
| "loss": -0.0348, | |
| "reward": -0.0405126977711916, | |
| "reward_std": 0.10752722714096308, | |
| "rewards/cosine_scaled_reward": -0.020256347954273224, | |
| "rewards/format_reward": 0.0, | |
| "step": 246 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1509.2678833007812, | |
| "epoch": 0.988, | |
| "grad_norm": 0.1023624986410141, | |
| "kl": 0.16748046875, | |
| "learning_rate": 6.374054580489873e-07, | |
| "loss": -0.0321, | |
| "reward": -0.13477133214473724, | |
| "reward_std": 0.13717731088399887, | |
| "rewards/cosine_scaled_reward": -0.06738566607236862, | |
| "rewards/format_reward": 0.0, | |
| "step": 247 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1487.4464416503906, | |
| "epoch": 0.992, | |
| "grad_norm": 0.09231995791196823, | |
| "kl": 0.17333984375, | |
| "learning_rate": 6.343215915635761e-07, | |
| "loss": -0.0601, | |
| "reward": -0.12894214503467083, | |
| "reward_std": 0.1458846628665924, | |
| "rewards/cosine_scaled_reward": -0.06447107251733541, | |
| "rewards/format_reward": 0.0, | |
| "step": 248 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.107177734375, | |
| "epoch": 0.996, | |
| "grad_norm": 0.08529460430145264, | |
| "kl": 0.171630859375, | |
| "learning_rate": 6.31233615362752e-07, | |
| "loss": -0.0203, | |
| "reward": -0.11800242215394974, | |
| "reward_std": 0.12636969611048698, | |
| "rewards/cosine_scaled_reward": -0.059001206420361996, | |
| "rewards/format_reward": 0.0, | |
| "step": 249 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1524.5001220703125, | |
| "epoch": 1.0, | |
| "grad_norm": 0.08003545552492142, | |
| "kl": 0.176513671875, | |
| "learning_rate": 6.281416799501187e-07, | |
| "loss": -0.0263, | |
| "reward": -0.11569427512586117, | |
| "reward_std": 0.10093419812619686, | |
| "rewards/cosine_scaled_reward": -0.05784713663160801, | |
| "rewards/format_reward": 0.0, | |
| "step": 250 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.452392578125, | |
| "epoch": 1.004, | |
| "grad_norm": 0.0908510684967041, | |
| "kl": 0.1728515625, | |
| "learning_rate": 6.25045936022246e-07, | |
| "loss": -0.0223, | |
| "reward": -0.10760886035859585, | |
| "reward_std": 0.10886831395328045, | |
| "rewards/cosine_scaled_reward": -0.053804428316652775, | |
| "rewards/format_reward": 0.0, | |
| "step": 251 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.6250305175781, | |
| "epoch": 1.008, | |
| "grad_norm": 0.12224633246660233, | |
| "kl": 0.18212890625, | |
| "learning_rate": 6.219465344613258e-07, | |
| "loss": -0.0196, | |
| "reward": -0.12489514984190464, | |
| "reward_std": 0.12947135604918003, | |
| "rewards/cosine_scaled_reward": -0.062447573989629745, | |
| "rewards/format_reward": 0.0, | |
| "step": 252 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.5654907226562, | |
| "epoch": 1.012, | |
| "grad_norm": 0.13140302896499634, | |
| "kl": 0.1845703125, | |
| "learning_rate": 6.188436263278172e-07, | |
| "loss": -0.0149, | |
| "reward": -0.12439864501357079, | |
| "reward_std": 0.12002300284802914, | |
| "rewards/cosine_scaled_reward": -0.06219932623207569, | |
| "rewards/format_reward": 0.0, | |
| "step": 253 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.0357360839844, | |
| "epoch": 1.016, | |
| "grad_norm": 0.11975951492786407, | |
| "kl": 0.17724609375, | |
| "learning_rate": 6.157373628530852e-07, | |
| "loss": -0.0206, | |
| "reward": -0.1344124898314476, | |
| "reward_std": 0.12755578383803368, | |
| "rewards/cosine_scaled_reward": -0.0672062449157238, | |
| "rewards/format_reward": 0.0, | |
| "step": 254 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.5, | |
| "epoch": 1.02, | |
| "grad_norm": 0.12696978449821472, | |
| "kl": 0.18701171875, | |
| "learning_rate": 6.126278954320294e-07, | |
| "loss": -0.0138, | |
| "reward": -0.13142068311572075, | |
| "reward_std": 0.13189822621643543, | |
| "rewards/cosine_scaled_reward": -0.0657103369012475, | |
| "rewards/format_reward": 0.0, | |
| "step": 255 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1505.4642944335938, | |
| "epoch": 1.024, | |
| "grad_norm": 0.11200578510761261, | |
| "kl": 0.177001953125, | |
| "learning_rate": 6.095153756157051e-07, | |
| "loss": -0.036, | |
| "reward": -0.12037310004234314, | |
| "reward_std": 0.11811047606170177, | |
| "rewards/cosine_scaled_reward": -0.06018654815852642, | |
| "rewards/format_reward": 0.0, | |
| "step": 256 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1499.672607421875, | |
| "epoch": 1.028, | |
| "grad_norm": 0.1471128910779953, | |
| "kl": 0.172607421875, | |
| "learning_rate": 6.06399955103937e-07, | |
| "loss": -0.0249, | |
| "reward": -0.12174288742244244, | |
| "reward_std": 0.1309817060828209, | |
| "rewards/cosine_scaled_reward": -0.06087144184857607, | |
| "rewards/format_reward": 0.0, | |
| "step": 257 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1520.2559814453125, | |
| "epoch": 1.032, | |
| "grad_norm": 0.10926475375890732, | |
| "kl": 0.170166015625, | |
| "learning_rate": 6.032817857379256e-07, | |
| "loss": -0.0154, | |
| "reward": -0.13306790590286255, | |
| "reward_std": 0.1421681884676218, | |
| "rewards/cosine_scaled_reward": -0.06653395667672157, | |
| "rewards/format_reward": 0.0, | |
| "step": 258 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1523.1666870117188, | |
| "epoch": 1.036, | |
| "grad_norm": 0.15364930033683777, | |
| "kl": 0.1787109375, | |
| "learning_rate": 6.001610194928464e-07, | |
| "loss": -0.0083, | |
| "reward": -0.11868332512676716, | |
| "reward_std": 0.11832969635725021, | |
| "rewards/cosine_scaled_reward": -0.059341663494706154, | |
| "rewards/format_reward": 0.0, | |
| "step": 259 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.9583435058594, | |
| "epoch": 1.04, | |
| "grad_norm": 0.10218587517738342, | |
| "kl": 0.163330078125, | |
| "learning_rate": 5.97037808470444e-07, | |
| "loss": -0.0278, | |
| "reward": -0.12172269076108932, | |
| "reward_std": 0.120206318795681, | |
| "rewards/cosine_scaled_reward": -0.06086134351789951, | |
| "rewards/format_reward": 0.0, | |
| "step": 260 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1503.9880981445312, | |
| "epoch": 1.044, | |
| "grad_norm": 0.11966069042682648, | |
| "kl": 0.16552734375, | |
| "learning_rate": 5.939123048916173e-07, | |
| "loss": -0.0226, | |
| "reward": -0.11986922658979893, | |
| "reward_std": 0.1164070088416338, | |
| "rewards/cosine_scaled_reward": -0.05993461608886719, | |
| "rewards/format_reward": 0.0, | |
| "step": 261 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.857177734375, | |
| "epoch": 1.048, | |
| "grad_norm": 0.11268185824155807, | |
| "kl": 0.161865234375, | |
| "learning_rate": 5.907846610890011e-07, | |
| "loss": -0.037, | |
| "reward": -0.11277966573834419, | |
| "reward_std": 0.11542832851409912, | |
| "rewards/cosine_scaled_reward": -0.05638983380049467, | |
| "rewards/format_reward": 0.0, | |
| "step": 262 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1522.702392578125, | |
| "epoch": 1.052, | |
| "grad_norm": 0.1593422144651413, | |
| "kl": 0.1494140625, | |
| "learning_rate": 5.87655029499542e-07, | |
| "loss": -0.0086, | |
| "reward": -0.11897107027471066, | |
| "reward_std": 0.12276079133152962, | |
| "rewards/cosine_scaled_reward": -0.0594855360686779, | |
| "rewards/format_reward": 0.0, | |
| "step": 263 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1520.1904907226562, | |
| "epoch": 1.056, | |
| "grad_norm": 0.09266054630279541, | |
| "kl": 0.1494140625, | |
| "learning_rate": 5.845235626570683e-07, | |
| "loss": -0.0117, | |
| "reward": -0.10893694311380386, | |
| "reward_std": 0.11326288990676403, | |
| "rewards/cosine_scaled_reward": -0.05446847062557936, | |
| "rewards/format_reward": 0.0, | |
| "step": 264 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1499.922607421875, | |
| "epoch": 1.06, | |
| "grad_norm": 0.10393787920475006, | |
| "kl": 0.157470703125, | |
| "learning_rate": 5.813904131848564e-07, | |
| "loss": -0.0333, | |
| "reward": -0.12131388112902641, | |
| "reward_std": 0.1239698026329279, | |
| "rewards/cosine_scaled_reward": -0.06065694149583578, | |
| "rewards/format_reward": 0.0, | |
| "step": 265 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.3511962890625, | |
| "epoch": 1.064, | |
| "grad_norm": 0.0964948907494545, | |
| "kl": 0.164794921875, | |
| "learning_rate": 5.78255733788191e-07, | |
| "loss": -0.0206, | |
| "reward": -0.10519931092858315, | |
| "reward_std": 0.11078912951052189, | |
| "rewards/cosine_scaled_reward": -0.05259965639561415, | |
| "rewards/format_reward": 0.0, | |
| "step": 266 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.2798156738281, | |
| "epoch": 1.068, | |
| "grad_norm": 0.1041577160358429, | |
| "kl": 0.167724609375, | |
| "learning_rate": 5.751196772469237e-07, | |
| "loss": -0.028, | |
| "reward": -0.11491566710174084, | |
| "reward_std": 0.13411646336317062, | |
| "rewards/cosine_scaled_reward": -0.05745783355087042, | |
| "rewards/format_reward": 0.0, | |
| "step": 267 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1527.4285888671875, | |
| "epoch": 1.072, | |
| "grad_norm": 0.12076977640390396, | |
| "kl": 0.172119140625, | |
| "learning_rate": 5.71982396408026e-07, | |
| "loss": -0.0013, | |
| "reward": -0.09519998729228973, | |
| "reward_std": 0.09795477986335754, | |
| "rewards/cosine_scaled_reward": -0.04759999457746744, | |
| "rewards/format_reward": 0.0, | |
| "step": 268 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1502.1964721679688, | |
| "epoch": 1.076, | |
| "grad_norm": 0.15113261342048645, | |
| "kl": 0.16552734375, | |
| "learning_rate": 5.688440441781398e-07, | |
| "loss": -0.0375, | |
| "reward": -0.11081114411354065, | |
| "reward_std": 0.12273510918021202, | |
| "rewards/cosine_scaled_reward": -0.05540557578206062, | |
| "rewards/format_reward": 0.0, | |
| "step": 269 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1536.0, | |
| "epoch": 1.08, | |
| "grad_norm": 0.14382889866828918, | |
| "kl": 0.154296875, | |
| "learning_rate": 5.657047735161255e-07, | |
| "loss": 0.0062, | |
| "reward": -0.09718998149037361, | |
| "reward_std": 0.09087535366415977, | |
| "rewards/cosine_scaled_reward": -0.04859498701989651, | |
| "rewards/format_reward": 0.0, | |
| "step": 270 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1500.0, | |
| "epoch": 1.084, | |
| "grad_norm": 0.16920308768749237, | |
| "kl": 0.1630859375, | |
| "learning_rate": 5.625647374256061e-07, | |
| "loss": -0.021, | |
| "reward": -0.1035971287637949, | |
| "reward_std": 0.11180637776851654, | |
| "rewards/cosine_scaled_reward": -0.051798563450574875, | |
| "rewards/format_reward": 0.0, | |
| "step": 271 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1490.7143249511719, | |
| "epoch": 1.088, | |
| "grad_norm": 0.13314908742904663, | |
| "kl": 0.157470703125, | |
| "learning_rate": 5.594240889475106e-07, | |
| "loss": -0.037, | |
| "reward": -0.11400851234793663, | |
| "reward_std": 0.12938306666910648, | |
| "rewards/cosine_scaled_reward": -0.05700425896793604, | |
| "rewards/format_reward": 0.0, | |
| "step": 272 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.9464721679688, | |
| "epoch": 1.092, | |
| "grad_norm": 0.07967788726091385, | |
| "kl": 0.166748046875, | |
| "learning_rate": 5.562829811526154e-07, | |
| "loss": -0.0369, | |
| "reward": -0.10690408386290073, | |
| "reward_std": 0.12804840318858624, | |
| "rewards/cosine_scaled_reward": -0.053452043794095516, | |
| "rewards/format_reward": 0.0, | |
| "step": 273 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1529.7440795898438, | |
| "epoch": 1.096, | |
| "grad_norm": 0.09101837128400803, | |
| "kl": 0.1650390625, | |
| "learning_rate": 5.531415671340826e-07, | |
| "loss": -0.0001, | |
| "reward": -0.10337771661579609, | |
| "reward_std": 0.10489914752542973, | |
| "rewards/cosine_scaled_reward": -0.051688858307898045, | |
| "rewards/format_reward": 0.0, | |
| "step": 274 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1527.9404907226562, | |
| "epoch": 1.1, | |
| "grad_norm": 0.16794751584529877, | |
| "kl": 0.168701171875, | |
| "learning_rate": 5.5e-07, | |
| "loss": -0.0041, | |
| "reward": -0.08096367586404085, | |
| "reward_std": 0.08801300823688507, | |
| "rewards/cosine_scaled_reward": -0.040481837932020426, | |
| "rewards/format_reward": 0.0, | |
| "step": 275 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1512.3274230957031, | |
| "epoch": 1.104, | |
| "grad_norm": 0.0922553613781929, | |
| "kl": 0.161376953125, | |
| "learning_rate": 5.468584328659172e-07, | |
| "loss": -0.0232, | |
| "reward": -0.1129224356263876, | |
| "reward_std": 0.12950855493545532, | |
| "rewards/cosine_scaled_reward": -0.056461221538484097, | |
| "rewards/format_reward": 0.0, | |
| "step": 276 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.65478515625, | |
| "epoch": 1.108, | |
| "grad_norm": 0.1329527050256729, | |
| "kl": 0.167236328125, | |
| "learning_rate": 5.437170188473847e-07, | |
| "loss": -0.0302, | |
| "reward": -0.11213578283786774, | |
| "reward_std": 0.11713657341897488, | |
| "rewards/cosine_scaled_reward": -0.05606789421290159, | |
| "rewards/format_reward": 0.0, | |
| "step": 277 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.0, | |
| "epoch": 1.112, | |
| "grad_norm": 0.10364029556512833, | |
| "kl": 0.163818359375, | |
| "learning_rate": 5.405759110524894e-07, | |
| "loss": -0.0298, | |
| "reward": -0.10770672746002674, | |
| "reward_std": 0.11716131307184696, | |
| "rewards/cosine_scaled_reward": -0.053853364661335945, | |
| "rewards/format_reward": 0.0, | |
| "step": 278 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1474.4702453613281, | |
| "epoch": 1.116, | |
| "grad_norm": 0.1940528154373169, | |
| "kl": 0.17138671875, | |
| "learning_rate": 5.37435262574394e-07, | |
| "loss": -0.0549, | |
| "reward": -0.06423771567642689, | |
| "reward_std": 0.14231404848396778, | |
| "rewards/cosine_scaled_reward": -0.03211886156350374, | |
| "rewards/format_reward": 0.0, | |
| "step": 279 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1515.9821472167969, | |
| "epoch": 1.12, | |
| "grad_norm": 0.09174785017967224, | |
| "kl": 0.165283203125, | |
| "learning_rate": 5.342952264838747e-07, | |
| "loss": -0.0167, | |
| "reward": -0.12564732693135738, | |
| "reward_std": 0.12320295721292496, | |
| "rewards/cosine_scaled_reward": -0.06282366160303354, | |
| "rewards/format_reward": 0.0, | |
| "step": 280 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1525.8214416503906, | |
| "epoch": 1.124, | |
| "grad_norm": 0.11484113335609436, | |
| "kl": 0.16650390625, | |
| "learning_rate": 5.311559558218603e-07, | |
| "loss": -0.0015, | |
| "reward": -0.09635571762919426, | |
| "reward_std": 0.09338909015059471, | |
| "rewards/cosine_scaled_reward": -0.048177859745919704, | |
| "rewards/format_reward": 0.0, | |
| "step": 281 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1502.0714721679688, | |
| "epoch": 1.1280000000000001, | |
| "grad_norm": 0.1825956404209137, | |
| "kl": 0.158203125, | |
| "learning_rate": 5.28017603591974e-07, | |
| "loss": -0.0234, | |
| "reward": -0.11337108165025711, | |
| "reward_std": 0.1339975707232952, | |
| "rewards/cosine_scaled_reward": -0.05668553803116083, | |
| "rewards/format_reward": 0.0, | |
| "step": 282 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.6964416503906, | |
| "epoch": 1.1320000000000001, | |
| "grad_norm": 0.1683872640132904, | |
| "kl": 0.185546875, | |
| "learning_rate": 5.248803227530763e-07, | |
| "loss": -0.0202, | |
| "reward": -0.1038710568100214, | |
| "reward_std": 0.1097688339650631, | |
| "rewards/cosine_scaled_reward": -0.051935529336333275, | |
| "rewards/format_reward": 0.0, | |
| "step": 283 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1528.0595397949219, | |
| "epoch": 1.1360000000000001, | |
| "grad_norm": 0.10006389766931534, | |
| "kl": 0.1591796875, | |
| "learning_rate": 5.21744266211809e-07, | |
| "loss": 0.0012, | |
| "reward": -0.12082774937152863, | |
| "reward_std": 0.11856056936085224, | |
| "rewards/cosine_scaled_reward": -0.06041387468576431, | |
| "rewards/format_reward": 0.0, | |
| "step": 284 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1505.7916870117188, | |
| "epoch": 1.1400000000000001, | |
| "grad_norm": 0.1140761449933052, | |
| "kl": 0.156982421875, | |
| "learning_rate": 5.186095868151436e-07, | |
| "loss": -0.035, | |
| "reward": -0.11102944053709507, | |
| "reward_std": 0.143316388130188, | |
| "rewards/cosine_scaled_reward": -0.05551472119987011, | |
| "rewards/format_reward": 0.0, | |
| "step": 285 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1529.6309509277344, | |
| "epoch": 1.144, | |
| "grad_norm": 0.09991593658924103, | |
| "kl": 0.158447265625, | |
| "learning_rate": 5.154764373429315e-07, | |
| "loss": -0.0016, | |
| "reward": -0.11252638325095177, | |
| "reward_std": 0.11945481784641743, | |
| "rewards/cosine_scaled_reward": -0.05626319348812103, | |
| "rewards/format_reward": 0.0, | |
| "step": 286 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1520.4821472167969, | |
| "epoch": 1.148, | |
| "grad_norm": 0.12035401910543442, | |
| "kl": 0.17529296875, | |
| "learning_rate": 5.123449705004581e-07, | |
| "loss": -0.0112, | |
| "reward": -0.1297306139022112, | |
| "reward_std": 0.13188758678734303, | |
| "rewards/cosine_scaled_reward": -0.0648653069511056, | |
| "rewards/format_reward": 0.0, | |
| "step": 287 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.7083740234375, | |
| "epoch": 1.152, | |
| "grad_norm": 0.11799659579992294, | |
| "kl": 0.169189453125, | |
| "learning_rate": 5.09215338910999e-07, | |
| "loss": -0.0217, | |
| "reward": -0.10410146042704582, | |
| "reward_std": 0.10413151048123837, | |
| "rewards/cosine_scaled_reward": -0.05205072835087776, | |
| "rewards/format_reward": 0.0, | |
| "step": 288 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1496.8512268066406, | |
| "epoch": 1.156, | |
| "grad_norm": 0.1316956877708435, | |
| "kl": 0.15869140625, | |
| "learning_rate": 5.060876951083828e-07, | |
| "loss": -0.0207, | |
| "reward": -0.11279321648180485, | |
| "reward_std": 0.11689519137144089, | |
| "rewards/cosine_scaled_reward": -0.05639660730957985, | |
| "rewards/format_reward": 0.0, | |
| "step": 289 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.6369323730469, | |
| "epoch": 1.16, | |
| "grad_norm": 0.07206040620803833, | |
| "kl": 0.165283203125, | |
| "learning_rate": 5.02962191529556e-07, | |
| "loss": -0.0101, | |
| "reward": -0.11338524892926216, | |
| "reward_std": 0.10832424648106098, | |
| "rewards/cosine_scaled_reward": -0.05669262260198593, | |
| "rewards/format_reward": 0.0, | |
| "step": 290 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.3511962890625, | |
| "epoch": 1.164, | |
| "grad_norm": 0.20136146247386932, | |
| "kl": 0.164794921875, | |
| "learning_rate": 4.998389805071536e-07, | |
| "loss": -0.0202, | |
| "reward": -0.12464358657598495, | |
| "reward_std": 0.10932311788201332, | |
| "rewards/cosine_scaled_reward": -0.06232179142534733, | |
| "rewards/format_reward": 0.0, | |
| "step": 291 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.1547546386719, | |
| "epoch": 1.168, | |
| "grad_norm": 0.10925502330064774, | |
| "kl": 0.185791015625, | |
| "learning_rate": 4.967182142620745e-07, | |
| "loss": 0.003, | |
| "reward": -0.09491665475070477, | |
| "reward_std": 0.08971338160336018, | |
| "rewards/cosine_scaled_reward": -0.04745833110064268, | |
| "rewards/format_reward": 0.0, | |
| "step": 292 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1501.0535888671875, | |
| "epoch": 1.172, | |
| "grad_norm": 0.09527327865362167, | |
| "kl": 0.173828125, | |
| "learning_rate": 4.93600044896063e-07, | |
| "loss": -0.0318, | |
| "reward": -0.13249907828867435, | |
| "reward_std": 0.1209456454962492, | |
| "rewards/cosine_scaled_reward": -0.06624954286962748, | |
| "rewards/format_reward": 0.0, | |
| "step": 293 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.3869323730469, | |
| "epoch": 1.176, | |
| "grad_norm": 0.13041535019874573, | |
| "kl": 0.188720703125, | |
| "learning_rate": 4.904846243842949e-07, | |
| "loss": -0.015, | |
| "reward": -0.11728105135262012, | |
| "reward_std": 0.11388706415891647, | |
| "rewards/cosine_scaled_reward": -0.058640528470277786, | |
| "rewards/format_reward": 0.0, | |
| "step": 294 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.1488342285156, | |
| "epoch": 1.18, | |
| "grad_norm": 0.13715095818042755, | |
| "kl": 0.184814453125, | |
| "learning_rate": 4.873721045679706e-07, | |
| "loss": -0.0191, | |
| "reward": -0.10163358226418495, | |
| "reward_std": 0.10592619515955448, | |
| "rewards/cosine_scaled_reward": -0.05081678926944733, | |
| "rewards/format_reward": 0.0, | |
| "step": 295 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1517.1548156738281, | |
| "epoch": 1.184, | |
| "grad_norm": 0.10485463589429855, | |
| "kl": 0.172119140625, | |
| "learning_rate": 4.842626371469149e-07, | |
| "loss": -0.0183, | |
| "reward": -0.11159243248403072, | |
| "reward_std": 0.11549564823508263, | |
| "rewards/cosine_scaled_reward": -0.05579621531069279, | |
| "rewards/format_reward": 0.0, | |
| "step": 296 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1505.4702453613281, | |
| "epoch": 1.188, | |
| "grad_norm": 0.16054122149944305, | |
| "kl": 0.19091796875, | |
| "learning_rate": 4.811563736721829e-07, | |
| "loss": -0.0199, | |
| "reward": -0.10421715676784515, | |
| "reward_std": 0.10152514837682247, | |
| "rewards/cosine_scaled_reward": -0.05210857838392258, | |
| "rewards/format_reward": 0.0, | |
| "step": 297 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.8630981445312, | |
| "epoch": 1.192, | |
| "grad_norm": 0.1103135496377945, | |
| "kl": 0.167236328125, | |
| "learning_rate": 4.780534655386743e-07, | |
| "loss": -0.0185, | |
| "reward": -0.11138204857707024, | |
| "reward_std": 0.12202793546020985, | |
| "rewards/cosine_scaled_reward": -0.05569102708250284, | |
| "rewards/format_reward": 0.0, | |
| "step": 298 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.1905212402344, | |
| "epoch": 1.196, | |
| "grad_norm": 0.13167892396450043, | |
| "kl": 0.165283203125, | |
| "learning_rate": 4.749540639777539e-07, | |
| "loss": -0.0422, | |
| "reward": -0.12753658182919025, | |
| "reward_std": 0.15048057585954666, | |
| "rewards/cosine_scaled_reward": -0.06376829091459513, | |
| "rewards/format_reward": 0.0, | |
| "step": 299 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.3333740234375, | |
| "epoch": 1.2, | |
| "grad_norm": 0.11807087808847427, | |
| "kl": 0.1787109375, | |
| "learning_rate": 4.7185832004988133e-07, | |
| "loss": -0.0233, | |
| "reward": -0.10466483794152737, | |
| "reward_std": 0.09747852385044098, | |
| "rewards/cosine_scaled_reward": -0.05233241897076368, | |
| "rewards/format_reward": 0.0, | |
| "step": 300 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.6369018554688, | |
| "epoch": 1.204, | |
| "grad_norm": 0.10298826545476913, | |
| "kl": 0.17041015625, | |
| "learning_rate": 4.68766384637248e-07, | |
| "loss": -0.0369, | |
| "reward": -0.030609130859375, | |
| "reward_std": 0.1283973567187786, | |
| "rewards/cosine_scaled_reward": -0.0153045654296875, | |
| "rewards/format_reward": 0.0, | |
| "step": 301 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1512.0119323730469, | |
| "epoch": 1.208, | |
| "grad_norm": 0.11705794930458069, | |
| "kl": 0.16259765625, | |
| "learning_rate": 4.656784084364238e-07, | |
| "loss": -0.0288, | |
| "reward": -0.1029995009303093, | |
| "reward_std": 0.1215117834508419, | |
| "rewards/cosine_scaled_reward": -0.0514997486025095, | |
| "rewards/format_reward": 0.0, | |
| "step": 302 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1512.077392578125, | |
| "epoch": 1.212, | |
| "grad_norm": 0.13367965817451477, | |
| "kl": 0.186279296875, | |
| "learning_rate": 4.6259454195101267e-07, | |
| "loss": -0.024, | |
| "reward": -0.1126109603792429, | |
| "reward_std": 0.11090057343244553, | |
| "rewards/cosine_scaled_reward": -0.05630548112094402, | |
| "rewards/format_reward": 0.0, | |
| "step": 303 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.8511962890625, | |
| "epoch": 1.216, | |
| "grad_norm": 0.1228313073515892, | |
| "kl": 0.17578125, | |
| "learning_rate": 4.59514935484316e-07, | |
| "loss": -0.0105, | |
| "reward": -0.08529405388981104, | |
| "reward_std": 0.08704109024256468, | |
| "rewards/cosine_scaled_reward": -0.042647027876228094, | |
| "rewards/format_reward": 0.0, | |
| "step": 304 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1480.511962890625, | |
| "epoch": 1.22, | |
| "grad_norm": 0.12493617087602615, | |
| "kl": 0.1513671875, | |
| "learning_rate": 4.5643973913200837e-07, | |
| "loss": -0.0485, | |
| "reward": -0.12973922118544579, | |
| "reward_std": 0.15159142762422562, | |
| "rewards/cosine_scaled_reward": -0.06486961059272289, | |
| "rewards/format_reward": 0.0, | |
| "step": 305 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1525.3988342285156, | |
| "epoch": 1.224, | |
| "grad_norm": 0.08764777332544327, | |
| "kl": 0.165283203125, | |
| "learning_rate": 4.5336910277482155e-07, | |
| "loss": -0.006, | |
| "reward": -0.08825473487377167, | |
| "reward_std": 0.10441902838647366, | |
| "rewards/cosine_scaled_reward": -0.04412736464291811, | |
| "rewards/format_reward": 0.0, | |
| "step": 306 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1529.3869018554688, | |
| "epoch": 1.228, | |
| "grad_norm": 0.08289927244186401, | |
| "kl": 0.169677734375, | |
| "learning_rate": 4.503031760712397e-07, | |
| "loss": -0.003, | |
| "reward": -0.07872879132628441, | |
| "reward_std": 0.07979759760200977, | |
| "rewards/cosine_scaled_reward": -0.03936439473181963, | |
| "rewards/format_reward": 0.0, | |
| "step": 307 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.3630981445312, | |
| "epoch": 1.232, | |
| "grad_norm": 0.14125800132751465, | |
| "kl": 0.17578125, | |
| "learning_rate": 4.4724210845020494e-07, | |
| "loss": -0.0177, | |
| "reward": -0.08932580798864365, | |
| "reward_std": 0.08838274143636227, | |
| "rewards/cosine_scaled_reward": -0.04466290678828955, | |
| "rewards/format_reward": 0.0, | |
| "step": 308 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1499.9642944335938, | |
| "epoch": 1.236, | |
| "grad_norm": 0.1731095314025879, | |
| "kl": 0.146484375, | |
| "learning_rate": 4.441860491038345e-07, | |
| "loss": -0.0359, | |
| "reward": -0.10843771509826183, | |
| "reward_std": 0.11413087695837021, | |
| "rewards/cosine_scaled_reward": -0.05421885754913092, | |
| "rewards/format_reward": 0.0, | |
| "step": 309 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1494.4702758789062, | |
| "epoch": 1.24, | |
| "grad_norm": 0.2260572761297226, | |
| "kl": 0.164306640625, | |
| "learning_rate": 4.4113514698014953e-07, | |
| "loss": -0.0351, | |
| "reward": -0.11317398212850094, | |
| "reward_std": 0.124705346301198, | |
| "rewards/cosine_scaled_reward": -0.05658698920160532, | |
| "rewards/format_reward": 0.0, | |
| "step": 310 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.1428833007812, | |
| "epoch": 1.244, | |
| "grad_norm": 0.10579942166805267, | |
| "kl": 0.17333984375, | |
| "learning_rate": 4.3808955077581546e-07, | |
| "loss": -0.0133, | |
| "reward": -0.10701234266161919, | |
| "reward_std": 0.11289746686816216, | |
| "rewards/cosine_scaled_reward": -0.05350616853684187, | |
| "rewards/format_reward": 0.0, | |
| "step": 311 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1499.4940795898438, | |
| "epoch": 1.248, | |
| "grad_norm": 0.12452121824026108, | |
| "kl": 0.151611328125, | |
| "learning_rate": 4.350494089288943e-07, | |
| "loss": -0.0416, | |
| "reward": -0.12627964280545712, | |
| "reward_std": 0.1439381241798401, | |
| "rewards/cosine_scaled_reward": -0.06313982233405113, | |
| "rewards/format_reward": 0.0, | |
| "step": 312 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.7559814453125, | |
| "epoch": 1.252, | |
| "grad_norm": 0.14036187529563904, | |
| "kl": 0.164306640625, | |
| "learning_rate": 4.3201486961161093e-07, | |
| "loss": -0.0152, | |
| "reward": -0.08883011713624, | |
| "reward_std": 0.09118240885436535, | |
| "rewards/cosine_scaled_reward": -0.0444150622934103, | |
| "rewards/format_reward": 0.0, | |
| "step": 313 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1525.2440795898438, | |
| "epoch": 1.256, | |
| "grad_norm": 0.1239011213183403, | |
| "kl": 0.165283203125, | |
| "learning_rate": 4.2898608072313045e-07, | |
| "loss": -0.0058, | |
| "reward": -0.10498439148068428, | |
| "reward_std": 0.11687885224819183, | |
| "rewards/cosine_scaled_reward": -0.05249219387769699, | |
| "rewards/format_reward": 0.0, | |
| "step": 314 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1490.8631286621094, | |
| "epoch": 1.26, | |
| "grad_norm": 0.09535997360944748, | |
| "kl": 0.159423828125, | |
| "learning_rate": 4.2596318988235037e-07, | |
| "loss": -0.0165, | |
| "reward": -0.02371996082365513, | |
| "reward_std": 0.09539724607020617, | |
| "rewards/cosine_scaled_reward": -0.011859980411827564, | |
| "rewards/format_reward": 0.0, | |
| "step": 315 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1494.3988342285156, | |
| "epoch": 1.264, | |
| "grad_norm": 0.11306377500295639, | |
| "kl": 0.161865234375, | |
| "learning_rate": 4.2294634442070553e-07, | |
| "loss": -0.0466, | |
| "reward": -0.03322407230734825, | |
| "reward_std": 0.12497628107666969, | |
| "rewards/cosine_scaled_reward": -0.0166120370849967, | |
| "rewards/format_reward": 0.0, | |
| "step": 316 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1530.4166870117188, | |
| "epoch": 1.268, | |
| "grad_norm": 0.11471430212259293, | |
| "kl": 0.160888671875, | |
| "learning_rate": 4.1993569137498776e-07, | |
| "loss": 0.0016, | |
| "reward": -0.08301959745585918, | |
| "reward_std": 0.08951563201844692, | |
| "rewards/cosine_scaled_reward": -0.04150979872792959, | |
| "rewards/format_reward": 0.0, | |
| "step": 317 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1523.1607055664062, | |
| "epoch": 1.272, | |
| "grad_norm": 0.18867182731628418, | |
| "kl": 0.155029296875, | |
| "learning_rate": 4.1693137748017915e-07, | |
| "loss": -0.0106, | |
| "reward": -0.10158013552427292, | |
| "reward_std": 0.10436173714697361, | |
| "rewards/cosine_scaled_reward": -0.050790068693459034, | |
| "rewards/format_reward": 0.0, | |
| "step": 318 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1492.1548156738281, | |
| "epoch": 1.276, | |
| "grad_norm": 0.1889064460992813, | |
| "kl": 0.16552734375, | |
| "learning_rate": 4.1393354916230005e-07, | |
| "loss": -0.0519, | |
| "reward": -0.1043181549757719, | |
| "reward_std": 0.13521616160869598, | |
| "rewards/cosine_scaled_reward": -0.0521590793505311, | |
| "rewards/format_reward": 0.0, | |
| "step": 319 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.2261962890625, | |
| "epoch": 1.28, | |
| "grad_norm": 0.12251739203929901, | |
| "kl": 0.18359375, | |
| "learning_rate": 4.1094235253127374e-07, | |
| "loss": -0.0218, | |
| "reward": -0.0895642340183258, | |
| "reward_std": 0.11007728800177574, | |
| "rewards/cosine_scaled_reward": -0.04478211794048548, | |
| "rewards/format_reward": 0.0, | |
| "step": 320 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1486.0059814453125, | |
| "epoch": 1.284, | |
| "grad_norm": 0.11772434413433075, | |
| "kl": 0.152587890625, | |
| "learning_rate": 4.079579333738039e-07, | |
| "loss": -0.0484, | |
| "reward": -0.1089986227452755, | |
| "reward_std": 0.12628877721726894, | |
| "rewards/cosine_scaled_reward": -0.0544993132352829, | |
| "rewards/format_reward": 0.0, | |
| "step": 321 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.9940490722656, | |
| "epoch": 1.288, | |
| "grad_norm": 0.11078700423240662, | |
| "kl": 0.1650390625, | |
| "learning_rate": 4.0498043714627006e-07, | |
| "loss": -0.0131, | |
| "reward": -0.0795932961627841, | |
| "reward_std": 0.08838632330298424, | |
| "rewards/cosine_scaled_reward": -0.03979664808139205, | |
| "rewards/format_reward": 0.0, | |
| "step": 322 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1493.6488342285156, | |
| "epoch": 1.292, | |
| "grad_norm": 0.11576636880636215, | |
| "kl": 0.16064453125, | |
| "learning_rate": 4.020100089676376e-07, | |
| "loss": -0.0516, | |
| "reward": -0.09283129125833511, | |
| "reward_std": 0.11138802394270897, | |
| "rewards/cosine_scaled_reward": -0.04641564283519983, | |
| "rewards/format_reward": 0.0, | |
| "step": 323 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.1904907226562, | |
| "epoch": 1.296, | |
| "grad_norm": 0.0929921567440033, | |
| "kl": 0.163818359375, | |
| "learning_rate": 3.9904679361238526e-07, | |
| "loss": -0.0256, | |
| "reward": -0.11852945201098919, | |
| "reward_std": 0.14786842092871666, | |
| "rewards/cosine_scaled_reward": -0.059264726005494595, | |
| "rewards/format_reward": 0.0, | |
| "step": 324 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.8809814453125, | |
| "epoch": 1.3, | |
| "grad_norm": 0.10295694321393967, | |
| "kl": 0.151123046875, | |
| "learning_rate": 3.9609093550344907e-07, | |
| "loss": -0.025, | |
| "reward": -0.09280366078019142, | |
| "reward_std": 0.1096403207629919, | |
| "rewards/cosine_scaled_reward": -0.046401829458773136, | |
| "rewards/format_reward": 0.0, | |
| "step": 325 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1526.6428833007812, | |
| "epoch": 1.304, | |
| "grad_norm": 0.08783736079931259, | |
| "kl": 0.15576171875, | |
| "learning_rate": 3.931425787051832e-07, | |
| "loss": -0.0069, | |
| "reward": -0.10956737771630287, | |
| "reward_std": 0.11006363853812218, | |
| "rewards/cosine_scaled_reward": -0.054783688858151436, | |
| "rewards/format_reward": 0.0, | |
| "step": 326 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1495.7024230957031, | |
| "epoch": 1.308, | |
| "grad_norm": 0.10409428924322128, | |
| "kl": 0.15234375, | |
| "learning_rate": 3.902018669163384e-07, | |
| "loss": -0.0457, | |
| "reward": -0.10653090476989746, | |
| "reward_std": 0.12193662486970425, | |
| "rewards/cosine_scaled_reward": -0.053265451453626156, | |
| "rewards/format_reward": 0.0, | |
| "step": 327 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1530.3988037109375, | |
| "epoch": 1.312, | |
| "grad_norm": 0.09973278641700745, | |
| "kl": 0.151123046875, | |
| "learning_rate": 3.872689434630585e-07, | |
| "loss": -0.0022, | |
| "reward": -0.08937697485089302, | |
| "reward_std": 0.09377033449709415, | |
| "rewards/cosine_scaled_reward": -0.04468849208205938, | |
| "rewards/format_reward": 0.0, | |
| "step": 328 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1499.6607360839844, | |
| "epoch": 1.316, | |
| "grad_norm": 0.13606639206409454, | |
| "kl": 0.166259765625, | |
| "learning_rate": 3.843439512918949e-07, | |
| "loss": -0.0237, | |
| "reward": -0.11537123657763004, | |
| "reward_std": 0.1290461514145136, | |
| "rewards/cosine_scaled_reward": -0.05768561642616987, | |
| "rewards/format_reward": 0.0, | |
| "step": 329 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.7857666015625, | |
| "epoch": 1.32, | |
| "grad_norm": 0.10613211989402771, | |
| "kl": 0.167724609375, | |
| "learning_rate": 3.8142703296283953e-07, | |
| "loss": -0.0159, | |
| "reward": -0.09533977694809437, | |
| "reward_std": 0.10748440586030483, | |
| "rewards/cosine_scaled_reward": -0.04766988940536976, | |
| "rewards/format_reward": 0.0, | |
| "step": 330 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1511.4464721679688, | |
| "epoch": 1.324, | |
| "grad_norm": 0.17454038560390472, | |
| "kl": 0.170166015625, | |
| "learning_rate": 3.785183306423767e-07, | |
| "loss": -0.0282, | |
| "reward": -0.015690762549638748, | |
| "reward_std": 0.0955708883702755, | |
| "rewards/cosine_scaled_reward": -0.007845382206141949, | |
| "rewards/format_reward": 0.0, | |
| "step": 331 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1499.7024230957031, | |
| "epoch": 1.328, | |
| "grad_norm": 0.10816742479801178, | |
| "kl": 0.153564453125, | |
| "learning_rate": 3.7561798609655373e-07, | |
| "loss": -0.0399, | |
| "reward": -0.09764312580227852, | |
| "reward_std": 0.10405797138810158, | |
| "rewards/cosine_scaled_reward": -0.04882156103849411, | |
| "rewards/format_reward": 0.0, | |
| "step": 332 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.2738037109375, | |
| "epoch": 1.332, | |
| "grad_norm": 0.09580235928297043, | |
| "kl": 0.1630859375, | |
| "learning_rate": 3.72726140684072e-07, | |
| "loss": -0.0238, | |
| "reward": -0.0930531919002533, | |
| "reward_std": 0.10378883965313435, | |
| "rewards/cosine_scaled_reward": -0.0465265978127718, | |
| "rewards/format_reward": 0.0, | |
| "step": 333 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1497.4762268066406, | |
| "epoch": 1.336, | |
| "grad_norm": 0.18943195044994354, | |
| "kl": 0.186767578125, | |
| "learning_rate": 3.6984293534939737e-07, | |
| "loss": -0.0458, | |
| "reward": -0.09320422261953354, | |
| "reward_std": 0.11783652380108833, | |
| "rewards/cosine_scaled_reward": -0.04660210944712162, | |
| "rewards/format_reward": 0.0, | |
| "step": 334 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1511.4583740234375, | |
| "epoch": 1.34, | |
| "grad_norm": 0.11527442187070847, | |
| "kl": 0.169677734375, | |
| "learning_rate": 3.6696851061588994e-07, | |
| "loss": -0.0222, | |
| "reward": -0.09490611962974072, | |
| "reward_std": 0.106621278449893, | |
| "rewards/cosine_scaled_reward": -0.04745305888354778, | |
| "rewards/format_reward": 0.0, | |
| "step": 335 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.3333435058594, | |
| "epoch": 1.3439999999999999, | |
| "grad_norm": 0.1746179610490799, | |
| "kl": 0.15966796875, | |
| "learning_rate": 3.641030065789562e-07, | |
| "loss": -0.0303, | |
| "reward": -0.0963439904153347, | |
| "reward_std": 0.11076842434704304, | |
| "rewards/cosine_scaled_reward": -0.04817199241369963, | |
| "rewards/format_reward": 0.0, | |
| "step": 336 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1512.5893249511719, | |
| "epoch": 1.3479999999999999, | |
| "grad_norm": 0.1353609561920166, | |
| "kl": 0.16748046875, | |
| "learning_rate": 3.612465628992203e-07, | |
| "loss": -0.0247, | |
| "reward": -0.09672348201274872, | |
| "reward_std": 0.1137369517236948, | |
| "rewards/cosine_scaled_reward": -0.04836174100637436, | |
| "rewards/format_reward": 0.0, | |
| "step": 337 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.3809509277344, | |
| "epoch": 1.3519999999999999, | |
| "grad_norm": 0.15681445598602295, | |
| "kl": 0.156982421875, | |
| "learning_rate": 3.5839931879571725e-07, | |
| "loss": -0.0329, | |
| "reward": -0.09822369925677776, | |
| "reward_std": 0.11475454457104206, | |
| "rewards/cosine_scaled_reward": -0.04911184962838888, | |
| "rewards/format_reward": 0.0, | |
| "step": 338 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1526.2619323730469, | |
| "epoch": 1.3559999999999999, | |
| "grad_norm": 0.12026900053024292, | |
| "kl": 0.15185546875, | |
| "learning_rate": 3.555614130391079e-07, | |
| "loss": -0.0027, | |
| "reward": -0.09019140899181366, | |
| "reward_std": 0.08315368928015232, | |
| "rewards/cosine_scaled_reward": -0.045095707289874554, | |
| "rewards/format_reward": 0.0, | |
| "step": 339 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.3988342285156, | |
| "epoch": 1.3599999999999999, | |
| "grad_norm": 0.189870685338974, | |
| "kl": 0.185791015625, | |
| "learning_rate": 3.5273298394491515e-07, | |
| "loss": -0.0085, | |
| "reward": -0.08764730766415596, | |
| "reward_std": 0.08974755555391312, | |
| "rewards/cosine_scaled_reward": -0.043823654763400555, | |
| "rewards/format_reward": 0.0, | |
| "step": 340 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1526.1309509277344, | |
| "epoch": 1.3639999999999999, | |
| "grad_norm": 0.11866843700408936, | |
| "kl": 0.1502685546875, | |
| "learning_rate": 3.4991416936678276e-07, | |
| "loss": 0.0082, | |
| "reward": -0.10054401028901339, | |
| "reward_std": 0.091705821454525, | |
| "rewards/cosine_scaled_reward": -0.05027200886979699, | |
| "rewards/format_reward": 0.0, | |
| "step": 341 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.6190795898438, | |
| "epoch": 1.3679999999999999, | |
| "grad_norm": 0.07965697348117828, | |
| "kl": 0.160888671875, | |
| "learning_rate": 3.471051066897562e-07, | |
| "loss": -0.0327, | |
| "reward": -0.03098013624548912, | |
| "reward_std": 0.10922298207879066, | |
| "rewards/cosine_scaled_reward": -0.015490064397454262, | |
| "rewards/format_reward": 0.0, | |
| "step": 342 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1515.702392578125, | |
| "epoch": 1.3719999999999999, | |
| "grad_norm": 0.12758509814739227, | |
| "kl": 0.172607421875, | |
| "learning_rate": 3.4430593282358777e-07, | |
| "loss": -0.0202, | |
| "reward": -0.09887174144387245, | |
| "reward_std": 0.11539069190621376, | |
| "rewards/cosine_scaled_reward": -0.049435872584581375, | |
| "rewards/format_reward": 0.0, | |
| "step": 343 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.0535888671875, | |
| "epoch": 1.376, | |
| "grad_norm": 0.09368550777435303, | |
| "kl": 0.18115234375, | |
| "learning_rate": 3.4151678419606233e-07, | |
| "loss": -0.0143, | |
| "reward": -0.09874763153493404, | |
| "reward_std": 0.0960962763056159, | |
| "rewards/cosine_scaled_reward": -0.049373818561434746, | |
| "rewards/format_reward": 0.0, | |
| "step": 344 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1505.8631286621094, | |
| "epoch": 1.38, | |
| "grad_norm": 0.10917885601520538, | |
| "kl": 0.177001953125, | |
| "learning_rate": 3.387377967463493e-07, | |
| "loss": -0.0331, | |
| "reward": -0.10075951926410198, | |
| "reward_std": 0.11745327524840832, | |
| "rewards/cosine_scaled_reward": -0.050379764288663864, | |
| "rewards/format_reward": 0.0, | |
| "step": 345 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.7976379394531, | |
| "epoch": 1.384, | |
| "grad_norm": 0.08625519275665283, | |
| "kl": 0.1611328125, | |
| "learning_rate": 3.359691059183761e-07, | |
| "loss": -0.0277, | |
| "reward": -0.11206395924091339, | |
| "reward_std": 0.13379977643489838, | |
| "rewards/cosine_scaled_reward": -0.05603197868913412, | |
| "rewards/format_reward": 0.0, | |
| "step": 346 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.77978515625, | |
| "epoch": 1.388, | |
| "grad_norm": 0.09115591645240784, | |
| "kl": 0.164306640625, | |
| "learning_rate": 3.3321084665422803e-07, | |
| "loss": -0.0129, | |
| "reward": -0.08784853294491768, | |
| "reward_std": 0.09035127516835928, | |
| "rewards/cosine_scaled_reward": -0.043924265541136265, | |
| "rewards/format_reward": 0.0, | |
| "step": 347 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.4702453613281, | |
| "epoch": 1.392, | |
| "grad_norm": 0.12537740170955658, | |
| "kl": 0.155517578125, | |
| "learning_rate": 3.3046315338757026e-07, | |
| "loss": -0.0323, | |
| "reward": -0.11530621163547039, | |
| "reward_std": 0.1266392320394516, | |
| "rewards/cosine_scaled_reward": -0.05765310861170292, | |
| "rewards/format_reward": 0.0, | |
| "step": 348 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1496.6726379394531, | |
| "epoch": 1.396, | |
| "grad_norm": 0.11020371317863464, | |
| "kl": 0.158935546875, | |
| "learning_rate": 3.2772616003709616e-07, | |
| "loss": -0.0268, | |
| "reward": -0.10983618721365929, | |
| "reward_std": 0.10708382353186607, | |
| "rewards/cosine_scaled_reward": -0.054918091744184494, | |
| "rewards/format_reward": 0.0, | |
| "step": 349 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1530.0416564941406, | |
| "epoch": 1.4, | |
| "grad_norm": 0.17068089544773102, | |
| "kl": 0.163818359375, | |
| "learning_rate": 3.250000000000001e-07, | |
| "loss": 0.0002, | |
| "reward": -0.08832419849932194, | |
| "reward_std": 0.09396599233150482, | |
| "rewards/cosine_scaled_reward": -0.04416209738701582, | |
| "rewards/format_reward": 0.0, | |
| "step": 350 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1515.4345397949219, | |
| "epoch": 1.404, | |
| "grad_norm": 0.1316055804491043, | |
| "kl": 0.1689453125, | |
| "learning_rate": 3.222848061454764e-07, | |
| "loss": -0.0178, | |
| "reward": -0.08349752612411976, | |
| "reward_std": 0.08428733702749014, | |
| "rewards/cosine_scaled_reward": -0.041748762130737305, | |
| "rewards/format_reward": 0.0, | |
| "step": 351 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1501.7976379394531, | |
| "epoch": 1.408, | |
| "grad_norm": 0.1486114114522934, | |
| "kl": 0.16552734375, | |
| "learning_rate": 3.195807108082429e-07, | |
| "loss": -0.0412, | |
| "reward": -0.0832710936665535, | |
| "reward_std": 0.10713749751448631, | |
| "rewards/cosine_scaled_reward": -0.04163554683327675, | |
| "rewards/format_reward": 0.0, | |
| "step": 352 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.6428833007812, | |
| "epoch": 1.412, | |
| "grad_norm": 0.12983661890029907, | |
| "kl": 0.163330078125, | |
| "learning_rate": 3.168878457820915e-07, | |
| "loss": -0.0096, | |
| "reward": -0.10671682469546795, | |
| "reward_std": 0.11679115891456604, | |
| "rewards/cosine_scaled_reward": -0.0533584114164114, | |
| "rewards/format_reward": 0.0, | |
| "step": 353 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1497.7976379394531, | |
| "epoch": 1.416, | |
| "grad_norm": 0.09482970088720322, | |
| "kl": 0.163818359375, | |
| "learning_rate": 3.142063423134644e-07, | |
| "loss": -0.0377, | |
| "reward": -0.10173431225121021, | |
| "reward_std": 0.11217576451599598, | |
| "rewards/cosine_scaled_reward": -0.050867156125605106, | |
| "rewards/format_reward": 0.0, | |
| "step": 354 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1511.9226379394531, | |
| "epoch": 1.42, | |
| "grad_norm": 0.11015576124191284, | |
| "kl": 0.16796875, | |
| "learning_rate": 3.115363310950578e-07, | |
| "loss": -0.027, | |
| "reward": -0.10424264334142208, | |
| "reward_std": 0.10744853690266609, | |
| "rewards/cosine_scaled_reward": -0.05212132353335619, | |
| "rewards/format_reward": 0.0, | |
| "step": 355 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.7143249511719, | |
| "epoch": 1.424, | |
| "grad_norm": 0.1039690375328064, | |
| "kl": 0.15771484375, | |
| "learning_rate": 3.0887794225945143e-07, | |
| "loss": -0.0104, | |
| "reward": -0.11364280618727207, | |
| "reward_std": 0.11577463708817959, | |
| "rewards/cosine_scaled_reward": -0.05682140402495861, | |
| "rewards/format_reward": 0.0, | |
| "step": 356 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1522.547607421875, | |
| "epoch": 1.428, | |
| "grad_norm": 0.13563141226768494, | |
| "kl": 0.16943359375, | |
| "learning_rate": 3.062313053727671e-07, | |
| "loss": -0.0127, | |
| "reward": -0.09091841243207455, | |
| "reward_std": 0.1005500927567482, | |
| "rewards/cosine_scaled_reward": -0.04545920621603727, | |
| "rewards/format_reward": 0.0, | |
| "step": 357 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.1607360839844, | |
| "epoch": 1.432, | |
| "grad_norm": 0.12330485880374908, | |
| "kl": 0.175048828125, | |
| "learning_rate": 3.0359654942835247e-07, | |
| "loss": -0.0276, | |
| "reward": -0.09949876181781292, | |
| "reward_std": 0.10788233578205109, | |
| "rewards/cosine_scaled_reward": -0.049749381840229034, | |
| "rewards/format_reward": 0.0, | |
| "step": 358 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1526.1785888671875, | |
| "epoch": 1.436, | |
| "grad_norm": 0.1008228212594986, | |
| "kl": 0.172607421875, | |
| "learning_rate": 3.0097380284049523e-07, | |
| "loss": -0.0072, | |
| "reward": -0.08119065128266811, | |
| "reward_std": 0.09274793975055218, | |
| "rewards/cosine_scaled_reward": -0.04059532564133406, | |
| "rewards/format_reward": 0.0, | |
| "step": 359 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.6964721679688, | |
| "epoch": 1.44, | |
| "grad_norm": 0.11536505818367004, | |
| "kl": 0.1640625, | |
| "learning_rate": 2.9836319343816397e-07, | |
| "loss": -0.0305, | |
| "reward": -0.10905157402157784, | |
| "reward_std": 0.11038926243782043, | |
| "rewards/cosine_scaled_reward": -0.05452578607946634, | |
| "rewards/format_reward": 0.0, | |
| "step": 360 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1518.827392578125, | |
| "epoch": 1.444, | |
| "grad_norm": 0.12276989966630936, | |
| "kl": 0.1514892578125, | |
| "learning_rate": 2.9576484845877793e-07, | |
| "loss": -0.0175, | |
| "reward": -0.08610734064131975, | |
| "reward_std": 0.09063750877976418, | |
| "rewards/cosine_scaled_reward": -0.043053670320659876, | |
| "rewards/format_reward": 0.0, | |
| "step": 361 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.4702453613281, | |
| "epoch": 1.448, | |
| "grad_norm": 0.08738084882497787, | |
| "kl": 0.18017578125, | |
| "learning_rate": 2.931788945420058e-07, | |
| "loss": -0.0112, | |
| "reward": -0.09291153028607368, | |
| "reward_std": 0.09842956997454166, | |
| "rewards/cosine_scaled_reward": -0.046455767937004566, | |
| "rewards/format_reward": 0.0, | |
| "step": 362 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1509.0774230957031, | |
| "epoch": 1.452, | |
| "grad_norm": 0.11346267908811569, | |
| "kl": 0.175048828125, | |
| "learning_rate": 2.9060545772359305e-07, | |
| "loss": -0.0277, | |
| "reward": -0.11039301194250584, | |
| "reward_std": 0.12665076181292534, | |
| "rewards/cosine_scaled_reward": -0.05519650410860777, | |
| "rewards/format_reward": 0.0, | |
| "step": 363 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1524.1964416503906, | |
| "epoch": 1.456, | |
| "grad_norm": 0.14776764810085297, | |
| "kl": 0.150634765625, | |
| "learning_rate": 2.8804466342921987e-07, | |
| "loss": -0.006, | |
| "reward": -0.022786946967244148, | |
| "reward_std": 0.10106383822858334, | |
| "rewards/cosine_scaled_reward": -0.011393471620976925, | |
| "rewards/format_reward": 0.0, | |
| "step": 364 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1493.607177734375, | |
| "epoch": 1.46, | |
| "grad_norm": 0.09510252624750137, | |
| "kl": 0.153076171875, | |
| "learning_rate": 2.854966364683872e-07, | |
| "loss": -0.0487, | |
| "reward": -0.10556191392242908, | |
| "reward_std": 0.10932666808366776, | |
| "rewards/cosine_scaled_reward": -0.05278095696121454, | |
| "rewards/format_reward": 0.0, | |
| "step": 365 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1529.875, | |
| "epoch": 1.464, | |
| "grad_norm": 0.15466201305389404, | |
| "kl": 0.15966796875, | |
| "learning_rate": 2.829615010283344e-07, | |
| "loss": 0.0007, | |
| "reward": -0.09042776376008987, | |
| "reward_std": 0.10296636447310448, | |
| "rewards/cosine_scaled_reward": -0.04521388094872236, | |
| "rewards/format_reward": 0.0, | |
| "step": 366 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.6786193847656, | |
| "epoch": 1.468, | |
| "grad_norm": 0.08847711980342865, | |
| "kl": 0.17041015625, | |
| "learning_rate": 2.8043938066798645e-07, | |
| "loss": -0.0311, | |
| "reward": -0.10542780347168446, | |
| "reward_std": 0.11852787062525749, | |
| "rewards/cosine_scaled_reward": -0.05271390173584223, | |
| "rewards/format_reward": 0.0, | |
| "step": 367 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.7083435058594, | |
| "epoch": 1.472, | |
| "grad_norm": 0.1147918626666069, | |
| "kl": 0.17041015625, | |
| "learning_rate": 2.7793039831193133e-07, | |
| "loss": -0.0404, | |
| "reward": -0.10119456797838211, | |
| "reward_std": 0.1359020471572876, | |
| "rewards/cosine_scaled_reward": -0.050597282126545906, | |
| "rewards/format_reward": 0.0, | |
| "step": 368 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.5774230957031, | |
| "epoch": 1.476, | |
| "grad_norm": 0.0867527574300766, | |
| "kl": 0.154052734375, | |
| "learning_rate": 2.7543467624442956e-07, | |
| "loss": -0.0327, | |
| "reward": -0.09615712240338326, | |
| "reward_std": 0.11924017407000065, | |
| "rewards/cosine_scaled_reward": -0.0480785621330142, | |
| "rewards/format_reward": 0.0, | |
| "step": 369 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1515.5357360839844, | |
| "epoch": 1.48, | |
| "grad_norm": 0.07760825008153915, | |
| "kl": 0.172607421875, | |
| "learning_rate": 2.729523361034538e-07, | |
| "loss": -0.012, | |
| "reward": -0.09595928713679314, | |
| "reward_std": 0.10662926360964775, | |
| "rewards/cosine_scaled_reward": -0.04797964543104172, | |
| "rewards/format_reward": 0.0, | |
| "step": 370 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.3809814453125, | |
| "epoch": 1.484, | |
| "grad_norm": 0.1310672163963318, | |
| "kl": 0.1671142578125, | |
| "learning_rate": 2.7048349887476037e-07, | |
| "loss": -0.0266, | |
| "reward": -0.08946863748133183, | |
| "reward_std": 0.0914797130972147, | |
| "rewards/cosine_scaled_reward": -0.04473431780934334, | |
| "rewards/format_reward": 0.0, | |
| "step": 371 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1491.5179138183594, | |
| "epoch": 1.488, | |
| "grad_norm": 0.08744286000728607, | |
| "kl": 0.156494140625, | |
| "learning_rate": 2.6802828488599294e-07, | |
| "loss": -0.045, | |
| "reward": -0.1184717956930399, | |
| "reward_std": 0.13941991329193115, | |
| "rewards/cosine_scaled_reward": -0.059235901571810246, | |
| "rewards/format_reward": 0.0, | |
| "step": 372 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1521.0238342285156, | |
| "epoch": 1.492, | |
| "grad_norm": 0.1646253615617752, | |
| "kl": 0.16357421875, | |
| "learning_rate": 2.655868138008171e-07, | |
| "loss": 0.0142, | |
| "reward": -0.084196537733078, | |
| "reward_std": 0.07485349848866463, | |
| "rewards/cosine_scaled_reward": -0.042098269797861576, | |
| "rewards/format_reward": 0.0, | |
| "step": 373 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1523.375, | |
| "epoch": 1.496, | |
| "grad_norm": 0.11430079489946365, | |
| "kl": 0.172607421875, | |
| "learning_rate": 2.631592046130896e-07, | |
| "loss": -0.0099, | |
| "reward": -0.07816067058593035, | |
| "reward_std": 0.08419617265462875, | |
| "rewards/cosine_scaled_reward": -0.03908033389598131, | |
| "rewards/format_reward": 0.0, | |
| "step": 374 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.4524230957031, | |
| "epoch": 1.5, | |
| "grad_norm": 0.14677973091602325, | |
| "kl": 0.18603515625, | |
| "learning_rate": 2.6074557564105724e-07, | |
| "loss": -0.0194, | |
| "reward": -0.08955581299960613, | |
| "reward_std": 0.09336170181632042, | |
| "rewards/cosine_scaled_reward": -0.04477790556848049, | |
| "rewards/format_reward": 0.0, | |
| "step": 375 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1527.3988342285156, | |
| "epoch": 1.504, | |
| "grad_norm": 0.12783505022525787, | |
| "kl": 0.159912109375, | |
| "learning_rate": 2.583460445215911e-07, | |
| "loss": -0.0049, | |
| "reward": -0.0952699575573206, | |
| "reward_std": 0.09568927250802517, | |
| "rewards/cosine_scaled_reward": -0.047634975984692574, | |
| "rewards/format_reward": 0.0, | |
| "step": 376 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1520.7678833007812, | |
| "epoch": 1.508, | |
| "grad_norm": 0.1176699697971344, | |
| "kl": 0.1650390625, | |
| "learning_rate": 2.5596072820445254e-07, | |
| "loss": -0.011, | |
| "reward": -0.019147060811519623, | |
| "reward_std": 0.09721549972891808, | |
| "rewards/cosine_scaled_reward": -0.009573530405759811, | |
| "rewards/format_reward": 0.0, | |
| "step": 377 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.297607421875, | |
| "epoch": 1.512, | |
| "grad_norm": 0.11060648411512375, | |
| "kl": 0.1650390625, | |
| "learning_rate": 2.5358974294659373e-07, | |
| "loss": -0.0134, | |
| "reward": -0.09460222348570824, | |
| "reward_std": 0.1032384280115366, | |
| "rewards/cosine_scaled_reward": -0.04730111453682184, | |
| "rewards/format_reward": 0.0, | |
| "step": 378 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1524.2916870117188, | |
| "epoch": 1.516, | |
| "grad_norm": 0.12652094662189484, | |
| "kl": 0.16064453125, | |
| "learning_rate": 2.512332043064913e-07, | |
| "loss": -0.0078, | |
| "reward": -0.07960367575287819, | |
| "reward_std": 0.08834364637732506, | |
| "rewards/cosine_scaled_reward": -0.03980184067040682, | |
| "rewards/format_reward": 0.0, | |
| "step": 379 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1524.8750305175781, | |
| "epoch": 1.52, | |
| "grad_norm": 0.10227189213037491, | |
| "kl": 0.16748046875, | |
| "learning_rate": 2.488912271385139e-07, | |
| "loss": -0.0064, | |
| "reward": -0.08977451547980309, | |
| "reward_std": 0.1080553438514471, | |
| "rewards/cosine_scaled_reward": -0.04488725960254669, | |
| "rewards/format_reward": 0.0, | |
| "step": 380 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.3511962890625, | |
| "epoch": 1.524, | |
| "grad_norm": 0.12043853104114532, | |
| "kl": 0.17138671875, | |
| "learning_rate": 2.465639255873246e-07, | |
| "loss": -0.035, | |
| "reward": -0.11090395227074623, | |
| "reward_std": 0.12006122805178165, | |
| "rewards/cosine_scaled_reward": -0.05545197706669569, | |
| "rewards/format_reward": 0.0, | |
| "step": 381 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1523.5178833007812, | |
| "epoch": 1.528, | |
| "grad_norm": 0.13229811191558838, | |
| "kl": 0.170654296875, | |
| "learning_rate": 2.4425141308231765e-07, | |
| "loss": -0.0068, | |
| "reward": -0.09728248044848442, | |
| "reward_std": 0.107889199629426, | |
| "rewards/cosine_scaled_reward": -0.048641239292919636, | |
| "rewards/format_reward": 0.0, | |
| "step": 382 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1527.2261962890625, | |
| "epoch": 1.532, | |
| "grad_norm": 0.10695023834705353, | |
| "kl": 0.1630859375, | |
| "learning_rate": 2.4195380233209006e-07, | |
| "loss": -0.0022, | |
| "reward": -0.09213725849986076, | |
| "reward_std": 0.10676849260926247, | |
| "rewards/cosine_scaled_reward": -0.046068630181252956, | |
| "rewards/format_reward": 0.0, | |
| "step": 383 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.7083740234375, | |
| "epoch": 1.536, | |
| "grad_norm": 0.13451160490512848, | |
| "kl": 0.150390625, | |
| "learning_rate": 2.3967120531894857e-07, | |
| "loss": -0.0256, | |
| "reward": -0.10359417460858822, | |
| "reward_std": 0.12065772153437138, | |
| "rewards/cosine_scaled_reward": -0.05179708730429411, | |
| "rewards/format_reward": 0.0, | |
| "step": 384 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1501.4940795898438, | |
| "epoch": 1.54, | |
| "grad_norm": 0.1391247659921646, | |
| "kl": 0.16357421875, | |
| "learning_rate": 2.374037332934512e-07, | |
| "loss": -0.0409, | |
| "reward": -0.09379393607378006, | |
| "reward_std": 0.10166217759251595, | |
| "rewards/cosine_scaled_reward": -0.04689696989953518, | |
| "rewards/format_reward": 0.0, | |
| "step": 385 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.2500305175781, | |
| "epoch": 1.544, | |
| "grad_norm": 0.1330152153968811, | |
| "kl": 0.166748046875, | |
| "learning_rate": 2.3515149676898552e-07, | |
| "loss": 0.0087, | |
| "reward": -0.07589279673993587, | |
| "reward_std": 0.09089674055576324, | |
| "rewards/cosine_scaled_reward": -0.03794640023261309, | |
| "rewards/format_reward": 0.0, | |
| "step": 386 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1488.6131286621094, | |
| "epoch": 1.548, | |
| "grad_norm": 0.10263092815876007, | |
| "kl": 0.1630859375, | |
| "learning_rate": 2.3291460551638237e-07, | |
| "loss": -0.0521, | |
| "reward": -0.12465786561369896, | |
| "reward_std": 0.1609484814107418, | |
| "rewards/cosine_scaled_reward": -0.06232893466949463, | |
| "rewards/format_reward": 0.0, | |
| "step": 387 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.8928833007812, | |
| "epoch": 1.552, | |
| "grad_norm": 0.09812143445014954, | |
| "kl": 0.15966796875, | |
| "learning_rate": 2.306931685585657e-07, | |
| "loss": -0.015, | |
| "reward": -0.0796813191846013, | |
| "reward_std": 0.08767454512417316, | |
| "rewards/cosine_scaled_reward": -0.039840660989284515, | |
| "rewards/format_reward": 0.0, | |
| "step": 388 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1505.2738342285156, | |
| "epoch": 1.556, | |
| "grad_norm": 0.16943664848804474, | |
| "kl": 0.1591796875, | |
| "learning_rate": 2.2848729416523859e-07, | |
| "loss": -0.0254, | |
| "reward": -0.11296515539288521, | |
| "reward_std": 0.12935122102499008, | |
| "rewards/cosine_scaled_reward": -0.05648257676512003, | |
| "rewards/format_reward": 0.0, | |
| "step": 389 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1511.9345397949219, | |
| "epoch": 1.56, | |
| "grad_norm": 0.1270017921924591, | |
| "kl": 0.1513671875, | |
| "learning_rate": 2.2629708984760706e-07, | |
| "loss": -0.0186, | |
| "reward": -0.08384528011083603, | |
| "reward_std": 0.08424858003854752, | |
| "rewards/cosine_scaled_reward": -0.04192264098674059, | |
| "rewards/format_reward": 0.0, | |
| "step": 390 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1525.982177734375, | |
| "epoch": 1.564, | |
| "grad_norm": 0.16950343549251556, | |
| "kl": 0.169921875, | |
| "learning_rate": 2.2412266235313973e-07, | |
| "loss": -0.0058, | |
| "reward": -0.08042520564049482, | |
| "reward_std": 0.08201098442077637, | |
| "rewards/cosine_scaled_reward": -0.04021260142326355, | |
| "rewards/format_reward": 0.0, | |
| "step": 391 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.6666564941406, | |
| "epoch": 1.568, | |
| "grad_norm": 0.13040253520011902, | |
| "kl": 0.14990234375, | |
| "learning_rate": 2.2196411766036487e-07, | |
| "loss": -0.0288, | |
| "reward": -0.10378818027675152, | |
| "reward_std": 0.1260694395750761, | |
| "rewards/cosine_scaled_reward": -0.05189409013837576, | |
| "rewards/format_reward": 0.0, | |
| "step": 392 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1518.4047546386719, | |
| "epoch": 1.572, | |
| "grad_norm": 0.11275047063827515, | |
| "kl": 0.1572265625, | |
| "learning_rate": 2.1982156097370557e-07, | |
| "loss": -0.0157, | |
| "reward": -0.09913922101259232, | |
| "reward_std": 0.10591815412044525, | |
| "rewards/cosine_scaled_reward": -0.04956961143761873, | |
| "rewards/format_reward": 0.0, | |
| "step": 393 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1524.8035888671875, | |
| "epoch": 1.576, | |
| "grad_norm": 0.11497998982667923, | |
| "kl": 0.17578125, | |
| "learning_rate": 2.1769509671835223e-07, | |
| "loss": -0.009, | |
| "reward": -0.08646929264068604, | |
| "reward_std": 0.09624841343611479, | |
| "rewards/cosine_scaled_reward": -0.04323464818298817, | |
| "rewards/format_reward": 0.0, | |
| "step": 394 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1518.5416870117188, | |
| "epoch": 1.58, | |
| "grad_norm": 0.19012141227722168, | |
| "kl": 0.148193359375, | |
| "learning_rate": 2.1558482853517253e-07, | |
| "loss": -0.0063, | |
| "reward": -0.09366242028772831, | |
| "reward_std": 0.1069308090955019, | |
| "rewards/cosine_scaled_reward": -0.046831210143864155, | |
| "rewards/format_reward": 0.0, | |
| "step": 395 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.1488342285156, | |
| "epoch": 1.584, | |
| "grad_norm": 0.1060405820608139, | |
| "kl": 0.156982421875, | |
| "learning_rate": 2.134908592756607e-07, | |
| "loss": -0.0309, | |
| "reward": -0.10527068562805653, | |
| "reward_std": 0.12328575551509857, | |
| "rewards/cosine_scaled_reward": -0.05263534560799599, | |
| "rewards/format_reward": 0.0, | |
| "step": 396 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.7916870117188, | |
| "epoch": 1.588, | |
| "grad_norm": 0.10801802575588226, | |
| "kl": 0.140869140625, | |
| "learning_rate": 2.1141329099692406e-07, | |
| "loss": -0.0205, | |
| "reward": -0.11307091265916824, | |
| "reward_std": 0.123080899938941, | |
| "rewards/cosine_scaled_reward": -0.05653545819222927, | |
| "rewards/format_reward": 0.0, | |
| "step": 397 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.5595703125, | |
| "epoch": 1.592, | |
| "grad_norm": 0.08320983499288559, | |
| "kl": 0.1591796875, | |
| "learning_rate": 2.0935222495670968e-07, | |
| "loss": -0.037, | |
| "reward": -0.09146481472998857, | |
| "reward_std": 0.09883083030581474, | |
| "rewards/cosine_scaled_reward": -0.04573240736499429, | |
| "rewards/format_reward": 0.0, | |
| "step": 398 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1531.6607360839844, | |
| "epoch": 1.596, | |
| "grad_norm": 0.09601892530918121, | |
| "kl": 0.1533203125, | |
| "learning_rate": 2.0730776160846853e-07, | |
| "loss": 0.0006, | |
| "reward": -0.08569015190005302, | |
| "reward_std": 0.0903671607375145, | |
| "rewards/cosine_scaled_reward": -0.04284507688134909, | |
| "rewards/format_reward": 0.0, | |
| "step": 399 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1501.8928833007812, | |
| "epoch": 1.6, | |
| "grad_norm": 0.12060719728469849, | |
| "kl": 0.16943359375, | |
| "learning_rate": 2.0528000059645995e-07, | |
| "loss": -0.0212, | |
| "reward": -0.09470336884260178, | |
| "reward_std": 0.11146636307239532, | |
| "rewards/cosine_scaled_reward": -0.04735168442130089, | |
| "rewards/format_reward": 0.0, | |
| "step": 400 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1523.4940795898438, | |
| "epoch": 1.604, | |
| "grad_norm": 0.10291819274425507, | |
| "kl": 0.1591796875, | |
| "learning_rate": 2.032690407508949e-07, | |
| "loss": -0.0098, | |
| "reward": -0.09152790158987045, | |
| "reward_std": 0.11163719370961189, | |
| "rewards/cosine_scaled_reward": -0.045763951260596514, | |
| "rewards/format_reward": 0.0, | |
| "step": 401 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1515.0952453613281, | |
| "epoch": 1.608, | |
| "grad_norm": 0.13543279469013214, | |
| "kl": 0.1689453125, | |
| "learning_rate": 2.0127498008311922e-07, | |
| "loss": -0.0193, | |
| "reward": -0.08428375516086817, | |
| "reward_std": 0.08265121094882488, | |
| "rewards/cosine_scaled_reward": -0.042141877580434084, | |
| "rewards/format_reward": 0.0, | |
| "step": 402 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.3154907226562, | |
| "epoch": 1.612, | |
| "grad_norm": 0.10554395616054535, | |
| "kl": 0.15380859375, | |
| "learning_rate": 1.9929791578083655e-07, | |
| "loss": -0.0233, | |
| "reward": -0.09138609375804663, | |
| "reward_std": 0.09994357451796532, | |
| "rewards/cosine_scaled_reward": -0.04569304594770074, | |
| "rewards/format_reward": 0.0, | |
| "step": 403 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1503.452392578125, | |
| "epoch": 1.616, | |
| "grad_norm": 0.10063979774713516, | |
| "kl": 0.155517578125, | |
| "learning_rate": 1.9733794420337213e-07, | |
| "loss": -0.0392, | |
| "reward": -0.100379329174757, | |
| "reward_std": 0.12372113950550556, | |
| "rewards/cosine_scaled_reward": -0.05018966645002365, | |
| "rewards/format_reward": 0.0, | |
| "step": 404 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.607177734375, | |
| "epoch": 1.62, | |
| "grad_norm": 0.090563103556633, | |
| "kl": 0.163818359375, | |
| "learning_rate": 1.9539516087697517e-07, | |
| "loss": -0.0215, | |
| "reward": -0.08255079202353954, | |
| "reward_std": 0.09494246542453766, | |
| "rewards/cosine_scaled_reward": -0.041275396943092346, | |
| "rewards/format_reward": 0.0, | |
| "step": 405 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1524.3988342285156, | |
| "epoch": 1.624, | |
| "grad_norm": 0.10463332384824753, | |
| "kl": 0.154541015625, | |
| "learning_rate": 1.934696604901642e-07, | |
| "loss": -0.0101, | |
| "reward": -0.09653126262128353, | |
| "reward_std": 0.11365084536373615, | |
| "rewards/cosine_scaled_reward": -0.04826563224196434, | |
| "rewards/format_reward": 0.0, | |
| "step": 406 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.4345703125, | |
| "epoch": 1.6280000000000001, | |
| "grad_norm": 0.18202035129070282, | |
| "kl": 0.164794921875, | |
| "learning_rate": 1.915615368891117e-07, | |
| "loss": -0.0156, | |
| "reward": -0.10028301551938057, | |
| "reward_std": 0.12200421467423439, | |
| "rewards/cosine_scaled_reward": -0.050141509622335434, | |
| "rewards/format_reward": 0.0, | |
| "step": 407 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1501.7857360839844, | |
| "epoch": 1.6320000000000001, | |
| "grad_norm": 0.10041651129722595, | |
| "kl": 0.1552734375, | |
| "learning_rate": 1.8967088307307e-07, | |
| "loss": -0.0405, | |
| "reward": -0.1025087870657444, | |
| "reward_std": 0.12081354483962059, | |
| "rewards/cosine_scaled_reward": -0.051254394464194775, | |
| "rewards/format_reward": 0.0, | |
| "step": 408 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.65478515625, | |
| "epoch": 1.6360000000000001, | |
| "grad_norm": 0.0929652526974678, | |
| "kl": 0.164306640625, | |
| "learning_rate": 1.8779779118983867e-07, | |
| "loss": -0.0336, | |
| "reward": -0.10521730966866016, | |
| "reward_std": 0.12063234858214855, | |
| "rewards/cosine_scaled_reward": -0.05260865669697523, | |
| "rewards/format_reward": 0.0, | |
| "step": 409 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1489.9404907226562, | |
| "epoch": 1.6400000000000001, | |
| "grad_norm": 0.08884437382221222, | |
| "kl": 0.16552734375, | |
| "learning_rate": 1.8594235253127372e-07, | |
| "loss": -0.0292, | |
| "reward": -0.09861567430198193, | |
| "reward_std": 0.10845682211220264, | |
| "rewards/cosine_scaled_reward": -0.04930783715099096, | |
| "rewards/format_reward": 0.0, | |
| "step": 410 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.0476379394531, | |
| "epoch": 1.6440000000000001, | |
| "grad_norm": 0.1006086990237236, | |
| "kl": 0.156494140625, | |
| "learning_rate": 1.8410465752883758e-07, | |
| "loss": -0.021, | |
| "reward": -0.09761104919016361, | |
| "reward_std": 0.10258225724101067, | |
| "rewards/cosine_scaled_reward": -0.048805526457726955, | |
| "rewards/format_reward": 0.0, | |
| "step": 411 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1519.90478515625, | |
| "epoch": 1.6480000000000001, | |
| "grad_norm": 0.11515481770038605, | |
| "kl": 0.16943359375, | |
| "learning_rate": 1.822847957491922e-07, | |
| "loss": -0.016, | |
| "reward": -0.08585721254348755, | |
| "reward_std": 0.10039913840591908, | |
| "rewards/cosine_scaled_reward": -0.042928608134388924, | |
| "rewards/format_reward": 0.0, | |
| "step": 412 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.357177734375, | |
| "epoch": 1.6520000000000001, | |
| "grad_norm": 0.09629681706428528, | |
| "kl": 0.159423828125, | |
| "learning_rate": 1.804828558898332e-07, | |
| "loss": -0.0283, | |
| "reward": -0.08894845098257065, | |
| "reward_std": 0.10276514105498791, | |
| "rewards/cosine_scaled_reward": -0.04447422595694661, | |
| "rewards/format_reward": 0.0, | |
| "step": 413 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1511.0357360839844, | |
| "epoch": 1.6560000000000001, | |
| "grad_norm": 0.12116753309965134, | |
| "kl": 0.169189453125, | |
| "learning_rate": 1.7869892577476722e-07, | |
| "loss": -0.0245, | |
| "reward": -0.10262815281748772, | |
| "reward_std": 0.12108992040157318, | |
| "rewards/cosine_scaled_reward": -0.051314075477421284, | |
| "rewards/format_reward": 0.0, | |
| "step": 414 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1521.9345397949219, | |
| "epoch": 1.6600000000000001, | |
| "grad_norm": 0.10831650346517563, | |
| "kl": 0.163818359375, | |
| "learning_rate": 1.7693309235023127e-07, | |
| "loss": -0.0085, | |
| "reward": -0.08154256083071232, | |
| "reward_std": 0.09072042256593704, | |
| "rewards/cosine_scaled_reward": -0.04077128041535616, | |
| "rewards/format_reward": 0.0, | |
| "step": 415 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.0059814453125, | |
| "epoch": 1.6640000000000001, | |
| "grad_norm": 0.10683077573776245, | |
| "kl": 0.175537109375, | |
| "learning_rate": 1.7518544168045524e-07, | |
| "loss": -0.0242, | |
| "reward": -0.1116462592035532, | |
| "reward_std": 0.10574496164917946, | |
| "rewards/cosine_scaled_reward": -0.05582312494516373, | |
| "rewards/format_reward": 0.0, | |
| "step": 416 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1515.3393249511719, | |
| "epoch": 1.6680000000000001, | |
| "grad_norm": 0.14900319278240204, | |
| "kl": 0.1650390625, | |
| "learning_rate": 1.7345605894346726e-07, | |
| "loss": -0.021, | |
| "reward": -0.08745052106678486, | |
| "reward_std": 0.11200828477740288, | |
| "rewards/cosine_scaled_reward": -0.04372526053339243, | |
| "rewards/format_reward": 0.0, | |
| "step": 417 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1523.3690490722656, | |
| "epoch": 1.6720000000000002, | |
| "grad_norm": 0.11940804123878479, | |
| "kl": 0.154541015625, | |
| "learning_rate": 1.7174502842694212e-07, | |
| "loss": -0.0124, | |
| "reward": -0.0070722997188568115, | |
| "reward_std": 0.09923059120774269, | |
| "rewards/cosine_scaled_reward": -0.0035361526533961296, | |
| "rewards/format_reward": 0.0, | |
| "step": 418 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1485.9583435058594, | |
| "epoch": 1.6760000000000002, | |
| "grad_norm": 0.08529967814683914, | |
| "kl": 0.172607421875, | |
| "learning_rate": 1.7005243352409333e-07, | |
| "loss": -0.0651, | |
| "reward": -0.04097301326692104, | |
| "reward_std": 0.15967968851327896, | |
| "rewards/cosine_scaled_reward": -0.02048650663346052, | |
| "rewards/format_reward": 0.0, | |
| "step": 419 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.0059814453125, | |
| "epoch": 1.6800000000000002, | |
| "grad_norm": 0.0939546748995781, | |
| "kl": 0.16015625, | |
| "learning_rate": 1.6837835672960831e-07, | |
| "loss": -0.0345, | |
| "reward": -0.091935895383358, | |
| "reward_std": 0.11023806594312191, | |
| "rewards/cosine_scaled_reward": -0.04596794489771128, | |
| "rewards/format_reward": 0.0, | |
| "step": 420 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1502.7500305175781, | |
| "epoch": 1.6840000000000002, | |
| "grad_norm": 0.114561066031456, | |
| "kl": 0.17724609375, | |
| "learning_rate": 1.6672287963562852e-07, | |
| "loss": -0.0193, | |
| "reward": -0.07856714259833097, | |
| "reward_std": 0.08897042460739613, | |
| "rewards/cosine_scaled_reward": -0.03928357409313321, | |
| "rewards/format_reward": 0.0, | |
| "step": 421 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1496.7261962890625, | |
| "epoch": 1.688, | |
| "grad_norm": 0.11227195709943771, | |
| "kl": 0.160888671875, | |
| "learning_rate": 1.6508608292777203e-07, | |
| "loss": -0.0359, | |
| "reward": -0.09738295152783394, | |
| "reward_std": 0.11914198100566864, | |
| "rewards/cosine_scaled_reward": -0.048691474832594395, | |
| "rewards/format_reward": 0.0, | |
| "step": 422 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1511.1964416503906, | |
| "epoch": 1.692, | |
| "grad_norm": 0.13162577152252197, | |
| "kl": 0.181640625, | |
| "learning_rate": 1.6346804638120098e-07, | |
| "loss": -0.0245, | |
| "reward": -0.07754436880350113, | |
| "reward_std": 0.10734674707055092, | |
| "rewards/cosine_scaled_reward": -0.0387721830047667, | |
| "rewards/format_reward": 0.0, | |
| "step": 423 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.1012268066406, | |
| "epoch": 1.696, | |
| "grad_norm": 0.10524528473615646, | |
| "kl": 0.164306640625, | |
| "learning_rate": 1.6186884885673413e-07, | |
| "loss": -0.024, | |
| "reward": -0.08680723141878843, | |
| "reward_std": 0.0982758505269885, | |
| "rewards/cosine_scaled_reward": -0.04340361384674907, | |
| "rewards/format_reward": 0.0, | |
| "step": 424 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1505.3154907226562, | |
| "epoch": 1.7, | |
| "grad_norm": 0.10563742369413376, | |
| "kl": 0.16015625, | |
| "learning_rate": 1.6028856829700258e-07, | |
| "loss": -0.0092, | |
| "reward": -0.08459902927279472, | |
| "reward_std": 0.09910181537270546, | |
| "rewards/cosine_scaled_reward": -0.04229951370507479, | |
| "rewards/format_reward": 0.0, | |
| "step": 425 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1520.6785888671875, | |
| "epoch": 1.704, | |
| "grad_norm": 0.08786718547344208, | |
| "kl": 0.161865234375, | |
| "learning_rate": 1.5872728172265146e-07, | |
| "loss": -0.0165, | |
| "reward": -0.07912362925708294, | |
| "reward_std": 0.08175937831401825, | |
| "rewards/cosine_scaled_reward": -0.03956181462854147, | |
| "rewards/format_reward": 0.0, | |
| "step": 426 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.9762573242188, | |
| "epoch": 1.708, | |
| "grad_norm": 0.08499140292406082, | |
| "kl": 0.15625, | |
| "learning_rate": 1.5718506522858572e-07, | |
| "loss": -0.0364, | |
| "reward": -0.0896658506244421, | |
| "reward_std": 0.10279479995369911, | |
| "rewards/cosine_scaled_reward": -0.0448329309001565, | |
| "rewards/format_reward": 0.0, | |
| "step": 427 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.3035583496094, | |
| "epoch": 1.712, | |
| "grad_norm": 0.08925153315067291, | |
| "kl": 0.14453125, | |
| "learning_rate": 1.5566199398026147e-07, | |
| "loss": -0.0309, | |
| "reward": -0.09496857039630413, | |
| "reward_std": 0.1123510580509901, | |
| "rewards/cosine_scaled_reward": -0.04748428799211979, | |
| "rewards/format_reward": 0.0, | |
| "step": 428 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1523.2916870117188, | |
| "epoch": 1.716, | |
| "grad_norm": 0.10608566552400589, | |
| "kl": 0.167724609375, | |
| "learning_rate": 1.5415814221002265e-07, | |
| "loss": -0.0113, | |
| "reward": -0.09526684321463108, | |
| "reward_std": 0.0988641269505024, | |
| "rewards/cosine_scaled_reward": -0.047633420675992966, | |
| "rewards/format_reward": 0.0, | |
| "step": 429 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.2261962890625, | |
| "epoch": 1.72, | |
| "grad_norm": 0.10655763745307922, | |
| "kl": 0.16162109375, | |
| "learning_rate": 1.5267358321348285e-07, | |
| "loss": -0.0414, | |
| "reward": -0.10186839010566473, | |
| "reward_std": 0.13254049234092236, | |
| "rewards/cosine_scaled_reward": -0.05093420064076781, | |
| "rewards/format_reward": 0.0, | |
| "step": 430 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1528.0476379394531, | |
| "epoch": 1.724, | |
| "grad_norm": 0.1068165972828865, | |
| "kl": 0.15673828125, | |
| "learning_rate": 1.5120838934595337e-07, | |
| "loss": -0.0041, | |
| "reward": -0.09541826322674751, | |
| "reward_std": 0.1032972726970911, | |
| "rewards/cosine_scaled_reward": -0.04770912975072861, | |
| "rewards/format_reward": 0.0, | |
| "step": 431 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1522.4583435058594, | |
| "epoch": 1.728, | |
| "grad_norm": 0.12408644706010818, | |
| "kl": 0.16796875, | |
| "learning_rate": 1.4976263201891613e-07, | |
| "loss": -0.013, | |
| "reward": -0.08289302699267864, | |
| "reward_std": 0.08351449854671955, | |
| "rewards/cosine_scaled_reward": -0.041446512565016747, | |
| "rewards/format_reward": 0.0, | |
| "step": 432 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1497.9107360839844, | |
| "epoch": 1.732, | |
| "grad_norm": 0.07734204828739166, | |
| "kl": 0.173095703125, | |
| "learning_rate": 1.483363816965435e-07, | |
| "loss": -0.0391, | |
| "reward": -0.10260258801281452, | |
| "reward_std": 0.12828159891068935, | |
| "rewards/cosine_scaled_reward": -0.051301293075084686, | |
| "rewards/format_reward": 0.0, | |
| "step": 433 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.7559814453125, | |
| "epoch": 1.736, | |
| "grad_norm": 0.12413759529590607, | |
| "kl": 0.17919921875, | |
| "learning_rate": 1.469297078922642e-07, | |
| "loss": -0.0244, | |
| "reward": -0.015836404636502266, | |
| "reward_std": 0.09649943746626377, | |
| "rewards/cosine_scaled_reward": -0.007918204180896282, | |
| "rewards/format_reward": 0.0, | |
| "step": 434 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1522.1190795898438, | |
| "epoch": 1.74, | |
| "grad_norm": 0.12912577390670776, | |
| "kl": 0.16650390625, | |
| "learning_rate": 1.4554267916537495e-07, | |
| "loss": -0.0112, | |
| "reward": -0.08571217954158783, | |
| "reward_std": 0.10151237808167934, | |
| "rewards/cosine_scaled_reward": -0.042856089770793915, | |
| "rewards/format_reward": 0.0, | |
| "step": 435 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1474.952392578125, | |
| "epoch": 1.744, | |
| "grad_norm": 0.08318183571100235, | |
| "kl": 0.1689453125, | |
| "learning_rate": 1.4417536311769885e-07, | |
| "loss": -0.0637, | |
| "reward": -0.09841407462954521, | |
| "reward_std": 0.1172296404838562, | |
| "rewards/cosine_scaled_reward": -0.049207039177417755, | |
| "rewards/format_reward": 0.0, | |
| "step": 436 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1481.3095397949219, | |
| "epoch": 1.748, | |
| "grad_norm": 0.0786975845694542, | |
| "kl": 0.156005859375, | |
| "learning_rate": 1.4282782639029128e-07, | |
| "loss": -0.0386, | |
| "reward": -0.08532883040606976, | |
| "reward_std": 0.09728906117379665, | |
| "rewards/cosine_scaled_reward": -0.04266441613435745, | |
| "rewards/format_reward": 0.0, | |
| "step": 437 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1500.7440490722656, | |
| "epoch": 1.752, | |
| "grad_norm": 0.0900636538863182, | |
| "kl": 0.161376953125, | |
| "learning_rate": 1.4150013466019114e-07, | |
| "loss": -0.0316, | |
| "reward": -0.08545132167637348, | |
| "reward_std": 0.10151121858507395, | |
| "rewards/cosine_scaled_reward": -0.042725661769509315, | |
| "rewards/format_reward": 0.0, | |
| "step": 438 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.9404907226562, | |
| "epoch": 1.756, | |
| "grad_norm": 0.11020209640264511, | |
| "kl": 0.1640625, | |
| "learning_rate": 1.4019235263722034e-07, | |
| "loss": -0.0259, | |
| "reward": -0.08197178691625595, | |
| "reward_std": 0.09423052612692118, | |
| "rewards/cosine_scaled_reward": -0.040985893458127975, | |
| "rewards/format_reward": 0.0, | |
| "step": 439 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1517.482177734375, | |
| "epoch": 1.76, | |
| "grad_norm": 0.08999020606279373, | |
| "kl": 0.165283203125, | |
| "learning_rate": 1.3890454406082956e-07, | |
| "loss": -0.017, | |
| "reward": -0.07763329334557056, | |
| "reward_std": 0.08629796095192432, | |
| "rewards/cosine_scaled_reward": -0.03881664574146271, | |
| "rewards/format_reward": 0.0, | |
| "step": 440 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.8511962890625, | |
| "epoch": 1.764, | |
| "grad_norm": 0.13127504289150238, | |
| "kl": 0.15234375, | |
| "learning_rate": 1.3763677169699217e-07, | |
| "loss": -0.0232, | |
| "reward": -0.08330708928406239, | |
| "reward_std": 0.09235509857535362, | |
| "rewards/cosine_scaled_reward": -0.04165354464203119, | |
| "rewards/format_reward": 0.0, | |
| "step": 441 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.7619323730469, | |
| "epoch": 1.768, | |
| "grad_norm": 0.14613445103168488, | |
| "kl": 0.152099609375, | |
| "learning_rate": 1.3638909733514452e-07, | |
| "loss": -0.0284, | |
| "reward": -0.09447834640741348, | |
| "reward_std": 0.09266001731157303, | |
| "rewards/cosine_scaled_reward": -0.04723917320370674, | |
| "rewards/format_reward": 0.0, | |
| "step": 442 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1516.875, | |
| "epoch": 1.772, | |
| "grad_norm": 0.18538497388362885, | |
| "kl": 0.15966796875, | |
| "learning_rate": 1.351615817851748e-07, | |
| "loss": -0.0153, | |
| "reward": -0.08249685540795326, | |
| "reward_std": 0.09769860841333866, | |
| "rewards/cosine_scaled_reward": -0.04124843003228307, | |
| "rewards/format_reward": 0.0, | |
| "step": 443 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1501.8155212402344, | |
| "epoch": 1.776, | |
| "grad_norm": 0.1319953352212906, | |
| "kl": 0.155029296875, | |
| "learning_rate": 1.3395428487445914e-07, | |
| "loss": -0.039, | |
| "reward": -0.09754344820976257, | |
| "reward_std": 0.11035412549972534, | |
| "rewards/cosine_scaled_reward": -0.04877172317355871, | |
| "rewards/format_reward": 0.0, | |
| "step": 444 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1511.9821472167969, | |
| "epoch": 1.78, | |
| "grad_norm": 0.1029873788356781, | |
| "kl": 0.1484375, | |
| "learning_rate": 1.3276726544494571e-07, | |
| "loss": -0.0251, | |
| "reward": -0.0899391695857048, | |
| "reward_std": 0.10835397988557816, | |
| "rewards/cosine_scaled_reward": -0.04496958386152983, | |
| "rewards/format_reward": 0.0, | |
| "step": 445 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1469.40478515625, | |
| "epoch": 1.784, | |
| "grad_norm": 0.08266568928956985, | |
| "kl": 0.158935546875, | |
| "learning_rate": 1.316005813502869e-07, | |
| "loss": -0.0788, | |
| "reward": -0.10532401315867901, | |
| "reward_std": 0.12493490241467953, | |
| "rewards/cosine_scaled_reward": -0.05266200751066208, | |
| "rewards/format_reward": 0.0, | |
| "step": 446 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1505.5952453613281, | |
| "epoch": 1.788, | |
| "grad_norm": 0.13063663244247437, | |
| "kl": 0.15576171875, | |
| "learning_rate": 1.3045428945301953e-07, | |
| "loss": -0.0317, | |
| "reward": -0.022020583972334862, | |
| "reward_std": 0.1138888020068407, | |
| "rewards/cosine_scaled_reward": -0.011010290123522282, | |
| "rewards/format_reward": 0.0, | |
| "step": 447 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1493.7142944335938, | |
| "epoch": 1.792, | |
| "grad_norm": 0.1297776997089386, | |
| "kl": 0.17236328125, | |
| "learning_rate": 1.2932844562179352e-07, | |
| "loss": -0.0294, | |
| "reward": -0.08543841261416674, | |
| "reward_std": 0.09460597112774849, | |
| "rewards/cosine_scaled_reward": -0.042719203513115644, | |
| "rewards/format_reward": 0.0, | |
| "step": 448 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1521.1071472167969, | |
| "epoch": 1.796, | |
| "grad_norm": 0.1407863050699234, | |
| "kl": 0.16357421875, | |
| "learning_rate": 1.2822310472864885e-07, | |
| "loss": -0.0138, | |
| "reward": -0.10117548704147339, | |
| "reward_std": 0.12595792300999165, | |
| "rewards/cosine_scaled_reward": -0.05058774631470442, | |
| "rewards/format_reward": 0.0, | |
| "step": 449 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.1964721679688, | |
| "epoch": 1.8, | |
| "grad_norm": 0.16165214776992798, | |
| "kl": 0.1787109375, | |
| "learning_rate": 1.2713832064634125e-07, | |
| "loss": -0.0099, | |
| "reward": -0.08527638856321573, | |
| "reward_std": 0.09594122413545847, | |
| "rewards/cosine_scaled_reward": -0.04263819335028529, | |
| "rewards/format_reward": 0.0, | |
| "step": 450 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1511.7202758789062, | |
| "epoch": 1.804, | |
| "grad_norm": 0.1142469272017479, | |
| "kl": 0.163818359375, | |
| "learning_rate": 1.260741462457165e-07, | |
| "loss": -0.023, | |
| "reward": -0.09475222788751125, | |
| "reward_std": 0.10312853008508682, | |
| "rewards/cosine_scaled_reward": -0.04737611673772335, | |
| "rewards/format_reward": 0.0, | |
| "step": 451 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1527.5654907226562, | |
| "epoch": 1.808, | |
| "grad_norm": 0.1640588343143463, | |
| "kl": 0.171142578125, | |
| "learning_rate": 1.2503063339313356e-07, | |
| "loss": -0.0028, | |
| "reward": -0.09184761717915535, | |
| "reward_std": 0.09996213018894196, | |
| "rewards/cosine_scaled_reward": -0.0459238076582551, | |
| "rewards/format_reward": 0.0, | |
| "step": 452 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1493.8036193847656, | |
| "epoch": 1.812, | |
| "grad_norm": 0.07719198614358902, | |
| "kl": 0.177978515625, | |
| "learning_rate": 1.2400783294793668e-07, | |
| "loss": -0.0428, | |
| "reward": -0.10916751623153687, | |
| "reward_std": 0.12887151166796684, | |
| "rewards/cosine_scaled_reward": -0.054583752527832985, | |
| "rewards/format_reward": 0.0, | |
| "step": 453 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1504.3631286621094, | |
| "epoch": 1.8159999999999998, | |
| "grad_norm": 0.12536108493804932, | |
| "kl": 0.152587890625, | |
| "learning_rate": 1.2300579475997657e-07, | |
| "loss": -0.038, | |
| "reward": -0.1067353542894125, | |
| "reward_std": 0.12835400737822056, | |
| "rewards/cosine_scaled_reward": -0.05336767714470625, | |
| "rewards/format_reward": 0.0, | |
| "step": 454 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.77978515625, | |
| "epoch": 1.8199999999999998, | |
| "grad_norm": 0.1128176897764206, | |
| "kl": 0.16259765625, | |
| "learning_rate": 1.220245676671809e-07, | |
| "loss": -0.0177, | |
| "reward": -0.10911162942647934, | |
| "reward_std": 0.13343517668545246, | |
| "rewards/cosine_scaled_reward": -0.054555815644562244, | |
| "rewards/format_reward": 0.0, | |
| "step": 455 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1536.0, | |
| "epoch": 1.8239999999999998, | |
| "grad_norm": 0.13748064637184143, | |
| "kl": 0.172119140625, | |
| "learning_rate": 1.2106419949317388e-07, | |
| "loss": 0.0069, | |
| "reward": -0.07034523971378803, | |
| "reward_std": 0.07846208661794662, | |
| "rewards/cosine_scaled_reward": -0.03517262078821659, | |
| "rewards/format_reward": 0.0, | |
| "step": 456 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1492.8690490722656, | |
| "epoch": 1.8279999999999998, | |
| "grad_norm": 0.1389494091272354, | |
| "kl": 0.152099609375, | |
| "learning_rate": 1.2012473704494537e-07, | |
| "loss": -0.0455, | |
| "reward": -0.11908807791769505, | |
| "reward_std": 0.13069945573806763, | |
| "rewards/cosine_scaled_reward": -0.059544037096202374, | |
| "rewards/format_reward": 0.0, | |
| "step": 457 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1499.9464721679688, | |
| "epoch": 1.8319999999999999, | |
| "grad_norm": 0.1537049114704132, | |
| "kl": 0.1683349609375, | |
| "learning_rate": 1.1920622611056974e-07, | |
| "loss": -0.0307, | |
| "reward": -0.08774650190025568, | |
| "reward_std": 0.10440967045724392, | |
| "rewards/cosine_scaled_reward": -0.043873251881450415, | |
| "rewards/format_reward": 0.0, | |
| "step": 458 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1524.2083435058594, | |
| "epoch": 1.8359999999999999, | |
| "grad_norm": 0.13081440329551697, | |
| "kl": 0.176513671875, | |
| "learning_rate": 1.1830871145697412e-07, | |
| "loss": -0.0066, | |
| "reward": -0.08506089821457863, | |
| "reward_std": 0.09712946228682995, | |
| "rewards/cosine_scaled_reward": -0.04253045003861189, | |
| "rewards/format_reward": 0.0, | |
| "step": 459 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1502.9583435058594, | |
| "epoch": 1.8399999999999999, | |
| "grad_norm": 0.09031596034765244, | |
| "kl": 0.154296875, | |
| "learning_rate": 1.1743223682775649e-07, | |
| "loss": -0.0358, | |
| "reward": -0.09923446178436279, | |
| "reward_std": 0.11484255269169807, | |
| "rewards/cosine_scaled_reward": -0.049617230892181396, | |
| "rewards/format_reward": 0.0, | |
| "step": 460 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.5059509277344, | |
| "epoch": 1.8439999999999999, | |
| "grad_norm": 0.15131881833076477, | |
| "kl": 0.161376953125, | |
| "learning_rate": 1.1657684494105386e-07, | |
| "loss": -0.0215, | |
| "reward": -0.09375773929059505, | |
| "reward_std": 0.1126671563833952, | |
| "rewards/cosine_scaled_reward": -0.04687886871397495, | |
| "rewards/format_reward": 0.0, | |
| "step": 461 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1515.2678833007812, | |
| "epoch": 1.8479999999999999, | |
| "grad_norm": 0.08426119387149811, | |
| "kl": 0.15576171875, | |
| "learning_rate": 1.1574257748745986e-07, | |
| "loss": -0.0228, | |
| "reward": -0.09476478770375252, | |
| "reward_std": 0.10140549577772617, | |
| "rewards/cosine_scaled_reward": -0.04738239198923111, | |
| "rewards/format_reward": 0.0, | |
| "step": 462 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1512.8333435058594, | |
| "epoch": 1.8519999999999999, | |
| "grad_norm": 0.08592584729194641, | |
| "kl": 0.17333984375, | |
| "learning_rate": 1.1492947512799328e-07, | |
| "loss": -0.0256, | |
| "reward": -0.011012900620698929, | |
| "reward_std": 0.08133355341851711, | |
| "rewards/cosine_scaled_reward": -0.005506448447704315, | |
| "rewards/format_reward": 0.0, | |
| "step": 463 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1515.3511962890625, | |
| "epoch": 1.8559999999999999, | |
| "grad_norm": 0.1047179102897644, | |
| "kl": 0.17529296875, | |
| "learning_rate": 1.1413757749211602e-07, | |
| "loss": -0.0208, | |
| "reward": -0.08088574931025505, | |
| "reward_std": 0.09925234131515026, | |
| "rewards/cosine_scaled_reward": -0.040442874655127525, | |
| "rewards/format_reward": 0.0, | |
| "step": 464 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.4642944335938, | |
| "epoch": 1.8599999999999999, | |
| "grad_norm": 0.1791323721408844, | |
| "kl": 0.177734375, | |
| "learning_rate": 1.1336692317580158e-07, | |
| "loss": -0.0299, | |
| "reward": -0.09140351600944996, | |
| "reward_std": 0.1100204586982727, | |
| "rewards/cosine_scaled_reward": -0.045701757073402405, | |
| "rewards/format_reward": 0.0, | |
| "step": 465 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1522.0535583496094, | |
| "epoch": 1.8639999999999999, | |
| "grad_norm": 0.13129960000514984, | |
| "kl": 0.1676025390625, | |
| "learning_rate": 1.1261754973965422e-07, | |
| "loss": -0.0129, | |
| "reward": -0.08949675410985947, | |
| "reward_std": 0.09439942799508572, | |
| "rewards/cosine_scaled_reward": -0.04474837705492973, | |
| "rewards/format_reward": 0.0, | |
| "step": 466 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1484.0535888671875, | |
| "epoch": 1.8679999999999999, | |
| "grad_norm": 0.13218103349208832, | |
| "kl": 0.15966796875, | |
| "learning_rate": 1.1188949370707787e-07, | |
| "loss": -0.0593, | |
| "reward": -0.11925767548382282, | |
| "reward_std": 0.15814346075057983, | |
| "rewards/cosine_scaled_reward": -0.05962884332984686, | |
| "rewards/format_reward": 0.0, | |
| "step": 467 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1496.4702453613281, | |
| "epoch": 1.8719999999999999, | |
| "grad_norm": 0.09863686561584473, | |
| "kl": 0.146484375, | |
| "learning_rate": 1.1118279056249653e-07, | |
| "loss": -0.0405, | |
| "reward": -0.0985277071595192, | |
| "reward_std": 0.11279423907399178, | |
| "rewards/cosine_scaled_reward": -0.04926385171711445, | |
| "rewards/format_reward": 0.0, | |
| "step": 468 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1515.8869323730469, | |
| "epoch": 1.876, | |
| "grad_norm": 0.09514996409416199, | |
| "kl": 0.158203125, | |
| "learning_rate": 1.1049747474962444e-07, | |
| "loss": -0.0164, | |
| "reward": -0.08131754398345947, | |
| "reward_std": 0.0914676021784544, | |
| "rewards/cosine_scaled_reward": -0.04065877292305231, | |
| "rewards/format_reward": 0.0, | |
| "step": 469 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1525.4464416503906, | |
| "epoch": 1.88, | |
| "grad_norm": 0.11126792430877686, | |
| "kl": 0.158447265625, | |
| "learning_rate": 1.0983357966978745e-07, | |
| "loss": -0.0064, | |
| "reward": -0.0882963128387928, | |
| "reward_std": 0.09903069026768208, | |
| "rewards/cosine_scaled_reward": -0.04414815828204155, | |
| "rewards/format_reward": 0.0, | |
| "step": 470 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1502.3392944335938, | |
| "epoch": 1.884, | |
| "grad_norm": 0.12479417026042938, | |
| "kl": 0.168701171875, | |
| "learning_rate": 1.0919113768029517e-07, | |
| "loss": -0.0409, | |
| "reward": -0.07126700505614281, | |
| "reward_std": 0.08810876682400703, | |
| "rewards/cosine_scaled_reward": -0.035633502528071404, | |
| "rewards/format_reward": 0.0, | |
| "step": 471 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1523.3511962890625, | |
| "epoch": 1.888, | |
| "grad_norm": 0.15230634808540344, | |
| "kl": 0.1669921875, | |
| "learning_rate": 1.0857018009286381e-07, | |
| "loss": -0.0107, | |
| "reward": -0.07650433294475079, | |
| "reward_std": 0.09276540018618107, | |
| "rewards/cosine_scaled_reward": -0.03825216554105282, | |
| "rewards/format_reward": 0.0, | |
| "step": 472 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1529.2916564941406, | |
| "epoch": 1.892, | |
| "grad_norm": 0.10235889256000519, | |
| "kl": 0.163818359375, | |
| "learning_rate": 1.0797073717209013e-07, | |
| "loss": -0.0031, | |
| "reward": -0.08031682576984167, | |
| "reward_std": 0.08574636466801167, | |
| "rewards/cosine_scaled_reward": -0.0401584105566144, | |
| "rewards/format_reward": 0.0, | |
| "step": 473 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1514.4643249511719, | |
| "epoch": 1.896, | |
| "grad_norm": 0.24448621273040771, | |
| "kl": 0.17724609375, | |
| "learning_rate": 1.0739283813397639e-07, | |
| "loss": -0.0168, | |
| "reward": -0.09807473048567772, | |
| "reward_std": 0.11232626810669899, | |
| "rewards/cosine_scaled_reward": -0.049037366174161434, | |
| "rewards/format_reward": 0.0, | |
| "step": 474 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.8155212402344, | |
| "epoch": 1.9, | |
| "grad_norm": 0.11423542350530624, | |
| "kl": 0.15625, | |
| "learning_rate": 1.068365111445064e-07, | |
| "loss": -0.0138, | |
| "reward": -0.09042352437973022, | |
| "reward_std": 0.10206466354429722, | |
| "rewards/cosine_scaled_reward": -0.04521176405251026, | |
| "rewards/format_reward": 0.0, | |
| "step": 475 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.4404907226562, | |
| "epoch": 1.904, | |
| "grad_norm": 0.1770845502614975, | |
| "kl": 0.150390625, | |
| "learning_rate": 1.063017833182728e-07, | |
| "loss": -0.0315, | |
| "reward": -0.1031611617654562, | |
| "reward_std": 0.11722332611680031, | |
| "rewards/cosine_scaled_reward": -0.051580581814050674, | |
| "rewards/format_reward": 0.0, | |
| "step": 476 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1512.5178527832031, | |
| "epoch": 1.908, | |
| "grad_norm": 0.1035989373922348, | |
| "kl": 0.155517578125, | |
| "learning_rate": 1.0578868071715544e-07, | |
| "loss": -0.0263, | |
| "reward": -0.10269530303776264, | |
| "reward_std": 0.13116441946476698, | |
| "rewards/cosine_scaled_reward": -0.05134765151888132, | |
| "rewards/format_reward": 0.0, | |
| "step": 477 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1506.7083435058594, | |
| "epoch": 1.912, | |
| "grad_norm": 0.1195254847407341, | |
| "kl": 0.162841796875, | |
| "learning_rate": 1.0529722834905125e-07, | |
| "loss": -0.0243, | |
| "reward": -0.10078963078558445, | |
| "reward_std": 0.11876899935305119, | |
| "rewards/cosine_scaled_reward": -0.050394815392792225, | |
| "rewards/format_reward": 0.0, | |
| "step": 478 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1515.0952453613281, | |
| "epoch": 1.916, | |
| "grad_norm": 0.1497546136379242, | |
| "kl": 0.1513671875, | |
| "learning_rate": 1.0482745016665526e-07, | |
| "loss": -0.0204, | |
| "reward": -0.10379143245518208, | |
| "reward_std": 0.11940331198275089, | |
| "rewards/cosine_scaled_reward": -0.05189571529626846, | |
| "rewards/format_reward": 0.0, | |
| "step": 479 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1522.8690490722656, | |
| "epoch": 1.92, | |
| "grad_norm": 0.12198466807603836, | |
| "kl": 0.169189453125, | |
| "learning_rate": 1.0437936906629334e-07, | |
| "loss": -0.0117, | |
| "reward": -0.0045996010303497314, | |
| "reward_std": 0.09211089462041855, | |
| "rewards/cosine_scaled_reward": -0.002299800980836153, | |
| "rewards/format_reward": 0.0, | |
| "step": 480 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1498.5833740234375, | |
| "epoch": 1.924, | |
| "grad_norm": 0.10352538526058197, | |
| "kl": 0.14697265625, | |
| "learning_rate": 1.0395300688680625e-07, | |
| "loss": -0.0433, | |
| "reward": -0.1318805105984211, | |
| "reward_std": 0.15495008416473866, | |
| "rewards/cosine_scaled_reward": -0.06594025250524282, | |
| "rewards/format_reward": 0.0, | |
| "step": 481 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1517.107177734375, | |
| "epoch": 1.928, | |
| "grad_norm": 0.08769793808460236, | |
| "kl": 0.14990234375, | |
| "learning_rate": 1.0354838440848501e-07, | |
| "loss": -0.0207, | |
| "reward": -0.10253190249204636, | |
| "reward_std": 0.121914217248559, | |
| "rewards/cosine_scaled_reward": -0.05126595124602318, | |
| "rewards/format_reward": 0.0, | |
| "step": 482 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1523.0059814453125, | |
| "epoch": 1.932, | |
| "grad_norm": 0.09510423243045807, | |
| "kl": 0.16064453125, | |
| "learning_rate": 1.0316552135205837e-07, | |
| "loss": -0.0073, | |
| "reward": -0.09257967211306095, | |
| "reward_std": 0.09288883674889803, | |
| "rewards/cosine_scaled_reward": -0.0462898388504982, | |
| "rewards/format_reward": 0.0, | |
| "step": 483 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1522.1666870117188, | |
| "epoch": 1.936, | |
| "grad_norm": 0.11582231521606445, | |
| "kl": 0.159912109375, | |
| "learning_rate": 1.0280443637773163e-07, | |
| "loss": -0.013, | |
| "reward": -0.09562139585614204, | |
| "reward_std": 0.10689939372241497, | |
| "rewards/cosine_scaled_reward": -0.04781070165336132, | |
| "rewards/format_reward": 0.0, | |
| "step": 484 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1527.6428527832031, | |
| "epoch": 1.94, | |
| "grad_norm": 0.1510264277458191, | |
| "kl": 0.168701171875, | |
| "learning_rate": 1.0246514708427701e-07, | |
| "loss": 0.0057, | |
| "reward": -0.07166448421776295, | |
| "reward_std": 0.06708121951669455, | |
| "rewards/cosine_scaled_reward": -0.03583224397152662, | |
| "rewards/format_reward": 0.0, | |
| "step": 485 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.6607360839844, | |
| "epoch": 1.944, | |
| "grad_norm": 0.10827223211526871, | |
| "kl": 0.153076171875, | |
| "learning_rate": 1.0214767000817596e-07, | |
| "loss": -0.0092, | |
| "reward": -0.077840281650424, | |
| "reward_std": 0.09841375425457954, | |
| "rewards/cosine_scaled_reward": -0.038920141756534576, | |
| "rewards/format_reward": 0.0, | |
| "step": 486 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1515.2857360839844, | |
| "epoch": 1.948, | |
| "grad_norm": 0.08925757557153702, | |
| "kl": 0.1376953125, | |
| "learning_rate": 1.0185202062281336e-07, | |
| "loss": -0.0088, | |
| "reward": -0.0860859602689743, | |
| "reward_std": 0.10117382928729057, | |
| "rewards/cosine_scaled_reward": -0.04304297920316458, | |
| "rewards/format_reward": 0.0, | |
| "step": 487 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.1130981445312, | |
| "epoch": 1.952, | |
| "grad_norm": 0.1778467744588852, | |
| "kl": 0.160400390625, | |
| "learning_rate": 1.0157821333772304e-07, | |
| "loss": -0.0269, | |
| "reward": -0.08845487236976624, | |
| "reward_std": 0.09980816766619682, | |
| "rewards/cosine_scaled_reward": -0.04422743525356054, | |
| "rewards/format_reward": 0.0, | |
| "step": 488 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1523.482177734375, | |
| "epoch": 1.956, | |
| "grad_norm": 0.09875297546386719, | |
| "kl": 0.162841796875, | |
| "learning_rate": 1.013262614978859e-07, | |
| "loss": -0.0109, | |
| "reward": -0.08624122757464647, | |
| "reward_std": 0.09759997017681599, | |
| "rewards/cosine_scaled_reward": -0.04312061285600066, | |
| "rewards/format_reward": 0.0, | |
| "step": 489 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1488.8929138183594, | |
| "epoch": 1.96, | |
| "grad_norm": 0.10072106122970581, | |
| "kl": 0.16064453125, | |
| "learning_rate": 1.0109617738307911e-07, | |
| "loss": -0.0561, | |
| "reward": -0.1024992810562253, | |
| "reward_std": 0.1395698133856058, | |
| "rewards/cosine_scaled_reward": -0.051249639596790075, | |
| "rewards/format_reward": 0.0, | |
| "step": 490 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1507.6190795898438, | |
| "epoch": 1.964, | |
| "grad_norm": 0.14594227075576782, | |
| "kl": 0.170166015625, | |
| "learning_rate": 1.0088797220727779e-07, | |
| "loss": -0.0329, | |
| "reward": -0.09847836010158062, | |
| "reward_std": 0.12085962668061256, | |
| "rewards/cosine_scaled_reward": -0.049239179119467735, | |
| "rewards/format_reward": 0.0, | |
| "step": 491 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1515.52978515625, | |
| "epoch": 1.968, | |
| "grad_norm": 0.1781827211380005, | |
| "kl": 0.171875, | |
| "learning_rate": 1.0070165611810855e-07, | |
| "loss": -0.0183, | |
| "reward": -0.09458879381418228, | |
| "reward_std": 0.10490395873785019, | |
| "rewards/cosine_scaled_reward": -0.047294397838413715, | |
| "rewards/format_reward": 0.0, | |
| "step": 492 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1510.4464416503906, | |
| "epoch": 1.972, | |
| "grad_norm": 0.14723263680934906, | |
| "kl": 0.166259765625, | |
| "learning_rate": 1.005372381963547e-07, | |
| "loss": -0.0245, | |
| "reward": -0.08563583716750145, | |
| "reward_std": 0.09580126218497753, | |
| "rewards/cosine_scaled_reward": -0.04281791765242815, | |
| "rewards/format_reward": 0.0, | |
| "step": 493 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.9822082519531, | |
| "epoch": 1.976, | |
| "grad_norm": 0.08827134966850281, | |
| "kl": 0.171142578125, | |
| "learning_rate": 1.0039472645551372e-07, | |
| "loss": -0.0314, | |
| "reward": -0.09158815257251263, | |
| "reward_std": 0.11639940552413464, | |
| "rewards/cosine_scaled_reward": -0.04579407814890146, | |
| "rewards/format_reward": 0.0, | |
| "step": 494 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1508.8095397949219, | |
| "epoch": 1.98, | |
| "grad_norm": 0.09312310069799423, | |
| "kl": 0.15380859375, | |
| "learning_rate": 1.002741278414069e-07, | |
| "loss": -0.0125, | |
| "reward": -0.08922230452299118, | |
| "reward_std": 0.09334707166999578, | |
| "rewards/cosine_scaled_reward": -0.044611155055463314, | |
| "rewards/format_reward": 0.0, | |
| "step": 495 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1511.452392578125, | |
| "epoch": 1.984, | |
| "grad_norm": 0.10467734187841415, | |
| "kl": 0.14697265625, | |
| "learning_rate": 1.0017544823184055e-07, | |
| "loss": -0.0302, | |
| "reward": -0.09572159126400948, | |
| "reward_std": 0.11608831025660038, | |
| "rewards/cosine_scaled_reward": -0.04786079656332731, | |
| "rewards/format_reward": 0.0, | |
| "step": 496 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1488.7262573242188, | |
| "epoch": 1.988, | |
| "grad_norm": 0.11462484300136566, | |
| "kl": 0.152099609375, | |
| "learning_rate": 1.0009869243631952e-07, | |
| "loss": -0.0413, | |
| "reward": -0.10070546343922615, | |
| "reward_std": 0.12976408563554287, | |
| "rewards/cosine_scaled_reward": -0.05035272892564535, | |
| "rewards/format_reward": 0.0, | |
| "step": 497 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.2143249511719, | |
| "epoch": 1.992, | |
| "grad_norm": 0.09213641285896301, | |
| "kl": 0.135498046875, | |
| "learning_rate": 1.000438641958131e-07, | |
| "loss": -0.0224, | |
| "reward": -0.09420822747051716, | |
| "reward_std": 0.1108301505446434, | |
| "rewards/cosine_scaled_reward": -0.04710411373525858, | |
| "rewards/format_reward": 0.0, | |
| "step": 498 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1523.3511962890625, | |
| "epoch": 1.996, | |
| "grad_norm": 0.10646738111972809, | |
| "kl": 0.1552734375, | |
| "learning_rate": 1.0001096618257236e-07, | |
| "loss": -0.0112, | |
| "reward": -0.10537549015134573, | |
| "reward_std": 0.11287760734558105, | |
| "rewards/cosine_scaled_reward": -0.052687746938318014, | |
| "rewards/format_reward": 0.0, | |
| "step": 499 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1513.0893859863281, | |
| "epoch": 2.0, | |
| "grad_norm": 0.12409133464097977, | |
| "kl": 0.166015625, | |
| "learning_rate": 1e-07, | |
| "loss": -0.0134, | |
| "reward": -0.08198001235723495, | |
| "reward_std": 0.0940225888043642, | |
| "rewards/cosine_scaled_reward": -0.04099000431597233, | |
| "rewards/format_reward": 0.0, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "step": 500, | |
| "total_flos": 0.0, | |
| "train_loss": -0.028187389324251855, | |
| "train_runtime": 65552.2916, | |
| "train_samples_per_second": 1.281, | |
| "train_steps_per_second": 0.008 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 500, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 2, | |
| "save_steps": 50, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 6, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |