diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,8433 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5154639175257731, + "eval_steps": 500, + "global_step": 600, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio": 0.0, + "completion_length": 143.8359375, + "epoch": 0.000859106529209622, + "grad_norm": 3.728695289308407, + "kl": 0.0, + "learning_rate": 9.995704467353951e-07, + "loss": -0.0, + "reward": 1.225318193435669, + "reward_std": 0.5892010927200317, + "rewards/format_reward_gen": 0.609375, + "rewards/llm_reward": 0.6159431040287018, + "step": 1 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.78125, + "epoch": 0.001718213058419244, + "grad_norm": 2.510726404973143, + "kl": 0.0007686614990234375, + "learning_rate": 9.991408934707903e-07, + "loss": 0.0, + "reward": 1.148492693901062, + "reward_std": 0.5800909698009491, + "rewards/format_reward_gen": 0.5390625, + "rewards/llm_reward": 0.6094301640987396, + "step": 2 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.578125, + "epoch": 0.002577319587628866, + "grad_norm": 5.237708678465742, + "kl": 0.001880645751953125, + "learning_rate": 9.987113402061855e-07, + "loss": 0.0001, + "reward": 1.2333476543426514, + "reward_std": 0.5131954550743103, + "rewards/format_reward_gen": 0.7109375, + "rewards/llm_reward": 0.5224101841449738, + "step": 3 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.265625, + "epoch": 0.003436426116838488, + "grad_norm": 4.430606101451457, + "kl": 0.0068817138671875, + "learning_rate": 9.982817869415807e-07, + "loss": 0.0003, + "reward": 1.3201585412025452, + "reward_std": 0.5908865928649902, + "rewards/format_reward_gen": 0.6640625, + "rewards/llm_reward": 0.6560960710048676, + "step": 4 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.34375, + "epoch": 0.00429553264604811, + "grad_norm": 2.966410650688808, + "kl": 0.00145721435546875, + "learning_rate": 9.97852233676976e-07, + "loss": 0.0001, + "reward": 1.5005085468292236, + "reward_std": 0.446560800075531, + "rewards/format_reward_gen": 0.8046875, + "rewards/llm_reward": 0.6958210468292236, + "step": 5 + }, + { + "clip_ratio": 0.0, + "completion_length": 149.0234375, + "epoch": 0.005154639175257732, + "grad_norm": 18.73054467782952, + "kl": 0.00228118896484375, + "learning_rate": 9.97422680412371e-07, + "loss": 0.0001, + "reward": 1.379498541355133, + "reward_std": 0.3884614408016205, + "rewards/format_reward_gen": 0.796875, + "rewards/llm_reward": 0.5826235115528107, + "step": 6 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.1796875, + "epoch": 0.006013745704467354, + "grad_norm": 2.1996526299389556, + "kl": 0.00290679931640625, + "learning_rate": 9.969931271477663e-07, + "loss": 0.0001, + "reward": 1.4183476567268372, + "reward_std": 0.46953827142715454, + "rewards/format_reward_gen": 0.828125, + "rewards/llm_reward": 0.5902226716279984, + "step": 7 + }, + { + "clip_ratio": 0.0, + "completion_length": 159.2265625, + "epoch": 0.006872852233676976, + "grad_norm": 1.5763458045998808, + "kl": 0.00386810302734375, + "learning_rate": 9.965635738831615e-07, + "loss": 0.0002, + "reward": 1.4628726840019226, + "reward_std": 0.3434259593486786, + "rewards/format_reward_gen": 0.859375, + "rewards/llm_reward": 0.6034976243972778, + "step": 8 + }, + { + "clip_ratio": 0.0, + "completion_length": 148.8203125, + "epoch": 0.007731958762886598, + "grad_norm": 1.8925562854231095, + "kl": 0.003387451171875, + "learning_rate": 9.961340206185566e-07, + "loss": 0.0001, + "reward": 1.466668725013733, + "reward_std": 0.3619833141565323, + "rewards/format_reward_gen": 0.828125, + "rewards/llm_reward": 0.6385437846183777, + "step": 9 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.6640625, + "epoch": 0.00859106529209622, + "grad_norm": 2.30348378416368, + "kl": 0.008270263671875, + "learning_rate": 9.957044673539518e-07, + "loss": 0.0003, + "reward": 1.4460369348526, + "reward_std": 0.2754833847284317, + "rewards/format_reward_gen": 0.8828125, + "rewards/llm_reward": 0.5632244646549225, + "step": 10 + }, + { + "clip_ratio": 0.0, + "completion_length": 152.671875, + "epoch": 0.009450171821305841, + "grad_norm": 4.5902996335202495, + "kl": 0.006683349609375, + "learning_rate": 9.95274914089347e-07, + "loss": 0.0003, + "reward": 1.6509097814559937, + "reward_std": 0.25286635756492615, + "rewards/format_reward_gen": 0.953125, + "rewards/llm_reward": 0.6977846920490265, + "step": 11 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.9140625, + "epoch": 0.010309278350515464, + "grad_norm": 4.341349059182411, + "kl": 0.0069732666015625, + "learning_rate": 9.948453608247422e-07, + "loss": 0.0003, + "reward": 1.63209068775177, + "reward_std": 0.20565086603164673, + "rewards/format_reward_gen": 0.9609375, + "rewards/llm_reward": 0.6711532175540924, + "step": 12 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.453125, + "epoch": 0.011168384879725086, + "grad_norm": 5.165117571002937, + "kl": 0.0079345703125, + "learning_rate": 9.944158075601374e-07, + "loss": 0.0003, + "reward": 1.7142389416694641, + "reward_std": 0.219488687813282, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7376764714717865, + "step": 13 + }, + { + "clip_ratio": 0.0, + "completion_length": 121.25, + "epoch": 0.012027491408934709, + "grad_norm": 1.529830865974652, + "kl": 0.017974853515625, + "learning_rate": 9.939862542955326e-07, + "loss": 0.0007, + "reward": 1.6100916266441345, + "reward_std": 0.15942473709583282, + "rewards/format_reward_gen": 0.9609375, + "rewards/llm_reward": 0.6491541266441345, + "step": 14 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.9765625, + "epoch": 0.01288659793814433, + "grad_norm": 1.959441191085793, + "kl": 0.01171875, + "learning_rate": 9.935567010309278e-07, + "loss": 0.0005, + "reward": 1.6476787328720093, + "reward_std": 0.21745194494724274, + "rewards/format_reward_gen": 0.9609375, + "rewards/llm_reward": 0.6867412328720093, + "step": 15 + }, + { + "clip_ratio": 0.0, + "completion_length": 117.8515625, + "epoch": 0.013745704467353952, + "grad_norm": 1.8012904974133117, + "kl": 0.01544189453125, + "learning_rate": 9.93127147766323e-07, + "loss": 0.0006, + "reward": 1.648992121219635, + "reward_std": 0.22647833824157715, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.6802421808242798, + "step": 16 + }, + { + "clip_ratio": 0.0, + "completion_length": 110.6796875, + "epoch": 0.014604810996563574, + "grad_norm": 2.0371515238907456, + "kl": 0.02178955078125, + "learning_rate": 9.926975945017182e-07, + "loss": 0.0009, + "reward": 1.7388362884521484, + "reward_std": 0.23847993463277817, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7544613182544708, + "step": 17 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.171875, + "epoch": 0.015463917525773196, + "grad_norm": 2.29872644703533, + "kl": 0.02154541015625, + "learning_rate": 9.922680412371133e-07, + "loss": 0.0009, + "reward": 1.698328673839569, + "reward_std": 0.21018245071172714, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7139536440372467, + "step": 18 + }, + { + "clip_ratio": 0.0, + "completion_length": 115.8203125, + "epoch": 0.01632302405498282, + "grad_norm": 1.642185067375333, + "kl": 0.026123046875, + "learning_rate": 9.918384879725085e-07, + "loss": 0.001, + "reward": 1.5711023807525635, + "reward_std": 0.24714628607034683, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.5945399105548859, + "step": 19 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.0859375, + "epoch": 0.01718213058419244, + "grad_norm": 1.8301018336586894, + "kl": 0.04449462890625, + "learning_rate": 9.914089347079037e-07, + "loss": 0.0018, + "reward": 1.7036264538764954, + "reward_std": 0.2401985377073288, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7348764538764954, + "step": 20 + }, + { + "clip_ratio": 0.0, + "completion_length": 105.5859375, + "epoch": 0.01804123711340206, + "grad_norm": 2.4502501155105696, + "kl": 0.02874755859375, + "learning_rate": 9.90979381443299e-07, + "loss": 0.0011, + "reward": 1.5346225500106812, + "reward_std": 0.18206386268138885, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.5502475053071976, + "step": 21 + }, + { + "clip_ratio": 0.0, + "completion_length": 117.828125, + "epoch": 0.018900343642611683, + "grad_norm": 4.579990678131132, + "kl": 0.020751953125, + "learning_rate": 9.90549828178694e-07, + "loss": 0.0008, + "reward": 1.5973477363586426, + "reward_std": 0.21275992691516876, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.605160266160965, + "step": 22 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.1171875, + "epoch": 0.019759450171821305, + "grad_norm": 1.9133262074405595, + "kl": 0.025390625, + "learning_rate": 9.901202749140893e-07, + "loss": 0.001, + "reward": 1.6978419423103333, + "reward_std": 0.1712438389658928, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7056544423103333, + "step": 23 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.3515625, + "epoch": 0.020618556701030927, + "grad_norm": 1.6795653310334677, + "kl": 0.0228271484375, + "learning_rate": 9.896907216494845e-07, + "loss": 0.0009, + "reward": 1.5728907585144043, + "reward_std": 0.23567190766334534, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.5885157585144043, + "step": 24 + }, + { + "clip_ratio": 0.0, + "completion_length": 114.4609375, + "epoch": 0.02147766323024055, + "grad_norm": 1.8945583882920514, + "kl": 0.02520751953125, + "learning_rate": 9.892611683848797e-07, + "loss": 0.001, + "reward": 1.6186752319335938, + "reward_std": 0.2236367017030716, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.6343001425266266, + "step": 25 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.0, + "epoch": 0.022336769759450172, + "grad_norm": 1.6471025834878852, + "kl": 0.02325439453125, + "learning_rate": 9.888316151202748e-07, + "loss": 0.0009, + "reward": 1.7836337089538574, + "reward_std": 0.18849463760852814, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.791446179151535, + "step": 26 + }, + { + "clip_ratio": 0.0, + "completion_length": 114.0546875, + "epoch": 0.023195876288659795, + "grad_norm": 1.8224790908918138, + "kl": 0.01788330078125, + "learning_rate": 9.8840206185567e-07, + "loss": 0.0007, + "reward": 1.6910547018051147, + "reward_std": 0.177225723862648, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6988671720027924, + "step": 27 + }, + { + "clip_ratio": 0.0, + "completion_length": 110.984375, + "epoch": 0.024054982817869417, + "grad_norm": 1.7847962406057651, + "kl": 0.0181884765625, + "learning_rate": 9.879725085910652e-07, + "loss": 0.0007, + "reward": 1.7889049053192139, + "reward_std": 0.17536725103855133, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7967174351215363, + "step": 28 + }, + { + "clip_ratio": 0.0, + "completion_length": 113.0234375, + "epoch": 0.02491408934707904, + "grad_norm": 3.1801733940089547, + "kl": 0.02703857421875, + "learning_rate": 9.875429553264604e-07, + "loss": 0.0011, + "reward": 1.7557846307754517, + "reward_std": 0.17591644823551178, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7714096307754517, + "step": 29 + }, + { + "clip_ratio": 0.0, + "completion_length": 109.265625, + "epoch": 0.02577319587628866, + "grad_norm": 3.160227437825882, + "kl": 0.0194091796875, + "learning_rate": 9.871134020618556e-07, + "loss": 0.0008, + "reward": 1.6949102878570557, + "reward_std": 0.1834886148571968, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.710535317659378, + "step": 30 + }, + { + "clip_ratio": 0.0, + "completion_length": 112.265625, + "epoch": 0.02663230240549828, + "grad_norm": 1.4868195880486625, + "kl": 0.0224609375, + "learning_rate": 9.866838487972508e-07, + "loss": 0.0009, + "reward": 1.6780639290809631, + "reward_std": 0.16649843007326126, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7015013694763184, + "step": 31 + }, + { + "clip_ratio": 0.0, + "completion_length": 105.984375, + "epoch": 0.027491408934707903, + "grad_norm": 22.472420255137777, + "kl": 0.017974853515625, + "learning_rate": 9.86254295532646e-07, + "loss": 0.0007, + "reward": 1.64022696018219, + "reward_std": 0.19799882918596268, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.6558520197868347, + "step": 32 + }, + { + "clip_ratio": 0.0, + "completion_length": 110.46875, + "epoch": 0.028350515463917526, + "grad_norm": 1.8825542488707594, + "kl": 0.027587890625, + "learning_rate": 9.858247422680412e-07, + "loss": 0.0011, + "reward": 1.686434805393219, + "reward_std": 0.1929420754313469, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.694247305393219, + "step": 33 + }, + { + "clip_ratio": 0.0, + "completion_length": 111.7109375, + "epoch": 0.029209621993127148, + "grad_norm": 1.318783449188221, + "kl": 0.011474609375, + "learning_rate": 9.853951890034363e-07, + "loss": 0.0005, + "reward": 1.7323167324066162, + "reward_std": 0.14837764203548431, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7323167026042938, + "step": 34 + }, + { + "clip_ratio": 0.0, + "completion_length": 117.7421875, + "epoch": 0.03006872852233677, + "grad_norm": 1.6922286395862565, + "kl": 0.016632080078125, + "learning_rate": 9.849656357388315e-07, + "loss": 0.0007, + "reward": 1.6450969576835632, + "reward_std": 0.18919343501329422, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6450969874858856, + "step": 35 + }, + { + "clip_ratio": 0.0, + "completion_length": 107.8828125, + "epoch": 0.030927835051546393, + "grad_norm": 1.8376143817871085, + "kl": 0.017822265625, + "learning_rate": 9.845360824742267e-07, + "loss": 0.0007, + "reward": 1.6301739811897278, + "reward_std": 0.20516958832740784, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6379865407943726, + "step": 36 + }, + { + "clip_ratio": 0.0, + "completion_length": 98.4921875, + "epoch": 0.03178694158075601, + "grad_norm": 2.8326207376956565, + "kl": 0.009918212890625, + "learning_rate": 9.84106529209622e-07, + "loss": 0.0004, + "reward": 1.709650695323944, + "reward_std": 0.1534641981124878, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7096506357192993, + "step": 37 + }, + { + "clip_ratio": 0.0, + "completion_length": 101.6796875, + "epoch": 0.03264604810996564, + "grad_norm": 2.0347285959045536, + "kl": 0.009307861328125, + "learning_rate": 9.83676975945017e-07, + "loss": 0.0004, + "reward": 1.6739385724067688, + "reward_std": 0.13153361529111862, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6739385724067688, + "step": 38 + }, + { + "clip_ratio": 0.0, + "completion_length": 112.5625, + "epoch": 0.03350515463917526, + "grad_norm": 1.9197526254739072, + "kl": 0.01287841796875, + "learning_rate": 9.832474226804123e-07, + "loss": 0.0005, + "reward": 1.7046000361442566, + "reward_std": 0.19367831200361252, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7124125361442566, + "step": 39 + }, + { + "clip_ratio": 0.0, + "completion_length": 108.1328125, + "epoch": 0.03436426116838488, + "grad_norm": 3.1475346486430174, + "kl": 0.011077880859375, + "learning_rate": 9.828178694158075e-07, + "loss": 0.0004, + "reward": 1.714475929737091, + "reward_std": 0.11251556500792503, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7144758999347687, + "step": 40 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.6484375, + "epoch": 0.0352233676975945, + "grad_norm": 1.9386038530957954, + "kl": 0.013763427734375, + "learning_rate": 9.823883161512027e-07, + "loss": 0.0006, + "reward": 1.7007797360420227, + "reward_std": 0.22316401451826096, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7085922360420227, + "step": 41 + }, + { + "clip_ratio": 0.0, + "completion_length": 111.2734375, + "epoch": 0.03608247422680412, + "grad_norm": 1.6026084575263893, + "kl": 0.009185791015625, + "learning_rate": 9.819587628865979e-07, + "loss": 0.0004, + "reward": 1.7506839632987976, + "reward_std": 0.16962886601686478, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.75068399310112, + "step": 42 + }, + { + "clip_ratio": 0.0, + "completion_length": 109.265625, + "epoch": 0.036941580756013746, + "grad_norm": 1.4019849035616754, + "kl": 0.015960693359375, + "learning_rate": 9.81529209621993e-07, + "loss": 0.0006, + "reward": 1.6929492950439453, + "reward_std": 0.15880103036761284, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6929492652416229, + "step": 43 + }, + { + "clip_ratio": 0.0, + "completion_length": 108.78125, + "epoch": 0.037800687285223365, + "grad_norm": 1.874751262052608, + "kl": 0.01318359375, + "learning_rate": 9.810996563573882e-07, + "loss": 0.0005, + "reward": 1.7645366191864014, + "reward_std": 0.1825566589832306, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7723491191864014, + "step": 44 + }, + { + "clip_ratio": 0.0, + "completion_length": 104.1171875, + "epoch": 0.03865979381443299, + "grad_norm": 1.628943635799199, + "kl": 0.012939453125, + "learning_rate": 9.806701030927834e-07, + "loss": 0.0005, + "reward": 1.7157459259033203, + "reward_std": 0.13008157908916473, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7157459259033203, + "step": 45 + }, + { + "clip_ratio": 0.0, + "completion_length": 112.3203125, + "epoch": 0.03951890034364261, + "grad_norm": 1.3333890922786833, + "kl": 0.02374267578125, + "learning_rate": 9.802405498281786e-07, + "loss": 0.001, + "reward": 1.7577217817306519, + "reward_std": 0.1191495880484581, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7577218413352966, + "step": 46 + }, + { + "clip_ratio": 0.0, + "completion_length": 117.3359375, + "epoch": 0.040378006872852236, + "grad_norm": 2.7773278425143313, + "kl": 0.014434814453125, + "learning_rate": 9.798109965635738e-07, + "loss": 0.0006, + "reward": 1.6698015332221985, + "reward_std": 0.18330103904008865, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6776140332221985, + "step": 47 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.4296875, + "epoch": 0.041237113402061855, + "grad_norm": 3.1125459908194886, + "kl": 0.0120849609375, + "learning_rate": 9.79381443298969e-07, + "loss": 0.0005, + "reward": 1.7181047797203064, + "reward_std": 0.2058340311050415, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.718104749917984, + "step": 48 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.65625, + "epoch": 0.04209621993127148, + "grad_norm": 1.7917182356261732, + "kl": 0.01312255859375, + "learning_rate": 9.789518900343642e-07, + "loss": 0.0005, + "reward": 1.6312512159347534, + "reward_std": 0.1998831108212471, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.631251186132431, + "step": 49 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.765625, + "epoch": 0.0429553264604811, + "grad_norm": 2.715958501901883, + "kl": 0.01019287109375, + "learning_rate": 9.785223367697594e-07, + "loss": 0.0004, + "reward": 1.758800745010376, + "reward_std": 0.17326835542917252, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.758800745010376, + "step": 50 + }, + { + "clip_ratio": 0.0, + "completion_length": 113.609375, + "epoch": 0.04381443298969072, + "grad_norm": 1.6233811186388696, + "kl": 0.0115966796875, + "learning_rate": 9.780927835051545e-07, + "loss": 0.0005, + "reward": 1.693307340145111, + "reward_std": 0.16266920417547226, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6933073401451111, + "step": 51 + }, + { + "clip_ratio": 0.0, + "completion_length": 108.203125, + "epoch": 0.044673539518900345, + "grad_norm": 1.655879971578571, + "kl": 0.024871826171875, + "learning_rate": 9.776632302405497e-07, + "loss": 0.001, + "reward": 1.6892904043197632, + "reward_std": 0.17997007817029953, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7049154043197632, + "step": 52 + }, + { + "clip_ratio": 0.0, + "completion_length": 121.65625, + "epoch": 0.04553264604810996, + "grad_norm": 1.6681145967268827, + "kl": 0.010528564453125, + "learning_rate": 9.77233676975945e-07, + "loss": 0.0004, + "reward": 1.7307260036468506, + "reward_std": 0.18761465698480606, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7307261228561401, + "step": 53 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.5078125, + "epoch": 0.04639175257731959, + "grad_norm": 1.9783878004259334, + "kl": 0.00994873046875, + "learning_rate": 9.768041237113401e-07, + "loss": 0.0004, + "reward": 1.6929954290390015, + "reward_std": 0.15189384669065475, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6929953396320343, + "step": 54 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.3828125, + "epoch": 0.04725085910652921, + "grad_norm": 1.6703845988223984, + "kl": 0.014129638671875, + "learning_rate": 9.763745704467353e-07, + "loss": 0.0006, + "reward": 1.616851270198822, + "reward_std": 0.18548668175935745, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6168512403964996, + "step": 55 + }, + { + "clip_ratio": 0.0, + "completion_length": 117.6015625, + "epoch": 0.048109965635738834, + "grad_norm": 1.5488291217502541, + "kl": 0.013580322265625, + "learning_rate": 9.759450171821305e-07, + "loss": 0.0005, + "reward": 1.7046599388122559, + "reward_std": 0.22010766714811325, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7124724388122559, + "step": 56 + }, + { + "clip_ratio": 0.0, + "completion_length": 138.9453125, + "epoch": 0.04896907216494845, + "grad_norm": 1.4634966134639982, + "kl": 0.0120849609375, + "learning_rate": 9.755154639175257e-07, + "loss": 0.0005, + "reward": 1.6978161334991455, + "reward_std": 0.17123103514313698, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7134410738945007, + "step": 57 + }, + { + "clip_ratio": 0.0, + "completion_length": 104.203125, + "epoch": 0.04982817869415808, + "grad_norm": 2.773139620924608, + "kl": 0.01214599609375, + "learning_rate": 9.750859106529209e-07, + "loss": 0.0005, + "reward": 1.7033132314682007, + "reward_std": 0.17858785390853882, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7033132314682007, + "step": 58 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.6484375, + "epoch": 0.0506872852233677, + "grad_norm": 2.112316846879123, + "kl": 0.015411376953125, + "learning_rate": 9.74656357388316e-07, + "loss": 0.0006, + "reward": 1.7434183359146118, + "reward_std": 0.1504114419221878, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7434183359146118, + "step": 59 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.8203125, + "epoch": 0.05154639175257732, + "grad_norm": 4.733899218910252, + "kl": 0.0123291015625, + "learning_rate": 9.742268041237112e-07, + "loss": 0.0005, + "reward": 1.7861030101776123, + "reward_std": 0.21234874427318573, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7939155101776123, + "step": 60 + }, + { + "clip_ratio": 0.0, + "completion_length": 110.7734375, + "epoch": 0.05240549828178694, + "grad_norm": 2.410724757522067, + "kl": 0.01287841796875, + "learning_rate": 9.737972508591064e-07, + "loss": 0.0005, + "reward": 1.7895718216896057, + "reward_std": 0.1751849427819252, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7973843216896057, + "step": 61 + }, + { + "clip_ratio": 0.0, + "completion_length": 114.0, + "epoch": 0.05326460481099656, + "grad_norm": 1.3683631722816978, + "kl": 0.0125732421875, + "learning_rate": 9.733676975945016e-07, + "loss": 0.0005, + "reward": 1.6282992959022522, + "reward_std": 0.1704944670200348, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.6439243257045746, + "step": 62 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.2421875, + "epoch": 0.05412371134020619, + "grad_norm": 1.6450659007412214, + "kl": 0.01019287109375, + "learning_rate": 9.729381443298968e-07, + "loss": 0.0004, + "reward": 1.6509077548980713, + "reward_std": 0.17651135474443436, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6509077548980713, + "step": 63 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.171875, + "epoch": 0.054982817869415807, + "grad_norm": 1.6294625649527108, + "kl": 0.01202392578125, + "learning_rate": 9.72508591065292e-07, + "loss": 0.0005, + "reward": 1.7486688494682312, + "reward_std": 0.2049519494175911, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7564813494682312, + "step": 64 + }, + { + "clip_ratio": 0.0, + "completion_length": 112.4375, + "epoch": 0.05584192439862543, + "grad_norm": 1.3207489104187833, + "kl": 0.014923095703125, + "learning_rate": 9.720790378006872e-07, + "loss": 0.0006, + "reward": 1.686493158340454, + "reward_std": 0.1424517035484314, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7021180987358093, + "step": 65 + }, + { + "clip_ratio": 0.0, + "completion_length": 107.4453125, + "epoch": 0.05670103092783505, + "grad_norm": 1.8194250939249632, + "kl": 0.01617431640625, + "learning_rate": 9.716494845360824e-07, + "loss": 0.0006, + "reward": 1.7264857292175293, + "reward_std": 0.16944430023431778, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7342982888221741, + "step": 66 + }, + { + "clip_ratio": 0.0, + "completion_length": 122.1953125, + "epoch": 0.05756013745704467, + "grad_norm": 1.8454925104090358, + "kl": 0.013275146484375, + "learning_rate": 9.712199312714776e-07, + "loss": 0.0005, + "reward": 1.7975256443023682, + "reward_std": 0.18923871964216232, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7975256443023682, + "step": 67 + }, + { + "clip_ratio": 0.0, + "completion_length": 110.2265625, + "epoch": 0.058419243986254296, + "grad_norm": 1.8448295930855816, + "kl": 0.012298583984375, + "learning_rate": 9.707903780068727e-07, + "loss": 0.0005, + "reward": 1.7086012363433838, + "reward_std": 0.14006467163562775, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7086012661457062, + "step": 68 + }, + { + "clip_ratio": 0.0, + "completion_length": 111.421875, + "epoch": 0.059278350515463915, + "grad_norm": 1.5791228430586928, + "kl": 0.0155029296875, + "learning_rate": 9.70360824742268e-07, + "loss": 0.0006, + "reward": 1.799054503440857, + "reward_std": 0.14379648119211197, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8068670034408569, + "step": 69 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.359375, + "epoch": 0.06013745704467354, + "grad_norm": 1.3943612865559436, + "kl": 0.010162353515625, + "learning_rate": 9.699312714776631e-07, + "loss": 0.0004, + "reward": 1.8229413032531738, + "reward_std": 0.15740595757961273, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8229413330554962, + "step": 70 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.6953125, + "epoch": 0.06099656357388316, + "grad_norm": 2.8156538308984262, + "kl": 0.009796142578125, + "learning_rate": 9.695017182130583e-07, + "loss": 0.0004, + "reward": 1.7003414630889893, + "reward_std": 0.19474071264266968, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7159664928913116, + "step": 71 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.2265625, + "epoch": 0.061855670103092786, + "grad_norm": 1.866123797516234, + "kl": 0.0084228515625, + "learning_rate": 9.690721649484535e-07, + "loss": 0.0003, + "reward": 1.7515292763710022, + "reward_std": 0.1583714485168457, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7515292763710022, + "step": 72 + }, + { + "clip_ratio": 0.0, + "completion_length": 143.3671875, + "epoch": 0.0627147766323024, + "grad_norm": 1.394074930738141, + "kl": 0.011199951171875, + "learning_rate": 9.686426116838487e-07, + "loss": 0.0004, + "reward": 1.679770588874817, + "reward_std": 0.21282349526882172, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7110206186771393, + "step": 73 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.3984375, + "epoch": 0.06357388316151202, + "grad_norm": 4.5617894751936285, + "kl": 0.019927978515625, + "learning_rate": 9.682130584192439e-07, + "loss": 0.0008, + "reward": 1.6840202808380127, + "reward_std": 0.18173471838235855, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6918327808380127, + "step": 74 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.40625, + "epoch": 0.06443298969072164, + "grad_norm": 20.644470378771178, + "kl": 0.0096435546875, + "learning_rate": 9.67783505154639e-07, + "loss": 0.0004, + "reward": 1.6653663516044617, + "reward_std": 0.15374305844306946, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6653663218021393, + "step": 75 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.75, + "epoch": 0.06529209621993128, + "grad_norm": 1.3520802500043023, + "kl": 0.011871337890625, + "learning_rate": 9.673539518900342e-07, + "loss": 0.0005, + "reward": 1.7905267477035522, + "reward_std": 0.11402217298746109, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.790526807308197, + "step": 76 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.0703125, + "epoch": 0.0661512027491409, + "grad_norm": 2.1828273386945662, + "kl": 0.008880615234375, + "learning_rate": 9.669243986254294e-07, + "loss": 0.0004, + "reward": 1.6396149396896362, + "reward_std": 0.22704153507947922, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.6552398800849915, + "step": 77 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.5625, + "epoch": 0.06701030927835051, + "grad_norm": 1.9960711680675047, + "kl": 0.01043701171875, + "learning_rate": 9.664948453608246e-07, + "loss": 0.0004, + "reward": 1.717953085899353, + "reward_std": 0.1279629021883011, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7179530560970306, + "step": 78 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.375, + "epoch": 0.06786941580756013, + "grad_norm": 1.573275119243184, + "kl": 0.009918212890625, + "learning_rate": 9.660652920962198e-07, + "loss": 0.0004, + "reward": 1.7117525339126587, + "reward_std": 0.17472369223833084, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7117525339126587, + "step": 79 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.6953125, + "epoch": 0.06872852233676977, + "grad_norm": 5.858171715596188, + "kl": 0.0074462890625, + "learning_rate": 9.65635738831615e-07, + "loss": 0.0003, + "reward": 1.7692703604698181, + "reward_std": 0.18383784592151642, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7692703902721405, + "step": 80 + }, + { + "clip_ratio": 0.0, + "completion_length": 121.8203125, + "epoch": 0.06958762886597938, + "grad_norm": 2.153346346698614, + "kl": 0.0079345703125, + "learning_rate": 9.652061855670102e-07, + "loss": 0.0003, + "reward": 1.7922507524490356, + "reward_std": 0.16792288422584534, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7922507226467133, + "step": 81 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.2890625, + "epoch": 0.070446735395189, + "grad_norm": 1.5730677509411617, + "kl": 0.009002685546875, + "learning_rate": 9.647766323024054e-07, + "loss": 0.0004, + "reward": 1.7323782444000244, + "reward_std": 0.17609868198633194, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7480032444000244, + "step": 82 + }, + { + "clip_ratio": 0.0, + "completion_length": 134.578125, + "epoch": 0.07130584192439862, + "grad_norm": 1.374124074188527, + "kl": 0.011138916015625, + "learning_rate": 9.643470790378006e-07, + "loss": 0.0004, + "reward": 1.6585241556167603, + "reward_std": 0.19784274697303772, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.6819616854190826, + "step": 83 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.9296875, + "epoch": 0.07216494845360824, + "grad_norm": 2.383137509998003, + "kl": 0.01141357421875, + "learning_rate": 9.639175257731957e-07, + "loss": 0.0005, + "reward": 1.6670758724212646, + "reward_std": 0.19278159737586975, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6670758724212646, + "step": 84 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.578125, + "epoch": 0.07302405498281787, + "grad_norm": 1.7362778804071193, + "kl": 0.011749267578125, + "learning_rate": 9.63487972508591e-07, + "loss": 0.0005, + "reward": 1.7598541378974915, + "reward_std": 0.1449267938733101, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7754791080951691, + "step": 85 + }, + { + "clip_ratio": 0.0, + "completion_length": 164.609375, + "epoch": 0.07388316151202749, + "grad_norm": 2.8650428143418325, + "kl": 0.012176513671875, + "learning_rate": 9.630584192439863e-07, + "loss": 0.0005, + "reward": 1.6261486411094666, + "reward_std": 0.29106324911117554, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.6573987007141113, + "step": 86 + }, + { + "clip_ratio": 0.0, + "completion_length": 134.90625, + "epoch": 0.07474226804123711, + "grad_norm": 1.9633221241492278, + "kl": 0.0074920654296875, + "learning_rate": 9.626288659793815e-07, + "loss": 0.0003, + "reward": 1.7269136905670166, + "reward_std": 0.1622983068227768, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7347261309623718, + "step": 87 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.75, + "epoch": 0.07560137457044673, + "grad_norm": 2.0681449704464954, + "kl": 0.0077056884765625, + "learning_rate": 9.621993127147767e-07, + "loss": 0.0003, + "reward": 1.7325759530067444, + "reward_std": 0.14999449253082275, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7325759530067444, + "step": 88 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.3984375, + "epoch": 0.07646048109965636, + "grad_norm": 1.1844639755913142, + "kl": 0.007354736328125, + "learning_rate": 9.61769759450172e-07, + "loss": 0.0003, + "reward": 1.7702937126159668, + "reward_std": 0.12665041163563728, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7702936828136444, + "step": 89 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.25, + "epoch": 0.07731958762886598, + "grad_norm": 1.3413729169105462, + "kl": 0.0083465576171875, + "learning_rate": 9.61340206185567e-07, + "loss": 0.0003, + "reward": 1.6900493502616882, + "reward_std": 0.08954241871833801, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6978618502616882, + "step": 90 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.421875, + "epoch": 0.0781786941580756, + "grad_norm": 1.9584992131927317, + "kl": 0.015350341796875, + "learning_rate": 9.609106529209623e-07, + "loss": 0.0006, + "reward": 1.6389594078063965, + "reward_std": 0.231736421585083, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.6545844078063965, + "step": 91 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.6796875, + "epoch": 0.07903780068728522, + "grad_norm": 1.4808391695134289, + "kl": 0.015228271484375, + "learning_rate": 9.604810996563575e-07, + "loss": 0.0006, + "reward": 1.7157284617424011, + "reward_std": 0.14664900302886963, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7313534915447235, + "step": 92 + }, + { + "clip_ratio": 0.0, + "completion_length": 145.8515625, + "epoch": 0.07989690721649484, + "grad_norm": 1.3560060082947887, + "kl": 0.011688232421875, + "learning_rate": 9.600515463917527e-07, + "loss": 0.0005, + "reward": 1.6663364171981812, + "reward_std": 0.17353109270334244, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.6897739470005035, + "step": 93 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.78125, + "epoch": 0.08075601374570447, + "grad_norm": 1.4417733286684913, + "kl": 0.009521484375, + "learning_rate": 9.596219931271478e-07, + "loss": 0.0004, + "reward": 1.695834457874298, + "reward_std": 0.15896284580230713, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7270845174789429, + "step": 94 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.7421875, + "epoch": 0.08161512027491409, + "grad_norm": 1.81422275990836, + "kl": 0.008056640625, + "learning_rate": 9.59192439862543e-07, + "loss": 0.0003, + "reward": 1.7134593725204468, + "reward_std": 0.16323717311024666, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7134594321250916, + "step": 95 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.21875, + "epoch": 0.08247422680412371, + "grad_norm": 1.2278195202315274, + "kl": 0.015380859375, + "learning_rate": 9.587628865979382e-07, + "loss": 0.0006, + "reward": 1.833047330379486, + "reward_std": 0.15521731600165367, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8486724197864532, + "step": 96 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.7890625, + "epoch": 0.08333333333333333, + "grad_norm": 1.7371338217464498, + "kl": 0.012603759765625, + "learning_rate": 9.583333333333334e-07, + "loss": 0.0005, + "reward": 1.6824183464050293, + "reward_std": 0.1946360394358635, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6824184060096741, + "step": 97 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.09375, + "epoch": 0.08419243986254296, + "grad_norm": 2.9141007581916285, + "kl": 0.01483154296875, + "learning_rate": 9.579037800687286e-07, + "loss": 0.0006, + "reward": 1.7687426805496216, + "reward_std": 0.16979511827230453, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7687426805496216, + "step": 98 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.703125, + "epoch": 0.08505154639175258, + "grad_norm": 3.157985367093318, + "kl": 0.012939453125, + "learning_rate": 9.574742268041238e-07, + "loss": 0.0005, + "reward": 1.8471919298171997, + "reward_std": 0.08835843577980995, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8471919894218445, + "step": 99 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.96875, + "epoch": 0.0859106529209622, + "grad_norm": 1.791601113718697, + "kl": 0.01171875, + "learning_rate": 9.57044673539519e-07, + "loss": 0.0005, + "reward": 1.7132195234298706, + "reward_std": 0.1518014371395111, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7132195234298706, + "step": 100 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.640625, + "epoch": 0.08676975945017182, + "grad_norm": 1.6750678433197985, + "kl": 0.01409912109375, + "learning_rate": 9.566151202749142e-07, + "loss": 0.0006, + "reward": 1.6660110354423523, + "reward_std": 0.16518359631299973, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6660110354423523, + "step": 101 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.453125, + "epoch": 0.08762886597938144, + "grad_norm": 2.100280147210653, + "kl": 0.014434814453125, + "learning_rate": 9.561855670103093e-07, + "loss": 0.0006, + "reward": 1.7369045615196228, + "reward_std": 0.11925114691257477, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.736904501914978, + "step": 102 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.5703125, + "epoch": 0.08848797250859107, + "grad_norm": 2.48977370571022, + "kl": 0.012603759765625, + "learning_rate": 9.557560137457045e-07, + "loss": 0.0005, + "reward": 1.7730196118354797, + "reward_std": 0.18081708252429962, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7808321118354797, + "step": 103 + }, + { + "clip_ratio": 0.0, + "completion_length": 115.546875, + "epoch": 0.08934707903780069, + "grad_norm": 2.083788782587846, + "kl": 0.0164794921875, + "learning_rate": 9.553264604810997e-07, + "loss": 0.0007, + "reward": 1.668508231639862, + "reward_std": 0.1277586668729782, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6763207614421844, + "step": 104 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.1953125, + "epoch": 0.09020618556701031, + "grad_norm": 3.157040349398891, + "kl": 0.015838623046875, + "learning_rate": 9.54896907216495e-07, + "loss": 0.0006, + "reward": 1.792285680770874, + "reward_std": 0.1403278186917305, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.800098180770874, + "step": 105 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.53125, + "epoch": 0.09106529209621993, + "grad_norm": 1.9043694578355221, + "kl": 0.01544189453125, + "learning_rate": 9.5446735395189e-07, + "loss": 0.0006, + "reward": 1.7426919341087341, + "reward_std": 0.17454702407121658, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7426919043064117, + "step": 106 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.0234375, + "epoch": 0.09192439862542956, + "grad_norm": 1.1762483491043985, + "kl": 0.013885498046875, + "learning_rate": 9.540378006872853e-07, + "loss": 0.0006, + "reward": 1.76499605178833, + "reward_std": 0.12454288452863693, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7649960517883301, + "step": 107 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.375, + "epoch": 0.09278350515463918, + "grad_norm": 2.4203700264289822, + "kl": 0.014434814453125, + "learning_rate": 9.536082474226805e-07, + "loss": 0.0006, + "reward": 1.7665973901748657, + "reward_std": 0.16096310317516327, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7900348901748657, + "step": 108 + }, + { + "clip_ratio": 0.0, + "completion_length": 122.6796875, + "epoch": 0.0936426116838488, + "grad_norm": 1.1516404581818234, + "kl": 0.013824462890625, + "learning_rate": 9.531786941580757e-07, + "loss": 0.0006, + "reward": 1.7484302520751953, + "reward_std": 0.11297959834337234, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7562427520751953, + "step": 109 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.765625, + "epoch": 0.09450171821305842, + "grad_norm": 1.8664322686617876, + "kl": 0.013092041015625, + "learning_rate": 9.527491408934707e-07, + "loss": 0.0005, + "reward": 1.7427197694778442, + "reward_std": 0.1969207152724266, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7505322694778442, + "step": 110 + }, + { + "clip_ratio": 0.0, + "completion_length": 134.4140625, + "epoch": 0.09536082474226804, + "grad_norm": 2.8553637446748055, + "kl": 0.01678466796875, + "learning_rate": 9.523195876288659e-07, + "loss": 0.0007, + "reward": 1.6789370775222778, + "reward_std": 0.14484084397554398, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6867496371269226, + "step": 111 + }, + { + "clip_ratio": 0.0, + "completion_length": 110.9765625, + "epoch": 0.09621993127147767, + "grad_norm": 2.8547235701713527, + "kl": 0.01629638671875, + "learning_rate": 9.518900343642611e-07, + "loss": 0.0007, + "reward": 1.7494203448295593, + "reward_std": 0.17836876958608627, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7572328150272369, + "step": 112 + }, + { + "clip_ratio": 0.0, + "completion_length": 122.1328125, + "epoch": 0.09707903780068729, + "grad_norm": 1.1274603410680148, + "kl": 0.014923095703125, + "learning_rate": 9.514604810996563e-07, + "loss": 0.0006, + "reward": 1.775137186050415, + "reward_std": 0.12154169753193855, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7829497158527374, + "step": 113 + }, + { + "clip_ratio": 0.0, + "completion_length": 107.5625, + "epoch": 0.0979381443298969, + "grad_norm": 2.4835835333144347, + "kl": 0.025482177734375, + "learning_rate": 9.510309278350515e-07, + "loss": 0.001, + "reward": 1.6830406785011292, + "reward_std": 0.07851138710975647, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.6986656486988068, + "step": 114 + }, + { + "clip_ratio": 0.0, + "completion_length": 117.2578125, + "epoch": 0.09879725085910653, + "grad_norm": 1.457424888302412, + "kl": 0.01715087890625, + "learning_rate": 9.506013745704467e-07, + "loss": 0.0007, + "reward": 1.8288711309432983, + "reward_std": 0.1552474945783615, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8366836309432983, + "step": 115 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.15625, + "epoch": 0.09965635738831616, + "grad_norm": 5.261633631939683, + "kl": 0.012969970703125, + "learning_rate": 9.501718213058419e-07, + "loss": 0.0005, + "reward": 1.7097431421279907, + "reward_std": 0.1742759346961975, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7097431421279907, + "step": 116 + }, + { + "clip_ratio": 0.0, + "completion_length": 142.25, + "epoch": 0.10051546391752578, + "grad_norm": 2.0866973884427784, + "kl": 0.01885986328125, + "learning_rate": 9.497422680412371e-07, + "loss": 0.0008, + "reward": 1.7714307308197021, + "reward_std": 0.2078239470720291, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7714307308197021, + "step": 117 + }, + { + "clip_ratio": 0.0, + "completion_length": 122.0546875, + "epoch": 0.1013745704467354, + "grad_norm": 1.9401421255992048, + "kl": 0.0369873046875, + "learning_rate": 9.493127147766322e-07, + "loss": 0.0015, + "reward": 1.7130677700042725, + "reward_std": 0.19828316569328308, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7208802998065948, + "step": 118 + }, + { + "clip_ratio": 0.0, + "completion_length": 120.390625, + "epoch": 0.10223367697594501, + "grad_norm": 2.5355319382822827, + "kl": 0.0181884765625, + "learning_rate": 9.488831615120274e-07, + "loss": 0.0007, + "reward": 1.737857460975647, + "reward_std": 0.1541828289628029, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7378574907779694, + "step": 119 + }, + { + "clip_ratio": 0.0, + "completion_length": 120.7265625, + "epoch": 0.10309278350515463, + "grad_norm": 1.4772557885572284, + "kl": 0.02142333984375, + "learning_rate": 9.484536082474226e-07, + "loss": 0.0009, + "reward": 1.7606690526008606, + "reward_std": 0.18398111313581467, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7606690227985382, + "step": 120 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.7578125, + "epoch": 0.10395189003436427, + "grad_norm": 2.4929887339743244, + "kl": 0.01837158203125, + "learning_rate": 9.480240549828178e-07, + "loss": 0.0007, + "reward": 1.727208137512207, + "reward_std": 0.13875007256865501, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.727208137512207, + "step": 121 + }, + { + "clip_ratio": 0.0, + "completion_length": 114.640625, + "epoch": 0.10481099656357389, + "grad_norm": 3.0006792162829257, + "kl": 0.02630615234375, + "learning_rate": 9.47594501718213e-07, + "loss": 0.001, + "reward": 1.8001123070716858, + "reward_std": 0.20054399222135544, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8157372772693634, + "step": 122 + }, + { + "clip_ratio": 0.0, + "completion_length": 113.046875, + "epoch": 0.1056701030927835, + "grad_norm": 2.6748232943569827, + "kl": 0.0206298828125, + "learning_rate": 9.471649484536082e-07, + "loss": 0.0008, + "reward": 1.7969177961349487, + "reward_std": 0.10830671712756157, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8047303557395935, + "step": 123 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.953125, + "epoch": 0.10652920962199312, + "grad_norm": 1.4841628651180314, + "kl": 0.024658203125, + "learning_rate": 9.467353951890034e-07, + "loss": 0.001, + "reward": 1.7056427001953125, + "reward_std": 0.1875780150294304, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7368926405906677, + "step": 124 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.203125, + "epoch": 0.10738831615120274, + "grad_norm": 1.874212030480472, + "kl": 0.023193359375, + "learning_rate": 9.463058419243986e-07, + "loss": 0.0009, + "reward": 1.6158097386360168, + "reward_std": 0.1327304244041443, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6236222088336945, + "step": 125 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.75, + "epoch": 0.10824742268041238, + "grad_norm": 1.0408020027656946, + "kl": 0.014923095703125, + "learning_rate": 9.458762886597938e-07, + "loss": 0.0006, + "reward": 1.7203240394592285, + "reward_std": 0.1105212289839983, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7281366288661957, + "step": 126 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.359375, + "epoch": 0.109106529209622, + "grad_norm": 1.2726802820456735, + "kl": 0.0220947265625, + "learning_rate": 9.454467353951889e-07, + "loss": 0.0009, + "reward": 1.6998094320297241, + "reward_std": 0.16664542257785797, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7076218724250793, + "step": 127 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.765625, + "epoch": 0.10996563573883161, + "grad_norm": 1.6961988401912567, + "kl": 0.015594482421875, + "learning_rate": 9.450171821305841e-07, + "loss": 0.0006, + "reward": 1.6439663171768188, + "reward_std": 0.16561322659254074, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6517787873744965, + "step": 128 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.390625, + "epoch": 0.11082474226804123, + "grad_norm": 2.510071105917193, + "kl": 0.015350341796875, + "learning_rate": 9.445876288659793e-07, + "loss": 0.0006, + "reward": 1.66585111618042, + "reward_std": 0.1521969810128212, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6736635565757751, + "step": 129 + }, + { + "clip_ratio": 0.0, + "completion_length": 117.484375, + "epoch": 0.11168384879725086, + "grad_norm": 5.638902477820217, + "kl": 0.01513671875, + "learning_rate": 9.441580756013745e-07, + "loss": 0.0006, + "reward": 1.7457653284072876, + "reward_std": 0.15448395907878876, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7457653880119324, + "step": 130 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.1484375, + "epoch": 0.11254295532646048, + "grad_norm": 1.513851686329793, + "kl": 0.015289306640625, + "learning_rate": 9.437285223367697e-07, + "loss": 0.0006, + "reward": 1.7635972499847412, + "reward_std": 0.17401929199695587, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7714097797870636, + "step": 131 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.3359375, + "epoch": 0.1134020618556701, + "grad_norm": 1.322125994104397, + "kl": 0.012359619140625, + "learning_rate": 9.432989690721649e-07, + "loss": 0.0005, + "reward": 1.823841154575348, + "reward_std": 0.14871416985988617, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8238411545753479, + "step": 132 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.0, + "epoch": 0.11426116838487972, + "grad_norm": 1.355698173704868, + "kl": 0.023406982421875, + "learning_rate": 9.428694158075601e-07, + "loss": 0.0009, + "reward": 1.766915202140808, + "reward_std": 0.14532212167978287, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7825402021408081, + "step": 133 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.4140625, + "epoch": 0.11512027491408934, + "grad_norm": 1.4739462448440241, + "kl": 0.0181884765625, + "learning_rate": 9.424398625429553e-07, + "loss": 0.0007, + "reward": 1.7099063992500305, + "reward_std": 0.1308385580778122, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7099063992500305, + "step": 134 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.9296875, + "epoch": 0.11597938144329897, + "grad_norm": 1.3274707295914516, + "kl": 0.014129638671875, + "learning_rate": 9.420103092783504e-07, + "loss": 0.0006, + "reward": 1.7741219997406006, + "reward_std": 0.16581101715564728, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7819344997406006, + "step": 135 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.140625, + "epoch": 0.11683848797250859, + "grad_norm": 1.5843650427627012, + "kl": 0.01092529296875, + "learning_rate": 9.415807560137456e-07, + "loss": 0.0004, + "reward": 1.6576457023620605, + "reward_std": 0.1768977865576744, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6654582023620605, + "step": 136 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.8984375, + "epoch": 0.11769759450171821, + "grad_norm": 1.670051625687803, + "kl": 0.012542724609375, + "learning_rate": 9.411512027491408e-07, + "loss": 0.0005, + "reward": 1.7060455083847046, + "reward_std": 0.14821957796812057, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.706045538187027, + "step": 137 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.953125, + "epoch": 0.11855670103092783, + "grad_norm": 1.64228564707855, + "kl": 0.0146484375, + "learning_rate": 9.40721649484536e-07, + "loss": 0.0006, + "reward": 1.698437750339508, + "reward_std": 0.1564936824142933, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6984376907348633, + "step": 138 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.15625, + "epoch": 0.11941580756013746, + "grad_norm": 1.376863304210578, + "kl": 0.010498046875, + "learning_rate": 9.402920962199312e-07, + "loss": 0.0004, + "reward": 1.8016321063041687, + "reward_std": 0.15034572780132294, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8172571659088135, + "step": 139 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.8359375, + "epoch": 0.12027491408934708, + "grad_norm": 6.099834295001363, + "kl": 0.013580322265625, + "learning_rate": 9.398625429553264e-07, + "loss": 0.0005, + "reward": 1.6821348667144775, + "reward_std": 0.13406795635819435, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6821348965167999, + "step": 140 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.953125, + "epoch": 0.1211340206185567, + "grad_norm": 1.6651173493841875, + "kl": 0.01617431640625, + "learning_rate": 9.394329896907216e-07, + "loss": 0.0006, + "reward": 1.7098830938339233, + "reward_std": 0.17125768959522247, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7333205342292786, + "step": 141 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.2265625, + "epoch": 0.12199312714776632, + "grad_norm": 2.528179883736634, + "kl": 0.013946533203125, + "learning_rate": 9.390034364261168e-07, + "loss": 0.0006, + "reward": 1.7531933784484863, + "reward_std": 0.15759651362895966, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7610058188438416, + "step": 142 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.453125, + "epoch": 0.12285223367697594, + "grad_norm": 3.1296650163459088, + "kl": 0.01806640625, + "learning_rate": 9.38573883161512e-07, + "loss": 0.0007, + "reward": 1.8122310042381287, + "reward_std": 0.17129109799861908, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8122310042381287, + "step": 143 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.421875, + "epoch": 0.12371134020618557, + "grad_norm": 2.002669092661699, + "kl": 0.01312255859375, + "learning_rate": 9.381443298969071e-07, + "loss": 0.0005, + "reward": 1.7977581024169922, + "reward_std": 0.145246010273695, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8055705428123474, + "step": 144 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.7421875, + "epoch": 0.12457044673539519, + "grad_norm": 1.2100201264770116, + "kl": 0.0140380859375, + "learning_rate": 9.377147766323023e-07, + "loss": 0.0006, + "reward": 1.829803466796875, + "reward_std": 0.1415480300784111, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.837615966796875, + "step": 145 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.3203125, + "epoch": 0.1254295532646048, + "grad_norm": 2.180495745625869, + "kl": 0.01171875, + "learning_rate": 9.372852233676975e-07, + "loss": 0.0005, + "reward": 1.7289317846298218, + "reward_std": 0.18080250918865204, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7445567846298218, + "step": 146 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.84375, + "epoch": 0.12628865979381443, + "grad_norm": 2.9127570085332892, + "kl": 0.01275634765625, + "learning_rate": 9.368556701030927e-07, + "loss": 0.0005, + "reward": 1.8339774012565613, + "reward_std": 0.09063693135976791, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8339774012565613, + "step": 147 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.8125, + "epoch": 0.12714776632302405, + "grad_norm": 1.5988443873795521, + "kl": 0.015777587890625, + "learning_rate": 9.364261168384879e-07, + "loss": 0.0006, + "reward": 1.7010595202445984, + "reward_std": 0.21928820759058, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7010595202445984, + "step": 148 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.015625, + "epoch": 0.12800687285223367, + "grad_norm": 2.3035156258187697, + "kl": 0.016021728515625, + "learning_rate": 9.359965635738831e-07, + "loss": 0.0006, + "reward": 1.7444458603858948, + "reward_std": 0.21774866431951523, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7444458901882172, + "step": 149 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.453125, + "epoch": 0.12886597938144329, + "grad_norm": 1.335338188075339, + "kl": 0.014312744140625, + "learning_rate": 9.355670103092783e-07, + "loss": 0.0006, + "reward": 1.8212101459503174, + "reward_std": 0.08616838604211807, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8212102353572845, + "step": 150 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.2734375, + "epoch": 0.12972508591065293, + "grad_norm": 1.4631690042303354, + "kl": 0.01123046875, + "learning_rate": 9.351374570446736e-07, + "loss": 0.0004, + "reward": 1.7321330308914185, + "reward_std": 0.1448444351553917, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7321330606937408, + "step": 151 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.3515625, + "epoch": 0.13058419243986255, + "grad_norm": 0.9273287057028194, + "kl": 0.009765625, + "learning_rate": 9.347079037800687e-07, + "loss": 0.0004, + "reward": 1.7624321579933167, + "reward_std": 0.09383735246956348, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7624321281909943, + "step": 152 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.8515625, + "epoch": 0.13144329896907217, + "grad_norm": 1.47382353081262, + "kl": 0.0152587890625, + "learning_rate": 9.342783505154639e-07, + "loss": 0.0006, + "reward": 1.6556521654129028, + "reward_std": 0.15647713094949722, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6634646654129028, + "step": 153 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.5859375, + "epoch": 0.1323024054982818, + "grad_norm": 7.187735004337097, + "kl": 0.01318359375, + "learning_rate": 9.338487972508591e-07, + "loss": 0.0005, + "reward": 1.781952142715454, + "reward_std": 0.20670166611671448, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7897646427154541, + "step": 154 + }, + { + "clip_ratio": 0.0, + "completion_length": 117.8828125, + "epoch": 0.1331615120274914, + "grad_norm": 1.5089418750844499, + "kl": 0.016754150390625, + "learning_rate": 9.334192439862543e-07, + "loss": 0.0007, + "reward": 1.763621985912323, + "reward_std": 0.18807729333639145, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7792469561100006, + "step": 155 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.15625, + "epoch": 0.13402061855670103, + "grad_norm": 14.193518545363872, + "kl": 0.012908935546875, + "learning_rate": 9.329896907216495e-07, + "loss": 0.0005, + "reward": 1.8192681670188904, + "reward_std": 0.11732286959886551, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8192681968212128, + "step": 156 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.5859375, + "epoch": 0.13487972508591065, + "grad_norm": 1.5467385079039138, + "kl": 0.012451171875, + "learning_rate": 9.325601374570447e-07, + "loss": 0.0005, + "reward": 1.710074007511139, + "reward_std": 0.2205553948879242, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7178865075111389, + "step": 157 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.3671875, + "epoch": 0.13573883161512026, + "grad_norm": 1.4389192395608355, + "kl": 0.0135498046875, + "learning_rate": 9.321305841924399e-07, + "loss": 0.0005, + "reward": 1.6988090872764587, + "reward_std": 0.10531151667237282, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6988090872764587, + "step": 158 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.109375, + "epoch": 0.13659793814432988, + "grad_norm": 1.6050785903093132, + "kl": 0.016448974609375, + "learning_rate": 9.317010309278351e-07, + "loss": 0.0007, + "reward": 1.7135010957717896, + "reward_std": 0.21779370307922363, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7213136553764343, + "step": 159 + }, + { + "clip_ratio": 0.0, + "completion_length": 122.8828125, + "epoch": 0.13745704467353953, + "grad_norm": 1.4854987369310133, + "kl": 0.016021728515625, + "learning_rate": 9.312714776632303e-07, + "loss": 0.0006, + "reward": 1.7385917901992798, + "reward_std": 0.15268366783857346, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7464043200016022, + "step": 160 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.8203125, + "epoch": 0.13831615120274915, + "grad_norm": 3.524052752645729, + "kl": 0.0146484375, + "learning_rate": 9.308419243986254e-07, + "loss": 0.0006, + "reward": 1.7990399599075317, + "reward_std": 0.15285367518663406, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.799039900302887, + "step": 161 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.203125, + "epoch": 0.13917525773195877, + "grad_norm": 2.86620135877599, + "kl": 0.014190673828125, + "learning_rate": 9.304123711340206e-07, + "loss": 0.0006, + "reward": 1.6504857540130615, + "reward_std": 0.1360258013010025, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6504857540130615, + "step": 162 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.4609375, + "epoch": 0.1400343642611684, + "grad_norm": 1.4169335821055606, + "kl": 0.014739990234375, + "learning_rate": 9.299828178694158e-07, + "loss": 0.0006, + "reward": 1.7095910906791687, + "reward_std": 0.08373668044805527, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7174035906791687, + "step": 163 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.8515625, + "epoch": 0.140893470790378, + "grad_norm": 1.3575110989008008, + "kl": 0.01556396484375, + "learning_rate": 9.29553264604811e-07, + "loss": 0.0006, + "reward": 1.7448028326034546, + "reward_std": 0.14756760746240616, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7448029220104218, + "step": 164 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.1171875, + "epoch": 0.14175257731958762, + "grad_norm": 1.8530325477228802, + "kl": 0.01324462890625, + "learning_rate": 9.291237113402062e-07, + "loss": 0.0005, + "reward": 1.742842435836792, + "reward_std": 0.16235359013080597, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7428424954414368, + "step": 165 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.75, + "epoch": 0.14261168384879724, + "grad_norm": 1.465950210564534, + "kl": 0.01416015625, + "learning_rate": 9.286941580756014e-07, + "loss": 0.0006, + "reward": 1.813442349433899, + "reward_std": 0.1405864767730236, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8134423494338989, + "step": 166 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.2265625, + "epoch": 0.14347079037800686, + "grad_norm": 1.7221592259263123, + "kl": 0.015533447265625, + "learning_rate": 9.282646048109966e-07, + "loss": 0.0006, + "reward": 1.6789227724075317, + "reward_std": 0.198103629052639, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6867353320121765, + "step": 167 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.0, + "epoch": 0.14432989690721648, + "grad_norm": 2.1528629212609407, + "kl": 0.013214111328125, + "learning_rate": 9.278350515463918e-07, + "loss": 0.0005, + "reward": 1.7893621921539307, + "reward_std": 0.13996141962707043, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7893621623516083, + "step": 168 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.7265625, + "epoch": 0.14518900343642613, + "grad_norm": 1.3004655134925518, + "kl": 0.0184326171875, + "learning_rate": 9.274054982817869e-07, + "loss": 0.0007, + "reward": 1.7796956300735474, + "reward_std": 0.14173657447099686, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7875081300735474, + "step": 169 + }, + { + "clip_ratio": 0.0, + "completion_length": 145.7578125, + "epoch": 0.14604810996563575, + "grad_norm": 1.4101337237690597, + "kl": 0.01959228515625, + "learning_rate": 9.269759450171821e-07, + "loss": 0.0008, + "reward": 1.8010075688362122, + "reward_std": 0.1661326214671135, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8010075390338898, + "step": 170 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.7265625, + "epoch": 0.14690721649484537, + "grad_norm": 1.4640575230347574, + "kl": 0.02215576171875, + "learning_rate": 9.265463917525773e-07, + "loss": 0.0009, + "reward": 1.6987681984901428, + "reward_std": 0.21677575260400772, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6987681686878204, + "step": 171 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.1171875, + "epoch": 0.14776632302405499, + "grad_norm": 1.6395435895829094, + "kl": 0.017822265625, + "learning_rate": 9.261168384879725e-07, + "loss": 0.0007, + "reward": 1.7570144534111023, + "reward_std": 0.14532551914453506, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7570143938064575, + "step": 172 + }, + { + "clip_ratio": 0.0, + "completion_length": 138.1796875, + "epoch": 0.1486254295532646, + "grad_norm": 1.696240174354143, + "kl": 0.0164794921875, + "learning_rate": 9.256872852233677e-07, + "loss": 0.0007, + "reward": 1.8731423616409302, + "reward_std": 0.14208180457353592, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8731423616409302, + "step": 173 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.1875, + "epoch": 0.14948453608247422, + "grad_norm": 1.3562918586715518, + "kl": 0.01910400390625, + "learning_rate": 9.252577319587629e-07, + "loss": 0.0008, + "reward": 1.7502126693725586, + "reward_std": 0.1445324867963791, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.758025199174881, + "step": 174 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.34375, + "epoch": 0.15034364261168384, + "grad_norm": 1.8285091498658437, + "kl": 0.01953125, + "learning_rate": 9.248281786941581e-07, + "loss": 0.0008, + "reward": 1.7030513286590576, + "reward_std": 0.20504559576511383, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7108637988567352, + "step": 175 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.4375, + "epoch": 0.15120274914089346, + "grad_norm": 2.1424066584845747, + "kl": 0.0216064453125, + "learning_rate": 9.243986254295533e-07, + "loss": 0.0009, + "reward": 1.6664610505104065, + "reward_std": 0.07924733310937881, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6664610505104065, + "step": 176 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.1484375, + "epoch": 0.15206185567010308, + "grad_norm": 1.8341695343755187, + "kl": 0.021728515625, + "learning_rate": 9.239690721649484e-07, + "loss": 0.0009, + "reward": 1.701620638370514, + "reward_std": 0.135139562189579, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7016206681728363, + "step": 177 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.5703125, + "epoch": 0.15292096219931273, + "grad_norm": 1.7560417473294632, + "kl": 0.02288818359375, + "learning_rate": 9.235395189003436e-07, + "loss": 0.0009, + "reward": 1.6936078667640686, + "reward_std": 0.20969898253679276, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.717045396566391, + "step": 178 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.453125, + "epoch": 0.15378006872852235, + "grad_norm": 1.66239183509703, + "kl": 0.02362060546875, + "learning_rate": 9.231099656357388e-07, + "loss": 0.0009, + "reward": 1.72810560464859, + "reward_std": 0.174056738615036, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7515431046485901, + "step": 179 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.2265625, + "epoch": 0.15463917525773196, + "grad_norm": 1.3638203436692424, + "kl": 0.0224609375, + "learning_rate": 9.22680412371134e-07, + "loss": 0.0009, + "reward": 1.7772685289382935, + "reward_std": 0.09648095071315765, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7850809693336487, + "step": 180 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.546875, + "epoch": 0.15549828178694158, + "grad_norm": 1.2134969783708147, + "kl": 0.02392578125, + "learning_rate": 9.222508591065292e-07, + "loss": 0.001, + "reward": 1.826979637145996, + "reward_std": 0.1237468272447586, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8347920775413513, + "step": 181 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.7890625, + "epoch": 0.1563573883161512, + "grad_norm": 1.3458969430241254, + "kl": 0.02227783203125, + "learning_rate": 9.218213058419243e-07, + "loss": 0.0009, + "reward": 1.7129201889038086, + "reward_std": 0.18879132717847824, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7285451889038086, + "step": 182 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.3359375, + "epoch": 0.15721649484536082, + "grad_norm": 1.750103363285581, + "kl": 0.06610107421875, + "learning_rate": 9.213917525773195e-07, + "loss": 0.0026, + "reward": 1.6476903557777405, + "reward_std": 0.265672467648983, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.67894047498703, + "step": 183 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.0078125, + "epoch": 0.15807560137457044, + "grad_norm": 1.820111828776988, + "kl": 0.02685546875, + "learning_rate": 9.209621993127147e-07, + "loss": 0.0011, + "reward": 1.774660885334015, + "reward_std": 0.15566430985927582, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7824733853340149, + "step": 184 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.703125, + "epoch": 0.15893470790378006, + "grad_norm": 1.6209110696010172, + "kl": 0.0299072265625, + "learning_rate": 9.205326460481098e-07, + "loss": 0.0012, + "reward": 1.7323817014694214, + "reward_std": 0.20174291729927063, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7636317014694214, + "step": 185 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.859375, + "epoch": 0.15979381443298968, + "grad_norm": 1.1877258799941435, + "kl": 0.02227783203125, + "learning_rate": 9.20103092783505e-07, + "loss": 0.0009, + "reward": 1.7020468711853027, + "reward_std": 0.14912591874599457, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7098594009876251, + "step": 186 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.25, + "epoch": 0.16065292096219932, + "grad_norm": 1.7483290111833802, + "kl": 0.02362060546875, + "learning_rate": 9.196735395189002e-07, + "loss": 0.0009, + "reward": 1.7843592166900635, + "reward_std": 0.18126945197582245, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7999841570854187, + "step": 187 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.625, + "epoch": 0.16151202749140894, + "grad_norm": 1.6918407108780025, + "kl": 0.02398681640625, + "learning_rate": 9.192439862542954e-07, + "loss": 0.001, + "reward": 1.70003342628479, + "reward_std": 0.15035928413271904, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7156584560871124, + "step": 188 + }, + { + "clip_ratio": 0.0, + "completion_length": 162.4609375, + "epoch": 0.16237113402061856, + "grad_norm": 1.3068095870355032, + "kl": 0.0291748046875, + "learning_rate": 9.188144329896906e-07, + "loss": 0.0012, + "reward": 1.6980275511741638, + "reward_std": 0.14009612798690796, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7058400511741638, + "step": 189 + }, + { + "clip_ratio": 0.0, + "completion_length": 149.5625, + "epoch": 0.16323024054982818, + "grad_norm": 5.9784870187151045, + "kl": 0.2835693359375, + "learning_rate": 9.183848797250858e-07, + "loss": 0.0113, + "reward": 1.5892316102981567, + "reward_std": 0.09760742634534836, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.604856550693512, + "step": 190 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.7734375, + "epoch": 0.1640893470790378, + "grad_norm": 1.2813901114072874, + "kl": 0.0228271484375, + "learning_rate": 9.17955326460481e-07, + "loss": 0.0009, + "reward": 1.6868805289268494, + "reward_std": 0.18260416388511658, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7025054693222046, + "step": 191 + }, + { + "clip_ratio": 0.0, + "completion_length": 142.046875, + "epoch": 0.16494845360824742, + "grad_norm": 6.416179300212472, + "kl": 0.02294921875, + "learning_rate": 9.175257731958762e-07, + "loss": 0.0009, + "reward": 1.6928731799125671, + "reward_std": 0.2074473798274994, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7241232395172119, + "step": 192 + }, + { + "clip_ratio": 0.0, + "completion_length": 157.703125, + "epoch": 0.16580756013745704, + "grad_norm": 2.6639805313439546, + "kl": 0.0198974609375, + "learning_rate": 9.170962199312713e-07, + "loss": 0.0008, + "reward": 1.7725132703781128, + "reward_std": 0.16904744878411293, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7881382405757904, + "step": 193 + }, + { + "clip_ratio": 0.0, + "completion_length": 150.21875, + "epoch": 0.16666666666666666, + "grad_norm": 2.6627936333789113, + "kl": 0.0616455078125, + "learning_rate": 9.166666666666665e-07, + "loss": 0.0025, + "reward": 1.726097822189331, + "reward_std": 0.17287509143352509, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7260977625846863, + "step": 194 + }, + { + "clip_ratio": 0.0, + "completion_length": 134.6171875, + "epoch": 0.16752577319587628, + "grad_norm": 2.3427841091043904, + "kl": 0.0228271484375, + "learning_rate": 9.162371134020618e-07, + "loss": 0.0009, + "reward": 1.6643958687782288, + "reward_std": 0.16956821829080582, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6722084581851959, + "step": 195 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.9765625, + "epoch": 0.16838487972508592, + "grad_norm": 1.2879832569815648, + "kl": 0.02276611328125, + "learning_rate": 9.15807560137457e-07, + "loss": 0.0009, + "reward": 1.7251017093658447, + "reward_std": 0.19944577664136887, + "rewards/format_reward_gen": 0.9609375, + "rewards/llm_reward": 0.7641642093658447, + "step": 196 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.2890625, + "epoch": 0.16924398625429554, + "grad_norm": 1.302637581591733, + "kl": 0.0181884765625, + "learning_rate": 9.153780068728522e-07, + "loss": 0.0007, + "reward": 1.7328863739967346, + "reward_std": 0.170260988175869, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7485113739967346, + "step": 197 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.390625, + "epoch": 0.17010309278350516, + "grad_norm": 1.185620411258173, + "kl": 0.0184326171875, + "learning_rate": 9.149484536082474e-07, + "loss": 0.0007, + "reward": 1.7054132223129272, + "reward_std": 0.14447502046823502, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7054132521152496, + "step": 198 + }, + { + "clip_ratio": 0.0, + "completion_length": 150.4296875, + "epoch": 0.17096219931271478, + "grad_norm": 1.7985827377480441, + "kl": 0.02508544921875, + "learning_rate": 9.145189003436426e-07, + "loss": 0.001, + "reward": 1.7419326901435852, + "reward_std": 0.10603855177760124, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7497451901435852, + "step": 199 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.734375, + "epoch": 0.1718213058419244, + "grad_norm": 1.4591627868559685, + "kl": 0.01800537109375, + "learning_rate": 9.140893470790378e-07, + "loss": 0.0007, + "reward": 1.7669562697410583, + "reward_std": 0.16979996114969254, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7747687101364136, + "step": 200 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.765625, + "epoch": 0.17268041237113402, + "grad_norm": 1.7341999103239758, + "kl": 0.02862548828125, + "learning_rate": 9.13659793814433e-07, + "loss": 0.0011, + "reward": 1.6440722942352295, + "reward_std": 0.13084383308887482, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6518846750259399, + "step": 201 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.2734375, + "epoch": 0.17353951890034364, + "grad_norm": 1.6909406413369592, + "kl": 0.01934814453125, + "learning_rate": 9.132302405498281e-07, + "loss": 0.0008, + "reward": 1.8257556557655334, + "reward_std": 0.1285765990614891, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8257556557655334, + "step": 202 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.515625, + "epoch": 0.17439862542955326, + "grad_norm": 0.9710832816221044, + "kl": 0.01922607421875, + "learning_rate": 9.128006872852233e-07, + "loss": 0.0008, + "reward": 1.7562236189842224, + "reward_std": 0.1861056238412857, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7796610593795776, + "step": 203 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.4609375, + "epoch": 0.17525773195876287, + "grad_norm": 1.5930942457393673, + "kl": 0.01959228515625, + "learning_rate": 9.123711340206185e-07, + "loss": 0.0008, + "reward": 1.7354264855384827, + "reward_std": 0.14093422889709473, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.743239015340805, + "step": 204 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.8828125, + "epoch": 0.17611683848797252, + "grad_norm": 1.1786198725658004, + "kl": 0.02020263671875, + "learning_rate": 9.119415807560137e-07, + "loss": 0.0008, + "reward": 1.8254988193511963, + "reward_std": 0.13964856415987015, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8254987895488739, + "step": 205 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.5, + "epoch": 0.17697594501718214, + "grad_norm": 1.561736876348039, + "kl": 0.03765869140625, + "learning_rate": 9.115120274914089e-07, + "loss": 0.0015, + "reward": 1.7375341653823853, + "reward_std": 0.16468166559934616, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7531591653823853, + "step": 206 + }, + { + "clip_ratio": 0.0, + "completion_length": 147.421875, + "epoch": 0.17783505154639176, + "grad_norm": 1.673938587392466, + "kl": 0.02197265625, + "learning_rate": 9.110824742268041e-07, + "loss": 0.0009, + "reward": 1.828542172908783, + "reward_std": 0.12264525145292282, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.844167172908783, + "step": 207 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.1171875, + "epoch": 0.17869415807560138, + "grad_norm": 1.7407490513436423, + "kl": 0.02032470703125, + "learning_rate": 9.106529209621993e-07, + "loss": 0.0008, + "reward": 1.7809680104255676, + "reward_std": 0.13108700513839722, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7809680104255676, + "step": 208 + }, + { + "clip_ratio": 0.0, + "completion_length": 120.921875, + "epoch": 0.179553264604811, + "grad_norm": 1.2574262300280894, + "kl": 0.028564453125, + "learning_rate": 9.102233676975945e-07, + "loss": 0.0011, + "reward": 1.7650489807128906, + "reward_std": 0.11492755264043808, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7728614807128906, + "step": 209 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.7578125, + "epoch": 0.18041237113402062, + "grad_norm": 1.803569017181721, + "kl": 0.01959228515625, + "learning_rate": 9.097938144329897e-07, + "loss": 0.0008, + "reward": 1.685543417930603, + "reward_std": 0.13148176670074463, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.693355917930603, + "step": 210 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.7265625, + "epoch": 0.18127147766323023, + "grad_norm": 1.8750450940771943, + "kl": 0.02978515625, + "learning_rate": 9.093642611683848e-07, + "loss": 0.0012, + "reward": 1.723173975944519, + "reward_std": 0.21225081384181976, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7309865355491638, + "step": 211 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.6640625, + "epoch": 0.18213058419243985, + "grad_norm": 1.8697489802004137, + "kl": 0.0194091796875, + "learning_rate": 9.0893470790378e-07, + "loss": 0.0008, + "reward": 1.773992896080017, + "reward_std": 0.10730047896504402, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7818053960800171, + "step": 212 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.6328125, + "epoch": 0.18298969072164947, + "grad_norm": 1.1778709789513797, + "kl": 0.0174560546875, + "learning_rate": 9.085051546391752e-07, + "loss": 0.0007, + "reward": 1.7802048325538635, + "reward_std": 0.1343718022108078, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7802048325538635, + "step": 213 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.171875, + "epoch": 0.18384879725085912, + "grad_norm": 1.6555684252295757, + "kl": 0.0272216796875, + "learning_rate": 9.080756013745704e-07, + "loss": 0.0011, + "reward": 1.6995335221290588, + "reward_std": 0.26308566331863403, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7229710221290588, + "step": 214 + }, + { + "clip_ratio": 0.0, + "completion_length": 139.921875, + "epoch": 0.18470790378006874, + "grad_norm": 1.2357582827778086, + "kl": 0.0206298828125, + "learning_rate": 9.076460481099656e-07, + "loss": 0.0008, + "reward": 1.6951950788497925, + "reward_std": 0.1771727241575718, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7186326384544373, + "step": 215 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.15625, + "epoch": 0.18556701030927836, + "grad_norm": 1.7499046552420514, + "kl": 0.02349853515625, + "learning_rate": 9.072164948453608e-07, + "loss": 0.0009, + "reward": 1.826527178287506, + "reward_std": 0.10365994274616241, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8343397080898285, + "step": 216 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.9609375, + "epoch": 0.18642611683848798, + "grad_norm": 1.8005132756117428, + "kl": 0.0301513671875, + "learning_rate": 9.06786941580756e-07, + "loss": 0.0012, + "reward": 1.746228814125061, + "reward_std": 0.20855771750211716, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7540413737297058, + "step": 217 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.3359375, + "epoch": 0.1872852233676976, + "grad_norm": 1.9030503926874742, + "kl": 0.0220947265625, + "learning_rate": 9.063573883161512e-07, + "loss": 0.0009, + "reward": 1.7833629846572876, + "reward_std": 0.12160376086831093, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7833629846572876, + "step": 218 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.546875, + "epoch": 0.18814432989690721, + "grad_norm": 2.346663613038113, + "kl": 0.02880859375, + "learning_rate": 9.059278350515463e-07, + "loss": 0.0012, + "reward": 1.810929536819458, + "reward_std": 0.12845928221940994, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8187420070171356, + "step": 219 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.4140625, + "epoch": 0.18900343642611683, + "grad_norm": 1.951232732702119, + "kl": 0.0296630859375, + "learning_rate": 9.054982817869415e-07, + "loss": 0.0012, + "reward": 1.7572931051254272, + "reward_std": 0.2043527588248253, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7807306349277496, + "step": 220 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.3671875, + "epoch": 0.18986254295532645, + "grad_norm": 1.4773283463402496, + "kl": 0.02398681640625, + "learning_rate": 9.050687285223367e-07, + "loss": 0.001, + "reward": 1.8425282835960388, + "reward_std": 0.09052041545510292, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8503407537937164, + "step": 221 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.546875, + "epoch": 0.19072164948453607, + "grad_norm": 1.6430837871831834, + "kl": 0.0224609375, + "learning_rate": 9.046391752577319e-07, + "loss": 0.0009, + "reward": 1.6133168935775757, + "reward_std": 0.1693805679678917, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.6367543637752533, + "step": 222 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.3359375, + "epoch": 0.19158075601374572, + "grad_norm": 1.5498241580052614, + "kl": 0.028076171875, + "learning_rate": 9.042096219931271e-07, + "loss": 0.0011, + "reward": 1.832928478717804, + "reward_std": 0.10157027468085289, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8407409191131592, + "step": 223 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.3828125, + "epoch": 0.19243986254295534, + "grad_norm": 1.427575606765542, + "kl": 0.02471923828125, + "learning_rate": 9.037800687285223e-07, + "loss": 0.001, + "reward": 1.7951682806015015, + "reward_std": 0.11901043727993965, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7951683104038239, + "step": 224 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.5234375, + "epoch": 0.19329896907216496, + "grad_norm": 1.745841693534615, + "kl": 0.05950927734375, + "learning_rate": 9.033505154639175e-07, + "loss": 0.0024, + "reward": 1.7870480418205261, + "reward_std": 0.1087818406522274, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.8104856014251709, + "step": 225 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.015625, + "epoch": 0.19415807560137457, + "grad_norm": 1.0774096537668199, + "kl": 0.022216796875, + "learning_rate": 9.029209621993127e-07, + "loss": 0.0009, + "reward": 1.8171106576919556, + "reward_std": 0.10428809374570847, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.824923187494278, + "step": 226 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.5234375, + "epoch": 0.1950171821305842, + "grad_norm": 2.0901239612312854, + "kl": 0.02642822265625, + "learning_rate": 9.024914089347078e-07, + "loss": 0.0011, + "reward": 1.7465097308158875, + "reward_std": 0.14316842332482338, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7465097308158875, + "step": 227 + }, + { + "clip_ratio": 0.0, + "completion_length": 138.8125, + "epoch": 0.1958762886597938, + "grad_norm": 1.3420343738640794, + "kl": 0.02850341796875, + "learning_rate": 9.02061855670103e-07, + "loss": 0.0011, + "reward": 1.7858530282974243, + "reward_std": 0.10892975702881813, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7936655879020691, + "step": 228 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.6484375, + "epoch": 0.19673539518900343, + "grad_norm": 1.2697191544586062, + "kl": 0.03155517578125, + "learning_rate": 9.016323024054982e-07, + "loss": 0.0013, + "reward": 1.7829349040985107, + "reward_std": 0.17472659796476364, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7985599339008331, + "step": 229 + }, + { + "clip_ratio": 0.0, + "completion_length": 148.53125, + "epoch": 0.19759450171821305, + "grad_norm": 1.2098132776705879, + "kl": 0.02154541015625, + "learning_rate": 9.012027491408934e-07, + "loss": 0.0009, + "reward": 1.8274175524711609, + "reward_std": 0.1526181548833847, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8430425822734833, + "step": 230 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.6015625, + "epoch": 0.19845360824742267, + "grad_norm": 1.3150045930691368, + "kl": 0.0223388671875, + "learning_rate": 9.007731958762886e-07, + "loss": 0.0009, + "reward": 1.7271833419799805, + "reward_std": 0.1835765838623047, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7428083121776581, + "step": 231 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.7578125, + "epoch": 0.19931271477663232, + "grad_norm": 1.1993647390638915, + "kl": 0.031494140625, + "learning_rate": 9.003436426116838e-07, + "loss": 0.0013, + "reward": 1.7216781377792358, + "reward_std": 0.14081217721104622, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7373031079769135, + "step": 232 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.8984375, + "epoch": 0.20017182130584193, + "grad_norm": 0.731243698465931, + "kl": 0.02880859375, + "learning_rate": 8.99914089347079e-07, + "loss": 0.0012, + "reward": 1.7585216164588928, + "reward_std": 0.07797488383948803, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7663341164588928, + "step": 233 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.171875, + "epoch": 0.20103092783505155, + "grad_norm": 1.4749482594585495, + "kl": 0.021240234375, + "learning_rate": 8.994845360824742e-07, + "loss": 0.0009, + "reward": 1.7960580587387085, + "reward_std": 0.15863198041915894, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8116830587387085, + "step": 234 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.21875, + "epoch": 0.20189003436426117, + "grad_norm": 1.773680287142342, + "kl": 0.0177001953125, + "learning_rate": 8.990549828178694e-07, + "loss": 0.0007, + "reward": 1.7810790538787842, + "reward_std": 0.13028161227703094, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7810790240764618, + "step": 235 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.6328125, + "epoch": 0.2027491408934708, + "grad_norm": 8.259511454922954, + "kl": 0.0208740234375, + "learning_rate": 8.986254295532645e-07, + "loss": 0.0008, + "reward": 1.8064086437225342, + "reward_std": 0.12397770583629608, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8064086735248566, + "step": 236 + }, + { + "clip_ratio": 0.0, + "completion_length": 147.7890625, + "epoch": 0.2036082474226804, + "grad_norm": 1.308723092707521, + "kl": 0.02813720703125, + "learning_rate": 8.981958762886598e-07, + "loss": 0.0011, + "reward": 1.7656047344207764, + "reward_std": 0.11740110442042351, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7656047642230988, + "step": 237 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.9296875, + "epoch": 0.20446735395189003, + "grad_norm": 1.4015170970241377, + "kl": 0.02276611328125, + "learning_rate": 8.97766323024055e-07, + "loss": 0.0009, + "reward": 1.7170850038528442, + "reward_std": 0.08975991420447826, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7170850038528442, + "step": 238 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.921875, + "epoch": 0.20532646048109965, + "grad_norm": 1.7165031378836524, + "kl": 0.031982421875, + "learning_rate": 8.973367697594502e-07, + "loss": 0.0013, + "reward": 1.714181363582611, + "reward_std": 0.1387975737452507, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7298063337802887, + "step": 239 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.7734375, + "epoch": 0.20618556701030927, + "grad_norm": 1.1986523220206222, + "kl": 0.021484375, + "learning_rate": 8.969072164948454e-07, + "loss": 0.0009, + "reward": 1.70902019739151, + "reward_std": 0.13189618289470673, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7090202271938324, + "step": 240 + }, + { + "clip_ratio": 0.0, + "completion_length": 148.0859375, + "epoch": 0.20704467353951891, + "grad_norm": 1.4338358835989133, + "kl": 0.029052734375, + "learning_rate": 8.964776632302406e-07, + "loss": 0.0012, + "reward": 1.7179991602897644, + "reward_std": 0.16914209723472595, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7492491602897644, + "step": 241 + }, + { + "clip_ratio": 0.0, + "completion_length": 142.640625, + "epoch": 0.20790378006872853, + "grad_norm": 0.8698039683331652, + "kl": 0.03466796875, + "learning_rate": 8.960481099656358e-07, + "loss": 0.0014, + "reward": 1.817783772945404, + "reward_std": 0.08906146325170994, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.825596272945404, + "step": 242 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.46875, + "epoch": 0.20876288659793815, + "grad_norm": 3.260349200890088, + "kl": 0.02471923828125, + "learning_rate": 8.95618556701031e-07, + "loss": 0.001, + "reward": 1.698820948600769, + "reward_std": 0.1781657487154007, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7066334784030914, + "step": 243 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.1796875, + "epoch": 0.20962199312714777, + "grad_norm": 1.0734369880505024, + "kl": 0.02301025390625, + "learning_rate": 8.951890034364261e-07, + "loss": 0.0009, + "reward": 1.7488206624984741, + "reward_std": 0.1078120581805706, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7566331923007965, + "step": 244 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.4609375, + "epoch": 0.2104810996563574, + "grad_norm": 1.230920345198539, + "kl": 0.03179931640625, + "learning_rate": 8.947594501718213e-07, + "loss": 0.0013, + "reward": 1.7286466360092163, + "reward_std": 0.16450222581624985, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7598966360092163, + "step": 245 + }, + { + "clip_ratio": 0.0, + "completion_length": 147.265625, + "epoch": 0.211340206185567, + "grad_norm": 4.523859706146446, + "kl": 0.0262451171875, + "learning_rate": 8.943298969072165e-07, + "loss": 0.0011, + "reward": 1.763322114944458, + "reward_std": 0.1752321869134903, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7867595553398132, + "step": 246 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.8359375, + "epoch": 0.21219931271477663, + "grad_norm": 1.363046353001964, + "kl": 0.02996826171875, + "learning_rate": 8.939003436426117e-07, + "loss": 0.0012, + "reward": 1.734560787677765, + "reward_std": 0.17826475948095322, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7423732876777649, + "step": 247 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.1015625, + "epoch": 0.21305841924398625, + "grad_norm": 1.2016846191079753, + "kl": 0.02099609375, + "learning_rate": 8.934707903780069e-07, + "loss": 0.0008, + "reward": 1.8029004335403442, + "reward_std": 0.15353290364146233, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8107129037380219, + "step": 248 + }, + { + "clip_ratio": 0.0, + "completion_length": 134.953125, + "epoch": 0.21391752577319587, + "grad_norm": 1.6472740449517276, + "kl": 0.03009033203125, + "learning_rate": 8.930412371134021e-07, + "loss": 0.0012, + "reward": 1.728321135044098, + "reward_std": 0.17561784386634827, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7439461648464203, + "step": 249 + }, + { + "clip_ratio": 0.0, + "completion_length": 114.2578125, + "epoch": 0.21477663230240548, + "grad_norm": 1.3208424429966092, + "kl": 0.0274658203125, + "learning_rate": 8.926116838487973e-07, + "loss": 0.0011, + "reward": 1.7464920282363892, + "reward_std": 0.1454971358180046, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7543044984340668, + "step": 250 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.8984375, + "epoch": 0.21563573883161513, + "grad_norm": 1.7163264390417992, + "kl": 0.02838134765625, + "learning_rate": 8.921821305841925e-07, + "loss": 0.0011, + "reward": 1.8135504722595215, + "reward_std": 0.14255821704864502, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8135504722595215, + "step": 251 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.46875, + "epoch": 0.21649484536082475, + "grad_norm": 1.4660984975320737, + "kl": 0.0269775390625, + "learning_rate": 8.917525773195877e-07, + "loss": 0.0011, + "reward": 1.6587898135185242, + "reward_std": 0.16928455978631973, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6666022539138794, + "step": 252 + }, + { + "clip_ratio": 0.0, + "completion_length": 138.2890625, + "epoch": 0.21735395189003437, + "grad_norm": 1.4026200377012226, + "kl": 0.02838134765625, + "learning_rate": 8.913230240549828e-07, + "loss": 0.0011, + "reward": 1.7479596138000488, + "reward_std": 0.09517625346779823, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7479596734046936, + "step": 253 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.734375, + "epoch": 0.218213058419244, + "grad_norm": 1.8900236317096133, + "kl": 0.03192138671875, + "learning_rate": 8.90893470790378e-07, + "loss": 0.0013, + "reward": 1.7791557312011719, + "reward_std": 0.08186223357915878, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7791557610034943, + "step": 254 + }, + { + "clip_ratio": 0.0, + "completion_length": 121.0234375, + "epoch": 0.2190721649484536, + "grad_norm": 1.348548755398183, + "kl": 0.035400390625, + "learning_rate": 8.904639175257731e-07, + "loss": 0.0014, + "reward": 1.762001395225525, + "reward_std": 0.10934092476963997, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7698139548301697, + "step": 255 + }, + { + "clip_ratio": 0.0, + "completion_length": 157.3359375, + "epoch": 0.21993127147766323, + "grad_norm": 1.4276465496185335, + "kl": 0.0269775390625, + "learning_rate": 8.900343642611683e-07, + "loss": 0.0011, + "reward": 1.675103783607483, + "reward_std": 0.21139457076787949, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7063537836074829, + "step": 256 + }, + { + "clip_ratio": 0.0, + "completion_length": 120.515625, + "epoch": 0.22079037800687284, + "grad_norm": 1.133588870905801, + "kl": 0.03448486328125, + "learning_rate": 8.896048109965635e-07, + "loss": 0.0014, + "reward": 1.71183180809021, + "reward_std": 0.10940879955887794, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7118318676948547, + "step": 257 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.203125, + "epoch": 0.22164948453608246, + "grad_norm": 0.8368892953167869, + "kl": 0.0330810546875, + "learning_rate": 8.891752577319587e-07, + "loss": 0.0013, + "reward": 1.8508310914039612, + "reward_std": 0.07327214255928993, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8508311212062836, + "step": 258 + }, + { + "clip_ratio": 0.0, + "completion_length": 144.3359375, + "epoch": 0.22250859106529208, + "grad_norm": 1.006775443305009, + "kl": 0.0328369140625, + "learning_rate": 8.887457044673539e-07, + "loss": 0.0013, + "reward": 1.8622272610664368, + "reward_std": 0.1266963928937912, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8778522610664368, + "step": 259 + }, + { + "clip_ratio": 0.0, + "completion_length": 120.421875, + "epoch": 0.22336769759450173, + "grad_norm": 0.9631829361222369, + "kl": 0.0396728515625, + "learning_rate": 8.88316151202749e-07, + "loss": 0.0016, + "reward": 1.8064301013946533, + "reward_std": 0.11555318906903267, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8142426609992981, + "step": 260 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.375, + "epoch": 0.22422680412371135, + "grad_norm": 1.5467830446043906, + "kl": 0.03759765625, + "learning_rate": 8.878865979381442e-07, + "loss": 0.0015, + "reward": 1.7076144218444824, + "reward_std": 0.16983914375305176, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7154269218444824, + "step": 261 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.078125, + "epoch": 0.22508591065292097, + "grad_norm": 1.385570288599824, + "kl": 0.0396728515625, + "learning_rate": 8.874570446735394e-07, + "loss": 0.0016, + "reward": 1.727626919746399, + "reward_std": 0.15141097083687782, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7432518899440765, + "step": 262 + }, + { + "clip_ratio": 0.0, + "completion_length": 144.109375, + "epoch": 0.2259450171821306, + "grad_norm": 1.0781029500650152, + "kl": 0.03558349609375, + "learning_rate": 8.870274914089346e-07, + "loss": 0.0014, + "reward": 1.8266533613204956, + "reward_std": 0.13985048979520798, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8344658315181732, + "step": 263 + }, + { + "clip_ratio": 0.0, + "completion_length": 122.7109375, + "epoch": 0.2268041237113402, + "grad_norm": 1.5051081251639746, + "kl": 0.04119873046875, + "learning_rate": 8.865979381443298e-07, + "loss": 0.0016, + "reward": 1.9263587594032288, + "reward_std": 0.06610674690455198, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.9263588190078735, + "step": 264 + }, + { + "clip_ratio": 0.0, + "completion_length": 114.953125, + "epoch": 0.22766323024054982, + "grad_norm": 1.2568559492969402, + "kl": 0.043212890625, + "learning_rate": 8.86168384879725e-07, + "loss": 0.0017, + "reward": 1.7441568970680237, + "reward_std": 0.052992574870586395, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7441569268703461, + "step": 265 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.8203125, + "epoch": 0.22852233676975944, + "grad_norm": 12.957475206253665, + "kl": 0.03289794921875, + "learning_rate": 8.857388316151202e-07, + "loss": 0.0013, + "reward": 1.7724300026893616, + "reward_std": 0.12845268845558167, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.780242532491684, + "step": 266 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.46875, + "epoch": 0.22938144329896906, + "grad_norm": 4.112320049307154, + "kl": 0.030517578125, + "learning_rate": 8.853092783505154e-07, + "loss": 0.0012, + "reward": 1.7406333684921265, + "reward_std": 0.16561638191342354, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7484458982944489, + "step": 267 + }, + { + "clip_ratio": 0.0, + "completion_length": 143.0859375, + "epoch": 0.23024054982817868, + "grad_norm": 1.3042211470111829, + "kl": 0.039306640625, + "learning_rate": 8.848797250859106e-07, + "loss": 0.0016, + "reward": 1.7143108248710632, + "reward_std": 0.13581188768148422, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7377482950687408, + "step": 268 + }, + { + "clip_ratio": 0.0, + "completion_length": 150.8828125, + "epoch": 0.23109965635738833, + "grad_norm": 1.928604268090887, + "kl": 0.037353515625, + "learning_rate": 8.844501718213057e-07, + "loss": 0.0015, + "reward": 1.8002280592918396, + "reward_std": 0.15524645149707794, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8080405294895172, + "step": 269 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.171875, + "epoch": 0.23195876288659795, + "grad_norm": 5.27894758100115, + "kl": 0.02862548828125, + "learning_rate": 8.840206185567009e-07, + "loss": 0.0011, + "reward": 1.8452274799346924, + "reward_std": 0.15420031547546387, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8530399203300476, + "step": 270 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.671875, + "epoch": 0.23281786941580757, + "grad_norm": 1.306834060633908, + "kl": 0.0325927734375, + "learning_rate": 8.835910652920961e-07, + "loss": 0.0013, + "reward": 1.6935831308364868, + "reward_std": 0.154945969581604, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7092081606388092, + "step": 271 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.8125, + "epoch": 0.23367697594501718, + "grad_norm": 1.2910579810108835, + "kl": 0.03216552734375, + "learning_rate": 8.831615120274913e-07, + "loss": 0.0013, + "reward": 1.785846471786499, + "reward_std": 0.08592062070965767, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7936590611934662, + "step": 272 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.4765625, + "epoch": 0.2345360824742268, + "grad_norm": 1.6016973929619667, + "kl": 0.0333251953125, + "learning_rate": 8.827319587628865e-07, + "loss": 0.0013, + "reward": 1.6971846222877502, + "reward_std": 0.1201891340315342, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7049971222877502, + "step": 273 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.0703125, + "epoch": 0.23539518900343642, + "grad_norm": 1.357010665541966, + "kl": 0.03564453125, + "learning_rate": 8.823024054982817e-07, + "loss": 0.0014, + "reward": 1.7180689573287964, + "reward_std": 0.19224438071250916, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7415064871311188, + "step": 274 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.0859375, + "epoch": 0.23625429553264604, + "grad_norm": 1.1159550961761515, + "kl": 0.03515625, + "learning_rate": 8.818728522336769e-07, + "loss": 0.0014, + "reward": 1.7949897646903992, + "reward_std": 0.14928951114416122, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.8184272944927216, + "step": 275 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.6328125, + "epoch": 0.23711340206185566, + "grad_norm": 3.5515611265357485, + "kl": 0.02630615234375, + "learning_rate": 8.814432989690721e-07, + "loss": 0.0011, + "reward": 1.7773825526237488, + "reward_std": 0.10580384358763695, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7773825824260712, + "step": 276 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.4140625, + "epoch": 0.23797250859106528, + "grad_norm": 1.5580462509961617, + "kl": 0.03564453125, + "learning_rate": 8.810137457044672e-07, + "loss": 0.0014, + "reward": 1.7888085842132568, + "reward_std": 0.21199411898851395, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.8122460544109344, + "step": 277 + }, + { + "clip_ratio": 0.0, + "completion_length": 154.171875, + "epoch": 0.23883161512027493, + "grad_norm": 1.0543303085586277, + "kl": 0.02984619140625, + "learning_rate": 8.805841924398624e-07, + "loss": 0.0012, + "reward": 1.8188990950584412, + "reward_std": 0.13441307097673416, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.8423365950584412, + "step": 278 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.8984375, + "epoch": 0.23969072164948454, + "grad_norm": 1.5265695805128905, + "kl": 0.02557373046875, + "learning_rate": 8.801546391752576e-07, + "loss": 0.001, + "reward": 1.7924699783325195, + "reward_std": 0.16526872664690018, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8002825081348419, + "step": 279 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.8671875, + "epoch": 0.24054982817869416, + "grad_norm": 1.1812679545358764, + "kl": 0.0225830078125, + "learning_rate": 8.797250859106528e-07, + "loss": 0.0009, + "reward": 1.827331304550171, + "reward_std": 0.10759979486465454, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8273313045501709, + "step": 280 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.28125, + "epoch": 0.24140893470790378, + "grad_norm": 1.5991468981516737, + "kl": 0.0291748046875, + "learning_rate": 8.792955326460481e-07, + "loss": 0.0012, + "reward": 1.7398195266723633, + "reward_std": 0.19154010713100433, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7476319670677185, + "step": 281 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.546875, + "epoch": 0.2422680412371134, + "grad_norm": 0.9116875920398242, + "kl": 0.0299072265625, + "learning_rate": 8.788659793814433e-07, + "loss": 0.0012, + "reward": 1.7467403411865234, + "reward_std": 0.08945905789732933, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7545528709888458, + "step": 282 + }, + { + "clip_ratio": 0.0, + "completion_length": 134.46875, + "epoch": 0.24312714776632302, + "grad_norm": 0.8380935360171634, + "kl": 0.0316162109375, + "learning_rate": 8.784364261168385e-07, + "loss": 0.0013, + "reward": 1.8638163208961487, + "reward_std": 0.09008487407118082, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8638162612915039, + "step": 283 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.296875, + "epoch": 0.24398625429553264, + "grad_norm": 0.9161141195578109, + "kl": 0.027587890625, + "learning_rate": 8.780068728522337e-07, + "loss": 0.0011, + "reward": 1.8011575937271118, + "reward_std": 0.0535787851549685, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8011575937271118, + "step": 284 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.3203125, + "epoch": 0.24484536082474226, + "grad_norm": 1.9166929431895137, + "kl": 0.024658203125, + "learning_rate": 8.775773195876289e-07, + "loss": 0.001, + "reward": 1.7418934106826782, + "reward_std": 0.14604221284389496, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7497059106826782, + "step": 285 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.2890625, + "epoch": 0.24570446735395188, + "grad_norm": 1.1252522693717517, + "kl": 0.02996826171875, + "learning_rate": 8.77147766323024e-07, + "loss": 0.0012, + "reward": 1.7983113527297974, + "reward_std": 0.09610910713672638, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.806123822927475, + "step": 286 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.921875, + "epoch": 0.24656357388316152, + "grad_norm": 1.3367048117093618, + "kl": 0.034912109375, + "learning_rate": 8.767182130584192e-07, + "loss": 0.0014, + "reward": 1.7578275799751282, + "reward_std": 0.14203480631113052, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.765640139579773, + "step": 287 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.625, + "epoch": 0.24742268041237114, + "grad_norm": 0.724502338816319, + "kl": 0.03143310546875, + "learning_rate": 8.762886597938144e-07, + "loss": 0.0013, + "reward": 1.7635814547538757, + "reward_std": 0.0678582051768899, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7635815143585205, + "step": 288 + }, + { + "clip_ratio": 0.0, + "completion_length": 163.6953125, + "epoch": 0.24828178694158076, + "grad_norm": 1.1733316115040695, + "kl": 0.0283203125, + "learning_rate": 8.758591065292096e-07, + "loss": 0.0011, + "reward": 1.7535958290100098, + "reward_std": 0.16154304146766663, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7614083290100098, + "step": 289 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.8046875, + "epoch": 0.24914089347079038, + "grad_norm": 1.4167971747096428, + "kl": 0.0279541015625, + "learning_rate": 8.754295532646048e-07, + "loss": 0.0011, + "reward": 1.795049250125885, + "reward_std": 0.07856442406773567, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7950493097305298, + "step": 290 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.734375, + "epoch": 0.25, + "grad_norm": 3.1009628872255646, + "kl": 0.03045654296875, + "learning_rate": 8.75e-07, + "loss": 0.0012, + "reward": 1.8120030760765076, + "reward_std": 0.16107841953635216, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8276280760765076, + "step": 291 + }, + { + "clip_ratio": 0.0, + "completion_length": 134.7265625, + "epoch": 0.2508591065292096, + "grad_norm": 1.1395121387903864, + "kl": 0.02764892578125, + "learning_rate": 8.745704467353952e-07, + "loss": 0.0011, + "reward": 1.7605109214782715, + "reward_std": 0.093373142182827, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7683233916759491, + "step": 292 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.34375, + "epoch": 0.25171821305841924, + "grad_norm": 1.0703218519168713, + "kl": 0.03216552734375, + "learning_rate": 8.741408934707904e-07, + "loss": 0.0013, + "reward": 1.880930781364441, + "reward_std": 0.11160749942064285, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8887432813644409, + "step": 293 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.1484375, + "epoch": 0.25257731958762886, + "grad_norm": 3.1378356819250217, + "kl": 0.0274658203125, + "learning_rate": 8.737113402061856e-07, + "loss": 0.0011, + "reward": 1.8660714626312256, + "reward_std": 0.1355828121304512, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8738839626312256, + "step": 294 + }, + { + "clip_ratio": 0.0, + "completion_length": 121.90625, + "epoch": 0.2534364261168385, + "grad_norm": 0.9752935666515755, + "kl": 0.02984619140625, + "learning_rate": 8.732817869415807e-07, + "loss": 0.0012, + "reward": 1.8270512223243713, + "reward_std": 0.09455467015504837, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8348636627197266, + "step": 295 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.4140625, + "epoch": 0.2542955326460481, + "grad_norm": 2.5028271019973256, + "kl": 0.02728271484375, + "learning_rate": 8.728522336769759e-07, + "loss": 0.0011, + "reward": 1.81145840883255, + "reward_std": 0.1066664457321167, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8114584386348724, + "step": 296 + }, + { + "clip_ratio": 0.0, + "completion_length": 115.6171875, + "epoch": 0.2551546391752577, + "grad_norm": 1.9066759357991514, + "kl": 0.03387451171875, + "learning_rate": 8.724226804123711e-07, + "loss": 0.0014, + "reward": 1.7507621049880981, + "reward_std": 0.1560019999742508, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7585746049880981, + "step": 297 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.296875, + "epoch": 0.25601374570446733, + "grad_norm": 1.659392268817117, + "kl": 0.03125, + "learning_rate": 8.719931271477663e-07, + "loss": 0.0012, + "reward": 1.8714821338653564, + "reward_std": 0.10832902416586876, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8792945742607117, + "step": 298 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.0234375, + "epoch": 0.25687285223367695, + "grad_norm": 1.109700712202529, + "kl": 0.02984619140625, + "learning_rate": 8.715635738831615e-07, + "loss": 0.0012, + "reward": 1.8865798711776733, + "reward_std": 0.08204978704452515, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8865799009799957, + "step": 299 + }, + { + "clip_ratio": 0.0, + "completion_length": 121.015625, + "epoch": 0.25773195876288657, + "grad_norm": 1.0487633124419409, + "kl": 0.0244140625, + "learning_rate": 8.711340206185567e-07, + "loss": 0.001, + "reward": 1.809591829776764, + "reward_std": 0.15175417065620422, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.8330293297767639, + "step": 300 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.25, + "epoch": 0.25859106529209624, + "grad_norm": 1.5750530363933386, + "kl": 0.04327392578125, + "learning_rate": 8.707044673539519e-07, + "loss": 0.0017, + "reward": 1.792675495147705, + "reward_std": 0.08446579799056053, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8004880249500275, + "step": 301 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.0546875, + "epoch": 0.25945017182130586, + "grad_norm": 1.9949049568284047, + "kl": 0.026123046875, + "learning_rate": 8.702749140893471e-07, + "loss": 0.001, + "reward": 1.8067399859428406, + "reward_std": 0.14312193542718887, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8145524859428406, + "step": 302 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.796875, + "epoch": 0.2603092783505155, + "grad_norm": 1.7326491889574818, + "kl": 0.02447509765625, + "learning_rate": 8.698453608247422e-07, + "loss": 0.001, + "reward": 1.8493663668632507, + "reward_std": 0.06654036790132523, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8493663370609283, + "step": 303 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.4296875, + "epoch": 0.2611683848797251, + "grad_norm": 1.9340178331586866, + "kl": 0.0367431640625, + "learning_rate": 8.694158075601374e-07, + "loss": 0.0015, + "reward": 1.7990782260894775, + "reward_std": 0.14300024509429932, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7990781962871552, + "step": 304 + }, + { + "clip_ratio": 0.0, + "completion_length": 138.3046875, + "epoch": 0.2620274914089347, + "grad_norm": 1.6168654993721436, + "kl": 0.0328369140625, + "learning_rate": 8.689862542955326e-07, + "loss": 0.0013, + "reward": 1.7844675183296204, + "reward_std": 0.10683033242821693, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.784467488527298, + "step": 305 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.4921875, + "epoch": 0.26288659793814434, + "grad_norm": 1.3435959118517327, + "kl": 0.0362548828125, + "learning_rate": 8.685567010309278e-07, + "loss": 0.0015, + "reward": 1.7007364630699158, + "reward_std": 0.10049042850732803, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7007364630699158, + "step": 306 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.8359375, + "epoch": 0.26374570446735396, + "grad_norm": 1.7041209629618606, + "kl": 0.02471923828125, + "learning_rate": 8.68127147766323e-07, + "loss": 0.001, + "reward": 1.7297524213790894, + "reward_std": 0.11169169098138809, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7297524213790894, + "step": 307 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.1171875, + "epoch": 0.2646048109965636, + "grad_norm": 1.0611383574670885, + "kl": 0.0352783203125, + "learning_rate": 8.676975945017182e-07, + "loss": 0.0014, + "reward": 1.765690803527832, + "reward_std": 0.07691102847456932, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7656907737255096, + "step": 308 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.5, + "epoch": 0.2654639175257732, + "grad_norm": 1.810412989768572, + "kl": 0.061279296875, + "learning_rate": 8.672680412371134e-07, + "loss": 0.0024, + "reward": 1.6709439158439636, + "reward_std": 0.11246037483215332, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6709439158439636, + "step": 309 + }, + { + "clip_ratio": 0.0, + "completion_length": 138.1796875, + "epoch": 0.2663230240549828, + "grad_norm": 3.183250415302431, + "kl": 0.03106689453125, + "learning_rate": 8.668384879725086e-07, + "loss": 0.0012, + "reward": 1.7835421562194824, + "reward_std": 0.08028195053339005, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7835421562194824, + "step": 310 + }, + { + "clip_ratio": 0.0, + "completion_length": 148.3515625, + "epoch": 0.26718213058419243, + "grad_norm": 1.290985809671743, + "kl": 0.033203125, + "learning_rate": 8.664089347079037e-07, + "loss": 0.0013, + "reward": 1.7408164143562317, + "reward_std": 0.15223591029644012, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7564414143562317, + "step": 311 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.75, + "epoch": 0.26804123711340205, + "grad_norm": 1.8077145483203556, + "kl": 0.040283203125, + "learning_rate": 8.659793814432989e-07, + "loss": 0.0016, + "reward": 1.7184379696846008, + "reward_std": 0.12002924084663391, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7340629696846008, + "step": 312 + }, + { + "clip_ratio": 0.0, + "completion_length": 134.3828125, + "epoch": 0.26890034364261167, + "grad_norm": 2.7429724901343433, + "kl": 0.048583984375, + "learning_rate": 8.655498281786941e-07, + "loss": 0.0019, + "reward": 1.731776773929596, + "reward_std": 0.1657547503709793, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.747401773929596, + "step": 313 + }, + { + "clip_ratio": 0.0, + "completion_length": 138.0234375, + "epoch": 0.2697594501718213, + "grad_norm": 0.9847082773950215, + "kl": 0.0380859375, + "learning_rate": 8.651202749140893e-07, + "loss": 0.0015, + "reward": 1.701866626739502, + "reward_std": 0.12148784846067429, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7018666565418243, + "step": 314 + }, + { + "clip_ratio": 0.0, + "completion_length": 156.46875, + "epoch": 0.2706185567010309, + "grad_norm": 1.5399880460978463, + "kl": 0.03057861328125, + "learning_rate": 8.646907216494845e-07, + "loss": 0.0012, + "reward": 1.6783122420310974, + "reward_std": 0.1540703848004341, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.686124712228775, + "step": 315 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.9921875, + "epoch": 0.27147766323024053, + "grad_norm": 1.5313679692857116, + "kl": 0.0394287109375, + "learning_rate": 8.642611683848797e-07, + "loss": 0.0016, + "reward": 1.7149747014045715, + "reward_std": 0.2113272026181221, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7462246716022491, + "step": 316 + }, + { + "clip_ratio": 0.0, + "completion_length": 153.3984375, + "epoch": 0.27233676975945015, + "grad_norm": 1.3181550613878923, + "kl": 0.0369873046875, + "learning_rate": 8.638316151202749e-07, + "loss": 0.0015, + "reward": 1.8070083856582642, + "reward_std": 0.18914027512073517, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8226334154605865, + "step": 317 + }, + { + "clip_ratio": 0.0, + "completion_length": 147.4453125, + "epoch": 0.27319587628865977, + "grad_norm": 1.1473725643237571, + "kl": 0.035400390625, + "learning_rate": 8.634020618556701e-07, + "loss": 0.0014, + "reward": 1.8190871477127075, + "reward_std": 0.15289827063679695, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.8503372073173523, + "step": 318 + }, + { + "clip_ratio": 0.0, + "completion_length": 157.6015625, + "epoch": 0.27405498281786944, + "grad_norm": 0.7143585607947084, + "kl": 0.03167724609375, + "learning_rate": 8.629725085910653e-07, + "loss": 0.0013, + "reward": 1.8210635781288147, + "reward_std": 0.07932163029909134, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8210635781288147, + "step": 319 + }, + { + "clip_ratio": 0.0, + "completion_length": 143.28125, + "epoch": 0.27491408934707906, + "grad_norm": 1.3665635736893489, + "kl": 0.02557373046875, + "learning_rate": 8.625429553264604e-07, + "loss": 0.001, + "reward": 1.7617181539535522, + "reward_std": 0.1237938292324543, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7617180943489075, + "step": 320 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.1953125, + "epoch": 0.2757731958762887, + "grad_norm": 1.1585944459835054, + "kl": 0.0296630859375, + "learning_rate": 8.621134020618556e-07, + "loss": 0.0012, + "reward": 1.7693681120872498, + "reward_std": 0.15534278005361557, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7771806418895721, + "step": 321 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.359375, + "epoch": 0.2766323024054983, + "grad_norm": 1.3885088694278251, + "kl": 0.03216552734375, + "learning_rate": 8.616838487972508e-07, + "loss": 0.0013, + "reward": 1.6785261631011963, + "reward_std": 0.11299961805343628, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6785261631011963, + "step": 322 + }, + { + "clip_ratio": 0.0, + "completion_length": 147.0234375, + "epoch": 0.2774914089347079, + "grad_norm": 1.0963555730545884, + "kl": 0.035888671875, + "learning_rate": 8.612542955326461e-07, + "loss": 0.0014, + "reward": 1.724756121635437, + "reward_std": 0.15419768542051315, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7325686514377594, + "step": 323 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.734375, + "epoch": 0.27835051546391754, + "grad_norm": 1.9036632085091891, + "kl": 0.029296875, + "learning_rate": 8.608247422680413e-07, + "loss": 0.0012, + "reward": 1.7815952897071838, + "reward_std": 0.07885266095399857, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7815952897071838, + "step": 324 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.078125, + "epoch": 0.27920962199312716, + "grad_norm": 0.8583001841176572, + "kl": 0.03070068359375, + "learning_rate": 8.603951890034365e-07, + "loss": 0.0012, + "reward": 1.805925726890564, + "reward_std": 0.08024111017584801, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8059256374835968, + "step": 325 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.4921875, + "epoch": 0.2800687285223368, + "grad_norm": 1.6115188524753052, + "kl": 0.0347900390625, + "learning_rate": 8.599656357388317e-07, + "loss": 0.0014, + "reward": 1.6898165941238403, + "reward_std": 0.17968852818012238, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7132540941238403, + "step": 326 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.09375, + "epoch": 0.2809278350515464, + "grad_norm": 3.0925571712058755, + "kl": 0.0787353515625, + "learning_rate": 8.595360824742269e-07, + "loss": 0.0032, + "reward": 1.7974568605422974, + "reward_std": 0.1517610400915146, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.8208943903446198, + "step": 327 + }, + { + "clip_ratio": 0.0, + "completion_length": 171.8125, + "epoch": 0.281786941580756, + "grad_norm": 1.0479920039978226, + "kl": 0.02923583984375, + "learning_rate": 8.591065292096219e-07, + "loss": 0.0012, + "reward": 1.8526938557624817, + "reward_std": 0.07072590291500092, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8526938557624817, + "step": 328 + }, + { + "clip_ratio": 0.0, + "completion_length": 148.2734375, + "epoch": 0.28264604810996563, + "grad_norm": 2.741217901691856, + "kl": 0.03240966796875, + "learning_rate": 8.586769759450171e-07, + "loss": 0.0013, + "reward": 1.7304821014404297, + "reward_std": 0.10214803740382195, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7461071014404297, + "step": 329 + }, + { + "clip_ratio": 0.0, + "completion_length": 161.578125, + "epoch": 0.28350515463917525, + "grad_norm": 1.1171189004763298, + "kl": 0.0263671875, + "learning_rate": 8.582474226804123e-07, + "loss": 0.0011, + "reward": 1.7979252934455872, + "reward_std": 0.12719742208719254, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8057378232479095, + "step": 330 + }, + { + "clip_ratio": 0.0, + "completion_length": 151.4765625, + "epoch": 0.28436426116838487, + "grad_norm": 2.031628516822189, + "kl": 0.05181884765625, + "learning_rate": 8.578178694158075e-07, + "loss": 0.0021, + "reward": 1.6497213244438171, + "reward_std": 0.1195969358086586, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.6731588840484619, + "step": 331 + }, + { + "clip_ratio": 0.0, + "completion_length": 134.4453125, + "epoch": 0.2852233676975945, + "grad_norm": 2.801651555129562, + "kl": 0.02777099609375, + "learning_rate": 8.573883161512027e-07, + "loss": 0.0011, + "reward": 1.8582192659378052, + "reward_std": 0.10834803432226181, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8738442659378052, + "step": 332 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.6640625, + "epoch": 0.2860824742268041, + "grad_norm": 11.067746215654495, + "kl": 0.02642822265625, + "learning_rate": 8.569587628865979e-07, + "loss": 0.0011, + "reward": 1.7104763984680176, + "reward_std": 0.19199497997760773, + "rewards/format_reward_gen": 0.9609375, + "rewards/llm_reward": 0.74953892827034, + "step": 333 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.6796875, + "epoch": 0.2869415807560137, + "grad_norm": 1.2489715002859365, + "kl": 0.0211181640625, + "learning_rate": 8.565292096219931e-07, + "loss": 0.0008, + "reward": 1.6395170092582703, + "reward_std": 0.1772407442331314, + "rewards/format_reward_gen": 0.9296875, + "rewards/llm_reward": 0.7098294794559479, + "step": 334 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.953125, + "epoch": 0.28780068728522334, + "grad_norm": 1.3905135992782467, + "kl": 0.02386474609375, + "learning_rate": 8.560996563573883e-07, + "loss": 0.001, + "reward": 1.6524567604064941, + "reward_std": 0.22227772325277328, + "rewards/format_reward_gen": 0.921875, + "rewards/llm_reward": 0.7305817306041718, + "step": 335 + }, + { + "clip_ratio": 0.0, + "completion_length": 154.75, + "epoch": 0.28865979381443296, + "grad_norm": 6.054324722106658, + "kl": 0.02081298828125, + "learning_rate": 8.556701030927834e-07, + "loss": 0.0008, + "reward": 1.7611451148986816, + "reward_std": 0.2188207283616066, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7923950850963593, + "step": 336 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.796875, + "epoch": 0.28951890034364264, + "grad_norm": 3.7876750335169507, + "kl": 0.021484375, + "learning_rate": 8.552405498281786e-07, + "loss": 0.0009, + "reward": 1.681132197380066, + "reward_std": 0.23194558173418045, + "rewards/format_reward_gen": 0.9296875, + "rewards/llm_reward": 0.7514447271823883, + "step": 337 + }, + { + "clip_ratio": 0.0, + "completion_length": 139.84375, + "epoch": 0.29037800687285226, + "grad_norm": 1.4224659056279383, + "kl": 0.02587890625, + "learning_rate": 8.548109965635738e-07, + "loss": 0.001, + "reward": 1.78468519449234, + "reward_std": 0.2202548086643219, + "rewards/format_reward_gen": 0.9453125, + "rewards/llm_reward": 0.8393726646900177, + "step": 338 + }, + { + "clip_ratio": 0.0, + "completion_length": 142.328125, + "epoch": 0.2912371134020619, + "grad_norm": 1.811194222155832, + "kl": 0.024658203125, + "learning_rate": 8.54381443298969e-07, + "loss": 0.001, + "reward": 1.8332806825637817, + "reward_std": 0.09518006816506386, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.8567183017730713, + "step": 339 + }, + { + "clip_ratio": 0.0, + "completion_length": 148.078125, + "epoch": 0.2920962199312715, + "grad_norm": 1.45115586999827, + "kl": 0.02301025390625, + "learning_rate": 8.539518900343642e-07, + "loss": 0.0009, + "reward": 1.7859101295471191, + "reward_std": 0.14324617385864258, + "rewards/format_reward_gen": 0.9609375, + "rewards/llm_reward": 0.8249726593494415, + "step": 340 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.6015625, + "epoch": 0.2929553264604811, + "grad_norm": 1.3891780059241028, + "kl": 0.0213623046875, + "learning_rate": 8.535223367697594e-07, + "loss": 0.0009, + "reward": 1.7494396567344666, + "reward_std": 0.14654994010925293, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7572520971298218, + "step": 341 + }, + { + "clip_ratio": 0.0, + "completion_length": 139.5, + "epoch": 0.29381443298969073, + "grad_norm": 1.0515730604032878, + "kl": 0.0264892578125, + "learning_rate": 8.530927835051546e-07, + "loss": 0.0011, + "reward": 1.7149075269699097, + "reward_std": 0.12536488845944405, + "rewards/format_reward_gen": 0.9453125, + "rewards/llm_reward": 0.7695950269699097, + "step": 342 + }, + { + "clip_ratio": 0.0, + "completion_length": 152.171875, + "epoch": 0.29467353951890035, + "grad_norm": 1.884237347147452, + "kl": 0.0213623046875, + "learning_rate": 8.526632302405498e-07, + "loss": 0.0009, + "reward": 1.7150232791900635, + "reward_std": 0.17639710754156113, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7306482791900635, + "step": 343 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.0390625, + "epoch": 0.29553264604810997, + "grad_norm": 1.0301696531429079, + "kl": 0.02069091796875, + "learning_rate": 8.52233676975945e-07, + "loss": 0.0008, + "reward": 1.8793914318084717, + "reward_std": 0.04007810167968273, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8793914318084717, + "step": 344 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.0859375, + "epoch": 0.2963917525773196, + "grad_norm": 1.6887023989278636, + "kl": 0.022216796875, + "learning_rate": 8.518041237113401e-07, + "loss": 0.0009, + "reward": 1.7447214126586914, + "reward_std": 0.16757074743509293, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7447213530540466, + "step": 345 + }, + { + "clip_ratio": 0.0, + "completion_length": 121.859375, + "epoch": 0.2972508591065292, + "grad_norm": 0.9804680400847956, + "kl": 0.026123046875, + "learning_rate": 8.513745704467353e-07, + "loss": 0.001, + "reward": 1.7684549689292908, + "reward_std": 0.06802868098020554, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7762674689292908, + "step": 346 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.203125, + "epoch": 0.2981099656357388, + "grad_norm": 1.4132838829734102, + "kl": 0.02655029296875, + "learning_rate": 8.509450171821305e-07, + "loss": 0.0011, + "reward": 1.7236329913139343, + "reward_std": 0.15013974159955978, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7392580211162567, + "step": 347 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.34375, + "epoch": 0.29896907216494845, + "grad_norm": 3.798539075577766, + "kl": 0.03009033203125, + "learning_rate": 8.505154639175257e-07, + "loss": 0.0012, + "reward": 1.7059985995292664, + "reward_std": 0.13966944441199303, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7059986293315887, + "step": 348 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.7578125, + "epoch": 0.29982817869415807, + "grad_norm": 1.4928606296691562, + "kl": 0.05828857421875, + "learning_rate": 8.500859106529209e-07, + "loss": 0.0023, + "reward": 1.7457636594772339, + "reward_std": 0.16971006989479065, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7770136296749115, + "step": 349 + }, + { + "clip_ratio": 0.0, + "completion_length": 121.828125, + "epoch": 0.3006872852233677, + "grad_norm": 8.775228795942514, + "kl": 0.02349853515625, + "learning_rate": 8.496563573883161e-07, + "loss": 0.0009, + "reward": 1.75225168466568, + "reward_std": 0.11466844379901886, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7522516846656799, + "step": 350 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.2578125, + "epoch": 0.3015463917525773, + "grad_norm": 1.2582262027913427, + "kl": 0.02496337890625, + "learning_rate": 8.492268041237113e-07, + "loss": 0.001, + "reward": 1.8221701383590698, + "reward_std": 0.09425491839647293, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8299825489521027, + "step": 351 + }, + { + "clip_ratio": 0.0, + "completion_length": 115.328125, + "epoch": 0.3024054982817869, + "grad_norm": 8.10337252047317, + "kl": 0.029541015625, + "learning_rate": 8.487972508591065e-07, + "loss": 0.0012, + "reward": 1.7997434735298157, + "reward_std": 0.07890489138662815, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7997434437274933, + "step": 352 + }, + { + "clip_ratio": 0.0, + "completion_length": 121.4375, + "epoch": 0.30326460481099654, + "grad_norm": 0.8630573702061485, + "kl": 0.025146484375, + "learning_rate": 8.483676975945016e-07, + "loss": 0.001, + "reward": 1.735402524471283, + "reward_std": 0.1313318181782961, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7510275840759277, + "step": 353 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.671875, + "epoch": 0.30412371134020616, + "grad_norm": 2.294001840527455, + "kl": 0.021240234375, + "learning_rate": 8.479381443298968e-07, + "loss": 0.0008, + "reward": 1.8022547364234924, + "reward_std": 0.14323440939188004, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.81006720662117, + "step": 354 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.0625, + "epoch": 0.30498281786941583, + "grad_norm": 1.3365205015914472, + "kl": 0.0291748046875, + "learning_rate": 8.47508591065292e-07, + "loss": 0.0012, + "reward": 1.798896074295044, + "reward_std": 0.14688794314861298, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.806708574295044, + "step": 355 + }, + { + "clip_ratio": 0.0, + "completion_length": 116.65625, + "epoch": 0.30584192439862545, + "grad_norm": 1.3467405037883613, + "kl": 0.03033447265625, + "learning_rate": 8.470790378006872e-07, + "loss": 0.0012, + "reward": 1.7370511293411255, + "reward_std": 0.1430470496416092, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7448635101318359, + "step": 356 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.3125, + "epoch": 0.30670103092783507, + "grad_norm": 1.4250852392657183, + "kl": 0.02288818359375, + "learning_rate": 8.466494845360824e-07, + "loss": 0.0009, + "reward": 1.7630817294120789, + "reward_std": 0.099862240254879, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7630816698074341, + "step": 357 + }, + { + "clip_ratio": 0.0, + "completion_length": 114.484375, + "epoch": 0.3075601374570447, + "grad_norm": 1.001564962464479, + "kl": 0.0289306640625, + "learning_rate": 8.462199312714776e-07, + "loss": 0.0012, + "reward": 1.7671634554862976, + "reward_std": 0.07000754773616791, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7671634256839752, + "step": 358 + }, + { + "clip_ratio": 0.0, + "completion_length": 110.234375, + "epoch": 0.3084192439862543, + "grad_norm": 1.7447567831684878, + "kl": 0.02752685546875, + "learning_rate": 8.457903780068728e-07, + "loss": 0.0011, + "reward": 1.6856686472892761, + "reward_std": 0.12096000462770462, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6856685876846313, + "step": 359 + }, + { + "clip_ratio": 0.0, + "completion_length": 122.0078125, + "epoch": 0.30927835051546393, + "grad_norm": 1.2242293521408898, + "kl": 0.02239990234375, + "learning_rate": 8.45360824742268e-07, + "loss": 0.0009, + "reward": 1.8242759108543396, + "reward_std": 0.08643890917301178, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8242759108543396, + "step": 360 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.65625, + "epoch": 0.31013745704467355, + "grad_norm": 1.5243424904852039, + "kl": 0.0262451171875, + "learning_rate": 8.449312714776631e-07, + "loss": 0.001, + "reward": 1.7807704210281372, + "reward_std": 0.0936017706990242, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7885829210281372, + "step": 361 + }, + { + "clip_ratio": 0.0, + "completion_length": 122.734375, + "epoch": 0.31099656357388317, + "grad_norm": 1.4884814724042141, + "kl": 0.022216796875, + "learning_rate": 8.445017182130583e-07, + "loss": 0.0009, + "reward": 1.8066073656082153, + "reward_std": 0.09719736129045486, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8144198656082153, + "step": 362 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.359375, + "epoch": 0.3118556701030928, + "grad_norm": 1.4940528435261182, + "kl": 0.02490234375, + "learning_rate": 8.440721649484535e-07, + "loss": 0.001, + "reward": 1.6919100880622864, + "reward_std": 0.08997226506471634, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6919100880622864, + "step": 363 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.2265625, + "epoch": 0.3127147766323024, + "grad_norm": 1.0097324218360122, + "kl": 0.023681640625, + "learning_rate": 8.436426116838487e-07, + "loss": 0.0009, + "reward": 1.7592207789421082, + "reward_std": 0.12093230336904526, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7592207789421082, + "step": 364 + }, + { + "clip_ratio": 0.0, + "completion_length": 120.3984375, + "epoch": 0.313573883161512, + "grad_norm": 1.1761577149895857, + "kl": 0.01922607421875, + "learning_rate": 8.432130584192439e-07, + "loss": 0.0008, + "reward": 1.7229613065719604, + "reward_std": 0.1399160847067833, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7229613661766052, + "step": 365 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.0390625, + "epoch": 0.31443298969072164, + "grad_norm": 2.443207308672, + "kl": 0.028076171875, + "learning_rate": 8.427835051546391e-07, + "loss": 0.0011, + "reward": 1.78279447555542, + "reward_std": 0.08378404378890991, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7827944755554199, + "step": 366 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.15625, + "epoch": 0.31529209621993126, + "grad_norm": 1.3122123216803352, + "kl": 0.0269775390625, + "learning_rate": 8.423539518900344e-07, + "loss": 0.0011, + "reward": 1.7783340811729431, + "reward_std": 0.13799217343330383, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7783340811729431, + "step": 367 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.8125, + "epoch": 0.3161512027491409, + "grad_norm": 1.6680208275489512, + "kl": 0.0198974609375, + "learning_rate": 8.419243986254296e-07, + "loss": 0.0008, + "reward": 1.7162877321243286, + "reward_std": 0.07326348312199116, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7162877917289734, + "step": 368 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.3125, + "epoch": 0.3170103092783505, + "grad_norm": 1.382745000875992, + "kl": 0.02130126953125, + "learning_rate": 8.414948453608248e-07, + "loss": 0.0009, + "reward": 1.79585862159729, + "reward_std": 0.13986750692129135, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.79585862159729, + "step": 369 + }, + { + "clip_ratio": 0.0, + "completion_length": 120.625, + "epoch": 0.3178694158075601, + "grad_norm": 1.1013774115401274, + "kl": 0.02069091796875, + "learning_rate": 8.410652920962199e-07, + "loss": 0.0008, + "reward": 1.753614366054535, + "reward_std": 0.10165842995047569, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7536143660545349, + "step": 370 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.4765625, + "epoch": 0.31872852233676974, + "grad_norm": 1.7113488988118148, + "kl": 0.0211181640625, + "learning_rate": 8.406357388316151e-07, + "loss": 0.0008, + "reward": 1.861440122127533, + "reward_std": 0.06895541399717331, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8692527115345001, + "step": 371 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.9765625, + "epoch": 0.31958762886597936, + "grad_norm": 1.103897326474956, + "kl": 0.021484375, + "learning_rate": 8.402061855670103e-07, + "loss": 0.0009, + "reward": 1.7671774625778198, + "reward_std": 0.09223662130534649, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7671774327754974, + "step": 372 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.59375, + "epoch": 0.32044673539518903, + "grad_norm": 1.1271147620986939, + "kl": 0.018157958984375, + "learning_rate": 8.397766323024055e-07, + "loss": 0.0007, + "reward": 1.7452142238616943, + "reward_std": 0.12823403999209404, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7530267238616943, + "step": 373 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.59375, + "epoch": 0.32130584192439865, + "grad_norm": 1.1312273949872451, + "kl": 0.0181884765625, + "learning_rate": 8.393470790378007e-07, + "loss": 0.0007, + "reward": 1.76444673538208, + "reward_std": 0.08396412804722786, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7644466161727905, + "step": 374 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.3359375, + "epoch": 0.32216494845360827, + "grad_norm": 1.1834607042303573, + "kl": 0.02215576171875, + "learning_rate": 8.389175257731959e-07, + "loss": 0.0009, + "reward": 1.7634702920913696, + "reward_std": 0.07862638682126999, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7712827622890472, + "step": 375 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.6796875, + "epoch": 0.3230240549828179, + "grad_norm": 1.218745212804485, + "kl": 0.027587890625, + "learning_rate": 8.384879725085911e-07, + "loss": 0.0011, + "reward": 1.773351788520813, + "reward_std": 0.10650411248207092, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7889767587184906, + "step": 376 + }, + { + "clip_ratio": 0.0, + "completion_length": 142.5625, + "epoch": 0.3238831615120275, + "grad_norm": 2.9765457354255247, + "kl": 0.0250244140625, + "learning_rate": 8.380584192439863e-07, + "loss": 0.001, + "reward": 1.8011161088943481, + "reward_std": 0.14274968206882477, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8089286386966705, + "step": 377 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.703125, + "epoch": 0.3247422680412371, + "grad_norm": 4.935642518044648, + "kl": 0.01898193359375, + "learning_rate": 8.376288659793815e-07, + "loss": 0.0008, + "reward": 1.8446325659751892, + "reward_std": 0.12704916298389435, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8524451553821564, + "step": 378 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.2109375, + "epoch": 0.32560137457044674, + "grad_norm": 0.8982810314616194, + "kl": 0.0203857421875, + "learning_rate": 8.371993127147766e-07, + "loss": 0.0008, + "reward": 1.8438200950622559, + "reward_std": 0.07897239178419113, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8438200354576111, + "step": 379 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.6328125, + "epoch": 0.32646048109965636, + "grad_norm": 0.8997947654002385, + "kl": 0.02166748046875, + "learning_rate": 8.367697594501718e-07, + "loss": 0.0009, + "reward": 1.8457976579666138, + "reward_std": 0.05379333579912782, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.845797598361969, + "step": 380 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.171875, + "epoch": 0.327319587628866, + "grad_norm": 1.221630816341642, + "kl": 0.0252685546875, + "learning_rate": 8.36340206185567e-07, + "loss": 0.001, + "reward": 1.755448818206787, + "reward_std": 0.12511924654245377, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7710737586021423, + "step": 381 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.0390625, + "epoch": 0.3281786941580756, + "grad_norm": 1.7175446698553867, + "kl": 0.0228271484375, + "learning_rate": 8.359106529209622e-07, + "loss": 0.0009, + "reward": 1.8109247088432312, + "reward_std": 0.10409371182322502, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8187372088432312, + "step": 382 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.4296875, + "epoch": 0.3290378006872852, + "grad_norm": 1.4545443289160316, + "kl": 0.0201416015625, + "learning_rate": 8.354810996563574e-07, + "loss": 0.0008, + "reward": 1.8992087244987488, + "reward_std": 0.07725157961249352, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8992086946964264, + "step": 383 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.6015625, + "epoch": 0.32989690721649484, + "grad_norm": 2.1100358243584743, + "kl": 0.0252685546875, + "learning_rate": 8.350515463917526e-07, + "loss": 0.001, + "reward": 1.8205956816673279, + "reward_std": 0.12146460637450218, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8284082114696503, + "step": 384 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.9140625, + "epoch": 0.33075601374570446, + "grad_norm": 1.3804484761106177, + "kl": 0.02508544921875, + "learning_rate": 8.346219931271478e-07, + "loss": 0.001, + "reward": 1.7877458333969116, + "reward_std": 0.13950634747743607, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7955583333969116, + "step": 385 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.703125, + "epoch": 0.3316151202749141, + "grad_norm": 1.0084199177905135, + "kl": 0.0240478515625, + "learning_rate": 8.34192439862543e-07, + "loss": 0.001, + "reward": 1.7355512380599976, + "reward_std": 0.11318296939134598, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7355512380599976, + "step": 386 + }, + { + "clip_ratio": 0.0, + "completion_length": 149.8125, + "epoch": 0.3324742268041237, + "grad_norm": 0.8344574864466167, + "kl": 0.02178955078125, + "learning_rate": 8.337628865979381e-07, + "loss": 0.0009, + "reward": 1.8301225900650024, + "reward_std": 0.08988046087324619, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.83012256026268, + "step": 387 + }, + { + "clip_ratio": 0.0, + "completion_length": 121.3203125, + "epoch": 0.3333333333333333, + "grad_norm": 0.835913485605179, + "kl": 0.0225830078125, + "learning_rate": 8.333333333333333e-07, + "loss": 0.0009, + "reward": 1.7770044803619385, + "reward_std": 0.11324162408709526, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7770044505596161, + "step": 388 + }, + { + "clip_ratio": 0.0, + "completion_length": 151.6640625, + "epoch": 0.33419243986254293, + "grad_norm": 1.0924953323879845, + "kl": 0.02471923828125, + "learning_rate": 8.329037800687285e-07, + "loss": 0.001, + "reward": 1.7987311482429504, + "reward_std": 0.08528102748095989, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.806543618440628, + "step": 389 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.8984375, + "epoch": 0.33505154639175255, + "grad_norm": 0.7425828858782676, + "kl": 0.02239990234375, + "learning_rate": 8.324742268041237e-07, + "loss": 0.0009, + "reward": 1.8864429593086243, + "reward_std": 0.09466363862156868, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8942554891109467, + "step": 390 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.359375, + "epoch": 0.33591065292096217, + "grad_norm": 1.4256854867993622, + "kl": 0.020355224609375, + "learning_rate": 8.320446735395189e-07, + "loss": 0.0008, + "reward": 1.805371105670929, + "reward_std": 0.08615696430206299, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8053711354732513, + "step": 391 + }, + { + "clip_ratio": 0.0, + "completion_length": 143.6796875, + "epoch": 0.33676975945017185, + "grad_norm": 1.2154506440783912, + "kl": 0.02349853515625, + "learning_rate": 8.316151202749141e-07, + "loss": 0.0009, + "reward": 1.7635123133659363, + "reward_std": 0.125723734498024, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7713248431682587, + "step": 392 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.3203125, + "epoch": 0.33762886597938147, + "grad_norm": 1.063545112594486, + "kl": 0.023193359375, + "learning_rate": 8.311855670103093e-07, + "loss": 0.0009, + "reward": 1.7525449395179749, + "reward_std": 0.09935981594026089, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7603574395179749, + "step": 393 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.2734375, + "epoch": 0.3384879725085911, + "grad_norm": 1.1910010378396947, + "kl": 0.0213623046875, + "learning_rate": 8.307560137457045e-07, + "loss": 0.0009, + "reward": 1.8792944550514221, + "reward_std": 0.07655412331223488, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8792945444583893, + "step": 394 + }, + { + "clip_ratio": 0.0, + "completion_length": 138.4453125, + "epoch": 0.3393470790378007, + "grad_norm": 1.0352744087470642, + "kl": 0.0286865234375, + "learning_rate": 8.303264604810996e-07, + "loss": 0.0011, + "reward": 1.817804753780365, + "reward_std": 0.09332024306058884, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.825617253780365, + "step": 395 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.984375, + "epoch": 0.3402061855670103, + "grad_norm": 3.4370614566686366, + "kl": 0.02947998046875, + "learning_rate": 8.298969072164948e-07, + "loss": 0.0012, + "reward": 1.7064816355705261, + "reward_std": 0.11401160806417465, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7064816355705261, + "step": 396 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.5546875, + "epoch": 0.34106529209621994, + "grad_norm": 1.8575760022730172, + "kl": 0.02227783203125, + "learning_rate": 8.2946735395189e-07, + "loss": 0.0009, + "reward": 1.7460897564888, + "reward_std": 0.1139952689409256, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7539022862911224, + "step": 397 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.09375, + "epoch": 0.34192439862542956, + "grad_norm": 1.2996032361936032, + "kl": 0.02801513671875, + "learning_rate": 8.290378006872852e-07, + "loss": 0.0011, + "reward": 1.8059646487236023, + "reward_std": 0.1373230665922165, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8215896785259247, + "step": 398 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.3984375, + "epoch": 0.3427835051546392, + "grad_norm": 1.6501841523869047, + "kl": 0.0233154296875, + "learning_rate": 8.286082474226804e-07, + "loss": 0.0009, + "reward": 1.809871256351471, + "reward_std": 0.12983518466353416, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8098713159561157, + "step": 399 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.34375, + "epoch": 0.3436426116838488, + "grad_norm": 1.111398957842247, + "kl": 0.02349853515625, + "learning_rate": 8.281786941580756e-07, + "loss": 0.0009, + "reward": 1.8126710057258606, + "reward_std": 0.0854789987206459, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8204834461212158, + "step": 400 + }, + { + "clip_ratio": 0.0, + "completion_length": 150.875, + "epoch": 0.3445017182130584, + "grad_norm": 1.5863639489406471, + "kl": 0.02459716796875, + "learning_rate": 8.277491408934707e-07, + "loss": 0.001, + "reward": 1.7722662687301636, + "reward_std": 0.0587652251124382, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.772266298532486, + "step": 401 + }, + { + "clip_ratio": 0.0, + "completion_length": 134.875, + "epoch": 0.34536082474226804, + "grad_norm": 1.1575507587682292, + "kl": 0.04693603515625, + "learning_rate": 8.273195876288659e-07, + "loss": 0.0019, + "reward": 1.8038502931594849, + "reward_std": 0.11479467153549194, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8116627931594849, + "step": 402 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.015625, + "epoch": 0.34621993127147765, + "grad_norm": 1.3175894104086836, + "kl": 0.02630615234375, + "learning_rate": 8.26890034364261e-07, + "loss": 0.0011, + "reward": 1.8177732825279236, + "reward_std": 0.061465613543987274, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8177732825279236, + "step": 403 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.4140625, + "epoch": 0.3470790378006873, + "grad_norm": 0.9613495919689865, + "kl": 0.03057861328125, + "learning_rate": 8.264604810996562e-07, + "loss": 0.0012, + "reward": 1.856289029121399, + "reward_std": 0.08307300135493279, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8562889695167542, + "step": 404 + }, + { + "clip_ratio": 0.0, + "completion_length": 155.5546875, + "epoch": 0.3479381443298969, + "grad_norm": 1.2236294083563708, + "kl": 0.02239990234375, + "learning_rate": 8.260309278350514e-07, + "loss": 0.0009, + "reward": 1.86646169424057, + "reward_std": 0.05629648268222809, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8664617240428925, + "step": 405 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.40625, + "epoch": 0.3487972508591065, + "grad_norm": 4.196708925682456, + "kl": 0.0263671875, + "learning_rate": 8.256013745704466e-07, + "loss": 0.0011, + "reward": 1.6891456842422485, + "reward_std": 0.13314834237098694, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6969581544399261, + "step": 406 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.9765625, + "epoch": 0.34965635738831613, + "grad_norm": 3.3390151553242826, + "kl": 0.02838134765625, + "learning_rate": 8.251718213058418e-07, + "loss": 0.0011, + "reward": 1.7792019844055176, + "reward_std": 0.08469441719353199, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7792020440101624, + "step": 407 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.796875, + "epoch": 0.35051546391752575, + "grad_norm": 1.0417856723793362, + "kl": 0.0313720703125, + "learning_rate": 8.24742268041237e-07, + "loss": 0.0013, + "reward": 1.745001196861267, + "reward_std": 0.08887993916869164, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7528136670589447, + "step": 408 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.2421875, + "epoch": 0.35137457044673537, + "grad_norm": 1.5185692601329688, + "kl": 0.02972412109375, + "learning_rate": 8.243127147766322e-07, + "loss": 0.0012, + "reward": 1.743471622467041, + "reward_std": 0.12599998712539673, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7434715926647186, + "step": 409 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.3828125, + "epoch": 0.35223367697594504, + "grad_norm": 3.698150098713745, + "kl": 0.037353515625, + "learning_rate": 8.238831615120274e-07, + "loss": 0.0015, + "reward": 1.7748391032218933, + "reward_std": 0.109963808208704, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7826516330242157, + "step": 410 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.84375, + "epoch": 0.35309278350515466, + "grad_norm": 0.7340273563755917, + "kl": 0.0301513671875, + "learning_rate": 8.234536082474227e-07, + "loss": 0.0012, + "reward": 1.747934103012085, + "reward_std": 0.029868584126234055, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7557465732097626, + "step": 411 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.140625, + "epoch": 0.3539518900343643, + "grad_norm": 6.5869920419752885, + "kl": 0.03033447265625, + "learning_rate": 8.230240549828178e-07, + "loss": 0.0012, + "reward": 1.803978145122528, + "reward_std": 0.11175262182950974, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8196031451225281, + "step": 412 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.0625, + "epoch": 0.3548109965635739, + "grad_norm": 1.2110163382375696, + "kl": 0.03582763671875, + "learning_rate": 8.22594501718213e-07, + "loss": 0.0014, + "reward": 1.7075645923614502, + "reward_std": 0.09112687408924103, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7075645625591278, + "step": 413 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.2109375, + "epoch": 0.3556701030927835, + "grad_norm": 0.9440184144682974, + "kl": 0.02886962890625, + "learning_rate": 8.221649484536082e-07, + "loss": 0.0012, + "reward": 1.879349708557129, + "reward_std": 0.077772606164217, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8871622085571289, + "step": 414 + }, + { + "clip_ratio": 0.0, + "completion_length": 158.203125, + "epoch": 0.35652920962199314, + "grad_norm": 1.1844513928260747, + "kl": 0.024658203125, + "learning_rate": 8.217353951890034e-07, + "loss": 0.001, + "reward": 1.8242889642715454, + "reward_std": 0.11754781007766724, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8321015238761902, + "step": 415 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.9453125, + "epoch": 0.35738831615120276, + "grad_norm": 1.1572419610665239, + "kl": 0.02496337890625, + "learning_rate": 8.213058419243986e-07, + "loss": 0.001, + "reward": 1.7361584305763245, + "reward_std": 0.0901523232460022, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7439709305763245, + "step": 416 + }, + { + "clip_ratio": 0.0, + "completion_length": 150.4140625, + "epoch": 0.3582474226804124, + "grad_norm": 1.1187566144341665, + "kl": 0.02642822265625, + "learning_rate": 8.208762886597938e-07, + "loss": 0.0011, + "reward": 1.787147879600525, + "reward_std": 0.08959689736366272, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7871478796005249, + "step": 417 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.8515625, + "epoch": 0.359106529209622, + "grad_norm": 1.469623021469994, + "kl": 0.0328369140625, + "learning_rate": 8.20446735395189e-07, + "loss": 0.0013, + "reward": 1.7043437957763672, + "reward_std": 0.13283608853816986, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7043437361717224, + "step": 418 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.7890625, + "epoch": 0.3599656357388316, + "grad_norm": 1.7756842310117262, + "kl": 0.03424072265625, + "learning_rate": 8.200171821305842e-07, + "loss": 0.0014, + "reward": 1.8168611526489258, + "reward_std": 0.09070071205496788, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8168611526489258, + "step": 419 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.0078125, + "epoch": 0.36082474226804123, + "grad_norm": 1.9241728030153786, + "kl": 0.03515625, + "learning_rate": 8.195876288659793e-07, + "loss": 0.0014, + "reward": 1.7493263483047485, + "reward_std": 0.12105973809957504, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7493264377117157, + "step": 420 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.203125, + "epoch": 0.36168384879725085, + "grad_norm": 2.3633953737617848, + "kl": 0.0306396484375, + "learning_rate": 8.191580756013745e-07, + "loss": 0.0012, + "reward": 1.7810669541358948, + "reward_std": 0.09037407860159874, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7810669541358948, + "step": 421 + }, + { + "clip_ratio": 0.0, + "completion_length": 142.0078125, + "epoch": 0.36254295532646047, + "grad_norm": 1.1965175464927247, + "kl": 0.0252685546875, + "learning_rate": 8.187285223367697e-07, + "loss": 0.001, + "reward": 1.82912939786911, + "reward_std": 0.13231520354747772, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8369418382644653, + "step": 422 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.6484375, + "epoch": 0.3634020618556701, + "grad_norm": 0.9630157435896298, + "kl": 0.025634765625, + "learning_rate": 8.182989690721649e-07, + "loss": 0.001, + "reward": 1.8102641105651855, + "reward_std": 0.10265975818037987, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8102641105651855, + "step": 423 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.0234375, + "epoch": 0.3642611683848797, + "grad_norm": 1.4555619232603647, + "kl": 0.0316162109375, + "learning_rate": 8.178694158075601e-07, + "loss": 0.0013, + "reward": 1.763112723827362, + "reward_std": 0.15452294051647186, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7709251940250397, + "step": 424 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.6953125, + "epoch": 0.3651202749140893, + "grad_norm": 1.6695763498386311, + "kl": 0.02679443359375, + "learning_rate": 8.174398625429553e-07, + "loss": 0.0011, + "reward": 1.8538659811019897, + "reward_std": 0.06878830399364233, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8538659512996674, + "step": 425 + }, + { + "clip_ratio": 0.0, + "completion_length": 138.7734375, + "epoch": 0.36597938144329895, + "grad_norm": 1.714285388160138, + "kl": 0.02813720703125, + "learning_rate": 8.170103092783505e-07, + "loss": 0.0011, + "reward": 1.7433908581733704, + "reward_std": 0.11577881872653961, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7512032985687256, + "step": 426 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.375, + "epoch": 0.36683848797250856, + "grad_norm": 2.0652235283446543, + "kl": 0.03082275390625, + "learning_rate": 8.165807560137457e-07, + "loss": 0.0012, + "reward": 1.7070344686508179, + "reward_std": 0.18474984169006348, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7226594686508179, + "step": 427 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.4609375, + "epoch": 0.36769759450171824, + "grad_norm": 0.952329796803498, + "kl": 0.03497314453125, + "learning_rate": 8.161512027491409e-07, + "loss": 0.0014, + "reward": 1.8179508447647095, + "reward_std": 0.06837813928723335, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8179508149623871, + "step": 428 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.7578125, + "epoch": 0.36855670103092786, + "grad_norm": 1.6252786551878542, + "kl": 0.0391845703125, + "learning_rate": 8.15721649484536e-07, + "loss": 0.0016, + "reward": 1.7616538405418396, + "reward_std": 0.0948004387319088, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7694664299488068, + "step": 429 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.1328125, + "epoch": 0.3694158075601375, + "grad_norm": 4.575833798673977, + "kl": 0.02618408203125, + "learning_rate": 8.152920962199312e-07, + "loss": 0.001, + "reward": 1.7446335554122925, + "reward_std": 0.14209628105163574, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7446335852146149, + "step": 430 + }, + { + "clip_ratio": 0.0, + "completion_length": 149.5078125, + "epoch": 0.3702749140893471, + "grad_norm": 1.5029038984828853, + "kl": 0.027099609375, + "learning_rate": 8.148625429553264e-07, + "loss": 0.0011, + "reward": 1.7977319955825806, + "reward_std": 0.12650492042303085, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7977319657802582, + "step": 431 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.9609375, + "epoch": 0.3711340206185567, + "grad_norm": 3.124108570449623, + "kl": 0.030517578125, + "learning_rate": 8.144329896907216e-07, + "loss": 0.0012, + "reward": 1.763170838356018, + "reward_std": 0.13480468839406967, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7631707787513733, + "step": 432 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.1328125, + "epoch": 0.37199312714776633, + "grad_norm": 0.9956952014358291, + "kl": 0.0367431640625, + "learning_rate": 8.140034364261168e-07, + "loss": 0.0015, + "reward": 1.8633191585540771, + "reward_std": 0.09405502304434776, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8633190989494324, + "step": 433 + }, + { + "clip_ratio": 0.0, + "completion_length": 143.8203125, + "epoch": 0.37285223367697595, + "grad_norm": 1.1621034752647021, + "kl": 0.02874755859375, + "learning_rate": 8.13573883161512e-07, + "loss": 0.0012, + "reward": 1.7654351592063904, + "reward_std": 0.10451111197471619, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7654351592063904, + "step": 434 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.84375, + "epoch": 0.37371134020618557, + "grad_norm": 1.0937552701332574, + "kl": 0.02410888671875, + "learning_rate": 8.131443298969072e-07, + "loss": 0.001, + "reward": 1.7504616975784302, + "reward_std": 0.13275029510259628, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7582742869853973, + "step": 435 + }, + { + "clip_ratio": 0.0, + "completion_length": 143.40625, + "epoch": 0.3745704467353952, + "grad_norm": 1.173542025190629, + "kl": 0.0269775390625, + "learning_rate": 8.127147766323024e-07, + "loss": 0.0011, + "reward": 1.7958321571350098, + "reward_std": 0.13957487791776657, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7958321869373322, + "step": 436 + }, + { + "clip_ratio": 0.0, + "completion_length": 155.5859375, + "epoch": 0.3754295532646048, + "grad_norm": 1.0809358432591143, + "kl": 0.0223388671875, + "learning_rate": 8.122852233676975e-07, + "loss": 0.0009, + "reward": 1.8025323748588562, + "reward_std": 0.10576809197664261, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8103449046611786, + "step": 437 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.296875, + "epoch": 0.37628865979381443, + "grad_norm": 0.9307591226779286, + "kl": 0.0291748046875, + "learning_rate": 8.118556701030927e-07, + "loss": 0.0012, + "reward": 1.745145320892334, + "reward_std": 0.09199497848749161, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7607703506946564, + "step": 438 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.3046875, + "epoch": 0.37714776632302405, + "grad_norm": 1.599730206934407, + "kl": 0.025634765625, + "learning_rate": 8.114261168384879e-07, + "loss": 0.001, + "reward": 1.7341384291648865, + "reward_std": 0.08559693396091461, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7341383695602417, + "step": 439 + }, + { + "clip_ratio": 0.0, + "completion_length": 150.046875, + "epoch": 0.37800687285223367, + "grad_norm": 1.1302132744799622, + "kl": 0.0223388671875, + "learning_rate": 8.109965635738831e-07, + "loss": 0.0009, + "reward": 1.7777530550956726, + "reward_std": 0.11446893960237503, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7855655252933502, + "step": 440 + }, + { + "clip_ratio": 0.0, + "completion_length": 158.390625, + "epoch": 0.3788659793814433, + "grad_norm": 0.8081005861278157, + "kl": 0.03192138671875, + "learning_rate": 8.105670103092783e-07, + "loss": 0.0013, + "reward": 1.7475718259811401, + "reward_std": 0.12176471576094627, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7475718557834625, + "step": 441 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.1640625, + "epoch": 0.3797250859106529, + "grad_norm": 1.4982803588386655, + "kl": 0.03009033203125, + "learning_rate": 8.101374570446735e-07, + "loss": 0.0012, + "reward": 1.7991742491722107, + "reward_std": 0.09572646208107471, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8069867491722107, + "step": 442 + }, + { + "clip_ratio": 0.0, + "completion_length": 139.40625, + "epoch": 0.3805841924398625, + "grad_norm": 4.788054491524054, + "kl": 0.02618408203125, + "learning_rate": 8.097079037800687e-07, + "loss": 0.001, + "reward": 1.7099648118019104, + "reward_std": 0.13577891886234283, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.725589781999588, + "step": 443 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.9140625, + "epoch": 0.38144329896907214, + "grad_norm": 1.0187295128992484, + "kl": 0.028564453125, + "learning_rate": 8.092783505154639e-07, + "loss": 0.0011, + "reward": 1.7704007029533386, + "reward_std": 0.07720200531184673, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.770400732755661, + "step": 444 + }, + { + "clip_ratio": 0.0, + "completion_length": 150.875, + "epoch": 0.38230240549828176, + "grad_norm": 1.617440205079106, + "kl": 0.0537109375, + "learning_rate": 8.08848797250859e-07, + "loss": 0.0022, + "reward": 1.745563805103302, + "reward_std": 0.12642088159918785, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7533764243125916, + "step": 445 + }, + { + "clip_ratio": 0.0, + "completion_length": 154.46875, + "epoch": 0.38316151202749144, + "grad_norm": 1.047930515930632, + "kl": 0.0303955078125, + "learning_rate": 8.084192439862542e-07, + "loss": 0.0012, + "reward": 1.8345497846603394, + "reward_std": 0.09045657515525818, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8423622250556946, + "step": 446 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.21875, + "epoch": 0.38402061855670105, + "grad_norm": 1.4220569982461002, + "kl": 0.0233154296875, + "learning_rate": 8.079896907216494e-07, + "loss": 0.0009, + "reward": 1.7613440155982971, + "reward_std": 0.11786005645990372, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7613440155982971, + "step": 447 + }, + { + "clip_ratio": 0.0, + "completion_length": 153.1875, + "epoch": 0.3848797250859107, + "grad_norm": 1.1920009174782409, + "kl": 0.02923583984375, + "learning_rate": 8.075601374570446e-07, + "loss": 0.0012, + "reward": 1.7767443656921387, + "reward_std": 0.1390495002269745, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7845568954944611, + "step": 448 + }, + { + "clip_ratio": 0.0, + "completion_length": 145.0703125, + "epoch": 0.3857388316151203, + "grad_norm": 0.8285644733256662, + "kl": 0.0247802734375, + "learning_rate": 8.071305841924398e-07, + "loss": 0.001, + "reward": 1.8258004784584045, + "reward_std": 0.06260296143591404, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8258004486560822, + "step": 449 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.75, + "epoch": 0.3865979381443299, + "grad_norm": 1.7038040697442591, + "kl": 0.026123046875, + "learning_rate": 8.06701030927835e-07, + "loss": 0.001, + "reward": 1.8847057819366455, + "reward_std": 0.1375606805086136, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.9081432819366455, + "step": 450 + }, + { + "clip_ratio": 0.0, + "completion_length": 158.8359375, + "epoch": 0.38745704467353953, + "grad_norm": 1.4889637606593127, + "kl": 0.0345458984375, + "learning_rate": 8.062714776632302e-07, + "loss": 0.0014, + "reward": 1.7658740282058716, + "reward_std": 0.18101423978805542, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7893114984035492, + "step": 451 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.59375, + "epoch": 0.38831615120274915, + "grad_norm": 1.046812904427794, + "kl": 0.0262451171875, + "learning_rate": 8.058419243986254e-07, + "loss": 0.001, + "reward": 1.7848870158195496, + "reward_std": 0.2099006325006485, + "rewards/format_reward_gen": 0.9375, + "rewards/llm_reward": 0.8473870158195496, + "step": 452 + }, + { + "clip_ratio": 0.0, + "completion_length": 156.5625, + "epoch": 0.38917525773195877, + "grad_norm": 1.4677940684663608, + "kl": 0.023193359375, + "learning_rate": 8.054123711340207e-07, + "loss": 0.0009, + "reward": 1.597209393978119, + "reward_std": 0.26560014486312866, + "rewards/format_reward_gen": 0.921875, + "rewards/llm_reward": 0.6753344535827637, + "step": 453 + }, + { + "clip_ratio": 0.0, + "completion_length": 142.2265625, + "epoch": 0.3900343642611684, + "grad_norm": 2.5890905532205957, + "kl": 0.02532958984375, + "learning_rate": 8.049828178694158e-07, + "loss": 0.001, + "reward": 1.741255521774292, + "reward_std": 0.1423398032784462, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.772505521774292, + "step": 454 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.7578125, + "epoch": 0.390893470790378, + "grad_norm": 1.877091810608803, + "kl": 0.021728515625, + "learning_rate": 8.04553264604811e-07, + "loss": 0.0009, + "reward": 1.746235966682434, + "reward_std": 0.1634746640920639, + "rewards/format_reward_gen": 0.9609375, + "rewards/llm_reward": 0.7852984368801117, + "step": 455 + }, + { + "clip_ratio": 0.0, + "completion_length": 143.0625, + "epoch": 0.3917525773195876, + "grad_norm": 1.9322462103510145, + "kl": 0.030029296875, + "learning_rate": 8.041237113402062e-07, + "loss": 0.0012, + "reward": 1.7314363718032837, + "reward_std": 0.19341952353715897, + "rewards/format_reward_gen": 0.9453125, + "rewards/llm_reward": 0.7861238718032837, + "step": 456 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.34375, + "epoch": 0.39261168384879724, + "grad_norm": 1.761668192474341, + "kl": 0.02813720703125, + "learning_rate": 8.036941580756014e-07, + "loss": 0.0011, + "reward": 1.6722000241279602, + "reward_std": 0.22043447196483612, + "rewards/format_reward_gen": 0.96875, + "rewards/llm_reward": 0.7034500241279602, + "step": 457 + }, + { + "clip_ratio": 0.0, + "completion_length": 177.1484375, + "epoch": 0.39347079037800686, + "grad_norm": 1.103623434836374, + "kl": 0.021484375, + "learning_rate": 8.032646048109966e-07, + "loss": 0.0009, + "reward": 1.7857837677001953, + "reward_std": 0.16772552579641342, + "rewards/format_reward_gen": 0.953125, + "rewards/llm_reward": 0.8326588273048401, + "step": 458 + }, + { + "clip_ratio": 0.0, + "completion_length": 149.78125, + "epoch": 0.3943298969072165, + "grad_norm": 2.9558490613021475, + "kl": 0.02484130859375, + "learning_rate": 8.028350515463918e-07, + "loss": 0.001, + "reward": 1.8172647356987, + "reward_std": 0.1397656500339508, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8172646760940552, + "step": 459 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.109375, + "epoch": 0.3951890034364261, + "grad_norm": 1.0942493174264873, + "kl": 0.02008056640625, + "learning_rate": 8.02405498281787e-07, + "loss": 0.0008, + "reward": 1.7567135095596313, + "reward_std": 0.08268255367875099, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.764525979757309, + "step": 460 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.4140625, + "epoch": 0.3960481099656357, + "grad_norm": 1.724231638784223, + "kl": 0.02337646484375, + "learning_rate": 8.019759450171822e-07, + "loss": 0.0009, + "reward": 1.812518298625946, + "reward_std": 0.1107301339507103, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8281432688236237, + "step": 461 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.875, + "epoch": 0.39690721649484534, + "grad_norm": 0.8421480615888612, + "kl": 0.02001953125, + "learning_rate": 8.015463917525774e-07, + "loss": 0.0008, + "reward": 1.8146081566810608, + "reward_std": 0.09295319765806198, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8224206566810608, + "step": 462 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.2265625, + "epoch": 0.39776632302405496, + "grad_norm": 0.9758819512120601, + "kl": 0.017822265625, + "learning_rate": 8.011168384879725e-07, + "loss": 0.0007, + "reward": 1.7702763676643372, + "reward_std": 0.10476777702569962, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7780888676643372, + "step": 463 + }, + { + "clip_ratio": 0.0, + "completion_length": 143.234375, + "epoch": 0.39862542955326463, + "grad_norm": 0.9223788317682841, + "kl": 0.02032470703125, + "learning_rate": 8.006872852233677e-07, + "loss": 0.0008, + "reward": 1.8337781429290771, + "reward_std": 0.10091580078005791, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8337781429290771, + "step": 464 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.0625, + "epoch": 0.39948453608247425, + "grad_norm": 1.1306004675623413, + "kl": 0.02081298828125, + "learning_rate": 8.002577319587629e-07, + "loss": 0.0008, + "reward": 1.6799623370170593, + "reward_std": 0.1472223922610283, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.6955873966217041, + "step": 465 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.28125, + "epoch": 0.40034364261168387, + "grad_norm": 2.7421308952943497, + "kl": 0.02642822265625, + "learning_rate": 7.998281786941581e-07, + "loss": 0.0011, + "reward": 1.7180203199386597, + "reward_std": 0.16847053170204163, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7336452901363373, + "step": 466 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.8984375, + "epoch": 0.4012027491408935, + "grad_norm": 1.1905386765274455, + "kl": 0.02166748046875, + "learning_rate": 7.993986254295533e-07, + "loss": 0.0009, + "reward": 1.824146568775177, + "reward_std": 0.09475822001695633, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8319591283798218, + "step": 467 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.828125, + "epoch": 0.4020618556701031, + "grad_norm": 1.0030306949143268, + "kl": 0.02276611328125, + "learning_rate": 7.989690721649485e-07, + "loss": 0.0009, + "reward": 1.6890740394592285, + "reward_std": 0.08577071130275726, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7046990394592285, + "step": 468 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.390625, + "epoch": 0.4029209621993127, + "grad_norm": 0.6829732849279407, + "kl": 0.020751953125, + "learning_rate": 7.985395189003437e-07, + "loss": 0.0008, + "reward": 1.8722723126411438, + "reward_std": 0.03336894512176514, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8722723126411438, + "step": 469 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.703125, + "epoch": 0.40378006872852235, + "grad_norm": 3.6315379563427315, + "kl": 0.08935546875, + "learning_rate": 7.981099656357389e-07, + "loss": 0.0036, + "reward": 1.669398546218872, + "reward_std": 0.16622114926576614, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6772109866142273, + "step": 470 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.3828125, + "epoch": 0.40463917525773196, + "grad_norm": 2.201255320619489, + "kl": 0.02001953125, + "learning_rate": 7.97680412371134e-07, + "loss": 0.0008, + "reward": 1.689961850643158, + "reward_std": 0.21605655923485756, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6977743208408356, + "step": 471 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.15625, + "epoch": 0.4054982817869416, + "grad_norm": 7.197996250278638, + "kl": 0.02166748046875, + "learning_rate": 7.972508591065292e-07, + "loss": 0.0009, + "reward": 1.8866642117500305, + "reward_std": 0.0431262981146574, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8866641819477081, + "step": 472 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.5625, + "epoch": 0.4063573883161512, + "grad_norm": 0.9395439850082784, + "kl": 0.02435302734375, + "learning_rate": 7.968213058419243e-07, + "loss": 0.001, + "reward": 1.7944100499153137, + "reward_std": 0.07158202305436134, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7944100797176361, + "step": 473 + }, + { + "clip_ratio": 0.0, + "completion_length": 127.46875, + "epoch": 0.4072164948453608, + "grad_norm": 10.764240409904978, + "kl": 0.02557373046875, + "learning_rate": 7.963917525773195e-07, + "loss": 0.001, + "reward": 1.7304428219795227, + "reward_std": 0.19595059752464294, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7382553517818451, + "step": 474 + }, + { + "clip_ratio": 0.0, + "completion_length": 123.9140625, + "epoch": 0.40807560137457044, + "grad_norm": 1.4034901060673528, + "kl": 0.02447509765625, + "learning_rate": 7.959621993127147e-07, + "loss": 0.001, + "reward": 1.7872547507286072, + "reward_std": 0.12548959627747536, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7950672507286072, + "step": 475 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.5546875, + "epoch": 0.40893470790378006, + "grad_norm": 1.8853927915690178, + "kl": 0.0291748046875, + "learning_rate": 7.955326460481099e-07, + "loss": 0.0012, + "reward": 1.6977566480636597, + "reward_std": 0.14235452935099602, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7055690884590149, + "step": 476 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.859375, + "epoch": 0.4097938144329897, + "grad_norm": 2.0359538756787243, + "kl": 0.02093505859375, + "learning_rate": 7.951030927835051e-07, + "loss": 0.0008, + "reward": 1.8194365501403809, + "reward_std": 0.14543933421373367, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8194365203380585, + "step": 477 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.0625, + "epoch": 0.4106529209621993, + "grad_norm": 2.3120838651943827, + "kl": 0.02349853515625, + "learning_rate": 7.946735395189003e-07, + "loss": 0.0009, + "reward": 1.7549501657485962, + "reward_std": 0.17499929666519165, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7627626061439514, + "step": 478 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.671875, + "epoch": 0.4115120274914089, + "grad_norm": 0.846611479751614, + "kl": 0.01953125, + "learning_rate": 7.942439862542954e-07, + "loss": 0.0008, + "reward": 1.761724591255188, + "reward_std": 0.1066037192940712, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7695370316505432, + "step": 479 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.7421875, + "epoch": 0.41237113402061853, + "grad_norm": 1.0153354601072486, + "kl": 0.021728515625, + "learning_rate": 7.938144329896906e-07, + "loss": 0.0009, + "reward": 1.8040355443954468, + "reward_std": 0.09103575721383095, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8118480145931244, + "step": 480 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.34375, + "epoch": 0.41323024054982815, + "grad_norm": 1.3316805075661329, + "kl": 0.03472900390625, + "learning_rate": 7.933848797250858e-07, + "loss": 0.0014, + "reward": 1.8269373178482056, + "reward_std": 0.0393183808773756, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8269373178482056, + "step": 481 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.4453125, + "epoch": 0.41408934707903783, + "grad_norm": 1.2169011867171224, + "kl": 0.0244140625, + "learning_rate": 7.92955326460481e-07, + "loss": 0.001, + "reward": 1.808565378189087, + "reward_std": 0.10053881630301476, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8163778781890869, + "step": 482 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.3671875, + "epoch": 0.41494845360824745, + "grad_norm": 1.9556380092566035, + "kl": 0.02239990234375, + "learning_rate": 7.925257731958762e-07, + "loss": 0.0009, + "reward": 1.7945858240127563, + "reward_std": 0.1085839495062828, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7945858240127563, + "step": 483 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.2890625, + "epoch": 0.41580756013745707, + "grad_norm": 1.4377206374386469, + "kl": 0.0299072265625, + "learning_rate": 7.920962199312714e-07, + "loss": 0.0012, + "reward": 1.7909988164901733, + "reward_std": 0.10544274002313614, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8066238462924957, + "step": 484 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.484375, + "epoch": 0.4166666666666667, + "grad_norm": 1.1637028605621145, + "kl": 0.0244140625, + "learning_rate": 7.916666666666666e-07, + "loss": 0.001, + "reward": 1.8535531759262085, + "reward_std": 0.09841496497392654, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8535531163215637, + "step": 485 + }, + { + "clip_ratio": 0.0, + "completion_length": 145.296875, + "epoch": 0.4175257731958763, + "grad_norm": 0.9089860152674513, + "kl": 0.0196533203125, + "learning_rate": 7.912371134020618e-07, + "loss": 0.0008, + "reward": 1.8009296655654907, + "reward_std": 0.08241390809416771, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.800929605960846, + "step": 486 + }, + { + "clip_ratio": 0.0, + "completion_length": 147.875, + "epoch": 0.4183848797250859, + "grad_norm": 1.1175991581337885, + "kl": 0.02105712890625, + "learning_rate": 7.908075601374569e-07, + "loss": 0.0008, + "reward": 1.7471742033958435, + "reward_std": 0.09226182475686073, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7471742033958435, + "step": 487 + }, + { + "clip_ratio": 0.0, + "completion_length": 138.65625, + "epoch": 0.41924398625429554, + "grad_norm": 1.5782041969321439, + "kl": 0.0245361328125, + "learning_rate": 7.903780068728521e-07, + "loss": 0.001, + "reward": 1.7752417922019958, + "reward_std": 0.07173944637179375, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7752417623996735, + "step": 488 + }, + { + "clip_ratio": 0.0, + "completion_length": 126.328125, + "epoch": 0.42010309278350516, + "grad_norm": 1.712440072733616, + "kl": 0.02490234375, + "learning_rate": 7.899484536082473e-07, + "loss": 0.001, + "reward": 1.7801220417022705, + "reward_std": 0.1850130558013916, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7957470417022705, + "step": 489 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.96875, + "epoch": 0.4209621993127148, + "grad_norm": 1.1903287634500803, + "kl": 0.021484375, + "learning_rate": 7.895189003436425e-07, + "loss": 0.0009, + "reward": 1.797283411026001, + "reward_std": 0.15069201588630676, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.797283411026001, + "step": 490 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.5390625, + "epoch": 0.4218213058419244, + "grad_norm": 1.3823311832547454, + "kl": 0.02471923828125, + "learning_rate": 7.890893470790377e-07, + "loss": 0.001, + "reward": 1.6768372654914856, + "reward_std": 0.14700639992952347, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6846497654914856, + "step": 491 + }, + { + "clip_ratio": 0.0, + "completion_length": 142.8828125, + "epoch": 0.422680412371134, + "grad_norm": 1.052318496217373, + "kl": 0.02825927734375, + "learning_rate": 7.886597938144329e-07, + "loss": 0.0011, + "reward": 1.7353659868240356, + "reward_std": 0.10273649916052818, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.735366016626358, + "step": 492 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.234375, + "epoch": 0.42353951890034364, + "grad_norm": 0.9659208234974426, + "kl": 0.0198974609375, + "learning_rate": 7.882302405498281e-07, + "loss": 0.0008, + "reward": 1.6551663875579834, + "reward_std": 0.10088075697422028, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.670791357755661, + "step": 493 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.6796875, + "epoch": 0.42439862542955326, + "grad_norm": 3.2851303393945175, + "kl": 0.0211181640625, + "learning_rate": 7.878006872852233e-07, + "loss": 0.0008, + "reward": 1.7861362099647522, + "reward_std": 0.1402701437473297, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.786136269569397, + "step": 494 + }, + { + "clip_ratio": 0.0, + "completion_length": 145.0390625, + "epoch": 0.4252577319587629, + "grad_norm": 1.3709194983778912, + "kl": 0.02056884765625, + "learning_rate": 7.873711340206184e-07, + "loss": 0.0008, + "reward": 1.7891325950622559, + "reward_std": 0.15913131088018417, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7891325950622559, + "step": 495 + }, + { + "clip_ratio": 0.0, + "completion_length": 153.5078125, + "epoch": 0.4261168384879725, + "grad_norm": 1.6084405058813327, + "kl": 0.02093505859375, + "learning_rate": 7.869415807560136e-07, + "loss": 0.0008, + "reward": 1.843230962753296, + "reward_std": 0.11450556665658951, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8432309925556183, + "step": 496 + }, + { + "clip_ratio": 0.0, + "completion_length": 142.6015625, + "epoch": 0.4269759450171821, + "grad_norm": 0.9098919677048347, + "kl": 0.0244140625, + "learning_rate": 7.865120274914089e-07, + "loss": 0.001, + "reward": 1.8330826163291931, + "reward_std": 0.13003884255886078, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8330826163291931, + "step": 497 + }, + { + "clip_ratio": 0.0, + "completion_length": 139.03125, + "epoch": 0.42783505154639173, + "grad_norm": 1.1077984476897802, + "kl": 0.0185546875, + "learning_rate": 7.860824742268041e-07, + "loss": 0.0007, + "reward": 1.7646691799163818, + "reward_std": 0.09061707556247711, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7646692395210266, + "step": 498 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.5, + "epoch": 0.42869415807560135, + "grad_norm": 1.5301258115042293, + "kl": 0.021728515625, + "learning_rate": 7.856529209621993e-07, + "loss": 0.0009, + "reward": 1.8485521078109741, + "reward_std": 0.14281748235225677, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8485520780086517, + "step": 499 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.15625, + "epoch": 0.42955326460481097, + "grad_norm": 0.6854626662078912, + "kl": 0.0198974609375, + "learning_rate": 7.852233676975945e-07, + "loss": 0.0008, + "reward": 1.8705613613128662, + "reward_std": 0.057761300355196, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8705613017082214, + "step": 500 + }, + { + "clip_ratio": 0.0, + "completion_length": 156.5, + "epoch": 0.43041237113402064, + "grad_norm": 4.684754333390039, + "kl": 0.02490234375, + "learning_rate": 7.847938144329897e-07, + "loss": 0.001, + "reward": 1.7904819250106812, + "reward_std": 0.11770961433649063, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.790481835603714, + "step": 501 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.1015625, + "epoch": 0.43127147766323026, + "grad_norm": 1.3603984953615473, + "kl": 0.0272216796875, + "learning_rate": 7.843642611683849e-07, + "loss": 0.0011, + "reward": 1.8305404782295227, + "reward_std": 0.08736425265669823, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8305404186248779, + "step": 502 + }, + { + "clip_ratio": 0.0, + "completion_length": 154.171875, + "epoch": 0.4321305841924399, + "grad_norm": 1.1418674217309672, + "kl": 0.026123046875, + "learning_rate": 7.839347079037801e-07, + "loss": 0.001, + "reward": 1.7259817123413086, + "reward_std": 0.16148880869150162, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7416067719459534, + "step": 503 + }, + { + "clip_ratio": 0.0, + "completion_length": 154.3359375, + "epoch": 0.4329896907216495, + "grad_norm": 1.1135192490403159, + "kl": 0.0301513671875, + "learning_rate": 7.835051546391752e-07, + "loss": 0.0012, + "reward": 1.675801932811737, + "reward_std": 0.10446542128920555, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.6758019924163818, + "step": 504 + }, + { + "clip_ratio": 0.0, + "completion_length": 158.8984375, + "epoch": 0.4338487972508591, + "grad_norm": 1.0026442135683145, + "kl": 0.02264404296875, + "learning_rate": 7.830756013745704e-07, + "loss": 0.0009, + "reward": 1.753090739250183, + "reward_std": 0.11342515423893929, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7687157094478607, + "step": 505 + }, + { + "clip_ratio": 0.0, + "completion_length": 156.4609375, + "epoch": 0.43470790378006874, + "grad_norm": 0.63327553743734, + "kl": 0.0238037109375, + "learning_rate": 7.826460481099656e-07, + "loss": 0.001, + "reward": 1.7929630875587463, + "reward_std": 0.06263483129441738, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8007755875587463, + "step": 506 + }, + { + "clip_ratio": 0.0, + "completion_length": 145.015625, + "epoch": 0.43556701030927836, + "grad_norm": 1.1405571467224787, + "kl": 0.02972412109375, + "learning_rate": 7.822164948453608e-07, + "loss": 0.0012, + "reward": 1.8316816091537476, + "reward_std": 0.13786949962377548, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8394941091537476, + "step": 507 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.7109375, + "epoch": 0.436426116838488, + "grad_norm": 2.491831127237855, + "kl": 0.0272216796875, + "learning_rate": 7.81786941580756e-07, + "loss": 0.0011, + "reward": 1.8621906638145447, + "reward_std": 0.08054150827229023, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8621906936168671, + "step": 508 + }, + { + "clip_ratio": 0.0, + "completion_length": 145.6640625, + "epoch": 0.4372852233676976, + "grad_norm": 5.202435035199377, + "kl": 0.02520751953125, + "learning_rate": 7.813573883161512e-07, + "loss": 0.001, + "reward": 1.6291198134422302, + "reward_std": 0.2496250867843628, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.6447448432445526, + "step": 509 + }, + { + "clip_ratio": 0.0, + "completion_length": 168.59375, + "epoch": 0.4381443298969072, + "grad_norm": 1.6848389247112312, + "kl": 0.02227783203125, + "learning_rate": 7.809278350515464e-07, + "loss": 0.0009, + "reward": 1.8260024189949036, + "reward_std": 0.0773128978908062, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8260024785995483, + "step": 510 + }, + { + "clip_ratio": 0.0, + "completion_length": 153.8671875, + "epoch": 0.43900343642611683, + "grad_norm": 1.2536511830670496, + "kl": 0.04034423828125, + "learning_rate": 7.804982817869416e-07, + "loss": 0.0016, + "reward": 1.8234487175941467, + "reward_std": 0.12906832993030548, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8390736877918243, + "step": 511 + }, + { + "clip_ratio": 0.0, + "completion_length": 162.9609375, + "epoch": 0.43986254295532645, + "grad_norm": 0.9056562066002685, + "kl": 0.0228271484375, + "learning_rate": 7.800687285223368e-07, + "loss": 0.0009, + "reward": 1.8035598993301392, + "reward_std": 0.10758433863520622, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.819184809923172, + "step": 512 + }, + { + "clip_ratio": 0.0, + "completion_length": 160.625, + "epoch": 0.44072164948453607, + "grad_norm": 1.9749807757861202, + "kl": 0.0284423828125, + "learning_rate": 7.796391752577319e-07, + "loss": 0.0011, + "reward": 1.7375337481498718, + "reward_std": 0.12667454779148102, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7375337779521942, + "step": 513 + }, + { + "clip_ratio": 0.0, + "completion_length": 177.203125, + "epoch": 0.4415807560137457, + "grad_norm": 2.7049577625952863, + "kl": 0.025634765625, + "learning_rate": 7.792096219931271e-07, + "loss": 0.001, + "reward": 1.665591835975647, + "reward_std": 0.1844305470585823, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.6890293061733246, + "step": 514 + }, + { + "clip_ratio": 0.0, + "completion_length": 176.421875, + "epoch": 0.4424398625429553, + "grad_norm": 1.1503646554833182, + "kl": 0.020751953125, + "learning_rate": 7.787800687285223e-07, + "loss": 0.0008, + "reward": 1.7526438236236572, + "reward_std": 0.15297304093837738, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7526438236236572, + "step": 515 + }, + { + "clip_ratio": 0.0, + "completion_length": 174.8203125, + "epoch": 0.44329896907216493, + "grad_norm": 2.528762349405692, + "kl": 0.0223388671875, + "learning_rate": 7.783505154639175e-07, + "loss": 0.0009, + "reward": 1.7877247333526611, + "reward_std": 0.1644352711737156, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7877247333526611, + "step": 516 + }, + { + "clip_ratio": 0.0, + "completion_length": 161.5859375, + "epoch": 0.44415807560137455, + "grad_norm": 1.0883366488533088, + "kl": 0.02557373046875, + "learning_rate": 7.779209621993127e-07, + "loss": 0.001, + "reward": 1.9020284414291382, + "reward_std": 0.0875653550028801, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.9098409414291382, + "step": 517 + }, + { + "clip_ratio": 0.0, + "completion_length": 168.234375, + "epoch": 0.44501718213058417, + "grad_norm": 1.238117408635782, + "kl": 0.02325439453125, + "learning_rate": 7.774914089347079e-07, + "loss": 0.0009, + "reward": 1.7550345659255981, + "reward_std": 0.11340263113379478, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7706595659255981, + "step": 518 + }, + { + "clip_ratio": 0.0, + "completion_length": 161.953125, + "epoch": 0.44587628865979384, + "grad_norm": 3.2324811204812818, + "kl": 0.02435302734375, + "learning_rate": 7.770618556701031e-07, + "loss": 0.001, + "reward": 1.7546863555908203, + "reward_std": 0.15567786246538162, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7624987959861755, + "step": 519 + }, + { + "clip_ratio": 0.0, + "completion_length": 159.5390625, + "epoch": 0.44673539518900346, + "grad_norm": 1.1906445700635893, + "kl": 0.0238037109375, + "learning_rate": 7.766323024054983e-07, + "loss": 0.001, + "reward": 1.770099401473999, + "reward_std": 0.15499432384967804, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7700993120670319, + "step": 520 + }, + { + "clip_ratio": 0.0, + "completion_length": 155.3359375, + "epoch": 0.4475945017182131, + "grad_norm": 1.4289037451333348, + "kl": 0.02899169921875, + "learning_rate": 7.762027491408934e-07, + "loss": 0.0012, + "reward": 1.7591895461082458, + "reward_std": 0.20220742374658585, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7826270163059235, + "step": 521 + }, + { + "clip_ratio": 0.0, + "completion_length": 153.859375, + "epoch": 0.4484536082474227, + "grad_norm": 0.9798567354541009, + "kl": 0.0318603515625, + "learning_rate": 7.757731958762886e-07, + "loss": 0.0013, + "reward": 1.8011258244514465, + "reward_std": 0.1115933284163475, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8089383244514465, + "step": 522 + }, + { + "clip_ratio": 0.0, + "completion_length": 162.453125, + "epoch": 0.4493127147766323, + "grad_norm": 0.9870529145672341, + "kl": 0.0245361328125, + "learning_rate": 7.753436426116838e-07, + "loss": 0.001, + "reward": 1.7835025191307068, + "reward_std": 0.10335457697510719, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7913150489330292, + "step": 523 + }, + { + "clip_ratio": 0.0, + "completion_length": 149.1796875, + "epoch": 0.45017182130584193, + "grad_norm": 1.590053867640443, + "kl": 0.02679443359375, + "learning_rate": 7.74914089347079e-07, + "loss": 0.0011, + "reward": 1.6806529760360718, + "reward_std": 0.12486770376563072, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.6962779760360718, + "step": 524 + }, + { + "clip_ratio": 0.0, + "completion_length": 150.203125, + "epoch": 0.45103092783505155, + "grad_norm": 1.288976196272062, + "kl": 0.0467529296875, + "learning_rate": 7.744845360824742e-07, + "loss": 0.0019, + "reward": 1.7338435053825378, + "reward_std": 0.21027009189128876, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7494684457778931, + "step": 525 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.875, + "epoch": 0.4518900343642612, + "grad_norm": 1.3796315870979017, + "kl": 0.03314208984375, + "learning_rate": 7.740549828178694e-07, + "loss": 0.0013, + "reward": 1.813571572303772, + "reward_std": 0.11932007595896721, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8135715425014496, + "step": 526 + }, + { + "clip_ratio": 0.0, + "completion_length": 155.8359375, + "epoch": 0.4527491408934708, + "grad_norm": 1.0856638820035733, + "kl": 0.02392578125, + "learning_rate": 7.736254295532646e-07, + "loss": 0.001, + "reward": 1.7808946371078491, + "reward_std": 0.15631069988012314, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7887071967124939, + "step": 527 + }, + { + "clip_ratio": 0.0, + "completion_length": 139.53125, + "epoch": 0.4536082474226804, + "grad_norm": 1.2741002088470732, + "kl": 0.0306396484375, + "learning_rate": 7.731958762886598e-07, + "loss": 0.0012, + "reward": 1.7552229762077332, + "reward_std": 0.16508981585502625, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7708480358123779, + "step": 528 + }, + { + "clip_ratio": 0.0, + "completion_length": 163.84375, + "epoch": 0.45446735395189003, + "grad_norm": 0.8166293208289542, + "kl": 0.0262451171875, + "learning_rate": 7.72766323024055e-07, + "loss": 0.0011, + "reward": 1.763866662979126, + "reward_std": 0.11001107096672058, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.771679162979126, + "step": 529 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.6171875, + "epoch": 0.45532646048109965, + "grad_norm": 1.0906821881418176, + "kl": 0.0263671875, + "learning_rate": 7.723367697594501e-07, + "loss": 0.0011, + "reward": 1.7570544481277466, + "reward_std": 0.09293561428785324, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7570544183254242, + "step": 530 + }, + { + "clip_ratio": 0.0, + "completion_length": 152.1328125, + "epoch": 0.45618556701030927, + "grad_norm": 0.6788651581622658, + "kl": 0.0257568359375, + "learning_rate": 7.719072164948453e-07, + "loss": 0.001, + "reward": 1.8436721563339233, + "reward_std": 0.05900268629193306, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8514846563339233, + "step": 531 + }, + { + "clip_ratio": 0.0, + "completion_length": 153.203125, + "epoch": 0.4570446735395189, + "grad_norm": 1.3523482508260602, + "kl": 0.02862548828125, + "learning_rate": 7.714776632302405e-07, + "loss": 0.0011, + "reward": 1.7728696465492249, + "reward_std": 0.0960187017917633, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7728695869445801, + "step": 532 + }, + { + "clip_ratio": 0.0, + "completion_length": 157.2578125, + "epoch": 0.4579037800687285, + "grad_norm": 2.080449621498438, + "kl": 0.02349853515625, + "learning_rate": 7.710481099656357e-07, + "loss": 0.0009, + "reward": 1.7926985025405884, + "reward_std": 0.10765637829899788, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7926985621452332, + "step": 533 + }, + { + "clip_ratio": 0.0, + "completion_length": 157.546875, + "epoch": 0.4587628865979381, + "grad_norm": 0.9990312365514723, + "kl": 0.030029296875, + "learning_rate": 7.706185567010309e-07, + "loss": 0.0012, + "reward": 1.6698935627937317, + "reward_std": 0.1365356780588627, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6777060627937317, + "step": 534 + }, + { + "clip_ratio": 0.0, + "completion_length": 147.921875, + "epoch": 0.45962199312714774, + "grad_norm": 1.423339665969349, + "kl": 0.02587890625, + "learning_rate": 7.701890034364261e-07, + "loss": 0.001, + "reward": 1.7801197171211243, + "reward_std": 0.12286467850208282, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.780119776725769, + "step": 535 + }, + { + "clip_ratio": 0.0, + "completion_length": 155.1328125, + "epoch": 0.46048109965635736, + "grad_norm": 2.0644152976523924, + "kl": 0.0234375, + "learning_rate": 7.697594501718213e-07, + "loss": 0.0009, + "reward": 1.8145374059677124, + "reward_std": 0.15781568735837936, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8223499655723572, + "step": 536 + }, + { + "clip_ratio": 0.0, + "completion_length": 153.828125, + "epoch": 0.46134020618556704, + "grad_norm": 2.419979165046296, + "kl": 0.03106689453125, + "learning_rate": 7.693298969072165e-07, + "loss": 0.0012, + "reward": 1.7789837718009949, + "reward_std": 0.1027519591152668, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7789837718009949, + "step": 537 + }, + { + "clip_ratio": 0.0, + "completion_length": 153.5859375, + "epoch": 0.46219931271477666, + "grad_norm": 0.9119452557469206, + "kl": 0.02142333984375, + "learning_rate": 7.689003436426116e-07, + "loss": 0.0009, + "reward": 1.8047268986701965, + "reward_std": 0.0921391174197197, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8047268688678741, + "step": 538 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.75, + "epoch": 0.4630584192439863, + "grad_norm": 1.0752489048608882, + "kl": 0.02978515625, + "learning_rate": 7.684707903780069e-07, + "loss": 0.0012, + "reward": 1.7013397812843323, + "reward_std": 0.10420958697795868, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7013397514820099, + "step": 539 + }, + { + "clip_ratio": 0.0, + "completion_length": 142.21875, + "epoch": 0.4639175257731959, + "grad_norm": 1.879235291737055, + "kl": 0.03424072265625, + "learning_rate": 7.680412371134021e-07, + "loss": 0.0014, + "reward": 1.7587386965751648, + "reward_std": 0.12074919883161783, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7665511965751648, + "step": 540 + }, + { + "clip_ratio": 0.0, + "completion_length": 144.1640625, + "epoch": 0.4647766323024055, + "grad_norm": 1.0694029027143328, + "kl": 0.02728271484375, + "learning_rate": 7.676116838487973e-07, + "loss": 0.0011, + "reward": 1.7316040396690369, + "reward_std": 0.1069042906165123, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7316040396690369, + "step": 541 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.390625, + "epoch": 0.46563573883161513, + "grad_norm": 1.0560010325147446, + "kl": 0.027099609375, + "learning_rate": 7.671821305841925e-07, + "loss": 0.0011, + "reward": 1.7262669205665588, + "reward_std": 0.1186705082654953, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7340793907642365, + "step": 542 + }, + { + "clip_ratio": 0.0, + "completion_length": 153.375, + "epoch": 0.46649484536082475, + "grad_norm": 0.8831242345243586, + "kl": 0.02362060546875, + "learning_rate": 7.667525773195877e-07, + "loss": 0.0009, + "reward": 1.8377864360809326, + "reward_std": 0.07232390902936459, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8455990254878998, + "step": 543 + }, + { + "clip_ratio": 0.0, + "completion_length": 150.5625, + "epoch": 0.46735395189003437, + "grad_norm": 0.808526359455059, + "kl": 0.025634765625, + "learning_rate": 7.663230240549829e-07, + "loss": 0.001, + "reward": 1.8943662643432617, + "reward_std": 0.08304097317159176, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8943662643432617, + "step": 544 + }, + { + "clip_ratio": 0.0, + "completion_length": 143.8359375, + "epoch": 0.468213058419244, + "grad_norm": 1.2364679060956676, + "kl": 0.03729248046875, + "learning_rate": 7.658934707903781e-07, + "loss": 0.0015, + "reward": 1.820011556148529, + "reward_std": 0.10274770110845566, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8278240263462067, + "step": 545 + }, + { + "clip_ratio": 0.0, + "completion_length": 165.7421875, + "epoch": 0.4690721649484536, + "grad_norm": 1.4203606446988248, + "kl": 0.02764892578125, + "learning_rate": 7.654639175257731e-07, + "loss": 0.0011, + "reward": 1.7455325722694397, + "reward_std": 0.08128118142485619, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7455325424671173, + "step": 546 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.140625, + "epoch": 0.4699312714776632, + "grad_norm": 2.804119845115769, + "kl": 0.03338623046875, + "learning_rate": 7.650343642611683e-07, + "loss": 0.0013, + "reward": 1.6912947297096252, + "reward_std": 0.1474270112812519, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.71473228931427, + "step": 547 + }, + { + "clip_ratio": 0.0, + "completion_length": 149.09375, + "epoch": 0.47079037800687284, + "grad_norm": 1.9971642189793117, + "kl": 0.07635498046875, + "learning_rate": 7.646048109965635e-07, + "loss": 0.0031, + "reward": 1.7125971913337708, + "reward_std": 0.17911146581172943, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.7360346615314484, + "step": 548 + }, + { + "clip_ratio": 0.0, + "completion_length": 142.2734375, + "epoch": 0.47164948453608246, + "grad_norm": 2.745551651788921, + "kl": 0.02740478515625, + "learning_rate": 7.641752577319587e-07, + "loss": 0.0011, + "reward": 1.8493961691856384, + "reward_std": 0.10139979794621468, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8493961095809937, + "step": 549 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.359375, + "epoch": 0.4725085910652921, + "grad_norm": 3.8143164658012823, + "kl": 0.02508544921875, + "learning_rate": 7.637457044673539e-07, + "loss": 0.001, + "reward": 1.717983603477478, + "reward_std": 0.11418138444423676, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7336086332798004, + "step": 550 + }, + { + "clip_ratio": 0.0, + "completion_length": 142.8671875, + "epoch": 0.4733676975945017, + "grad_norm": 1.0145675064339164, + "kl": 0.02392578125, + "learning_rate": 7.633161512027491e-07, + "loss": 0.001, + "reward": 1.7650344371795654, + "reward_std": 0.07981048338115215, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7728469967842102, + "step": 551 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.8515625, + "epoch": 0.4742268041237113, + "grad_norm": 18.881776716124634, + "kl": 0.0201416015625, + "learning_rate": 7.628865979381443e-07, + "loss": 0.0008, + "reward": 1.7701881527900696, + "reward_std": 0.17208566516637802, + "rewards/format_reward_gen": 0.953125, + "rewards/llm_reward": 0.817063182592392, + "step": 552 + }, + { + "clip_ratio": 0.0, + "completion_length": 148.9765625, + "epoch": 0.47508591065292094, + "grad_norm": 2.102100470034523, + "kl": 0.02630615234375, + "learning_rate": 7.624570446735395e-07, + "loss": 0.0011, + "reward": 1.8304003477096558, + "reward_std": 0.1711219921708107, + "rewards/format_reward_gen": 0.921875, + "rewards/llm_reward": 0.908525288105011, + "step": 553 + }, + { + "clip_ratio": 0.0, + "completion_length": 152.96875, + "epoch": 0.47594501718213056, + "grad_norm": 1.444717715857585, + "kl": 0.021728515625, + "learning_rate": 7.620274914089346e-07, + "loss": 0.0009, + "reward": 1.806252360343933, + "reward_std": 0.1447359174489975, + "rewards/format_reward_gen": 0.953125, + "rewards/llm_reward": 0.8531273901462555, + "step": 554 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.8828125, + "epoch": 0.47680412371134023, + "grad_norm": 2.156085264005566, + "kl": 0.02764892578125, + "learning_rate": 7.615979381443298e-07, + "loss": 0.0011, + "reward": 1.6459792852401733, + "reward_std": 0.2492372989654541, + "rewards/format_reward_gen": 0.921875, + "rewards/llm_reward": 0.7241042256355286, + "step": 555 + }, + { + "clip_ratio": 0.0, + "completion_length": 139.4453125, + "epoch": 0.47766323024054985, + "grad_norm": 2.1944379120092123, + "kl": 0.0213623046875, + "learning_rate": 7.61168384879725e-07, + "loss": 0.0009, + "reward": 1.7091248035430908, + "reward_std": 0.16284260153770447, + "rewards/format_reward_gen": 0.953125, + "rewards/llm_reward": 0.7559998035430908, + "step": 556 + }, + { + "clip_ratio": 0.0, + "completion_length": 139.359375, + "epoch": 0.47852233676975947, + "grad_norm": 1.607470492713128, + "kl": 0.0281982421875, + "learning_rate": 7.607388316151202e-07, + "loss": 0.0011, + "reward": 1.687252402305603, + "reward_std": 0.16054657474160194, + "rewards/format_reward_gen": 0.9765625, + "rewards/llm_reward": 0.710689902305603, + "step": 557 + }, + { + "clip_ratio": 0.0, + "completion_length": 125.53125, + "epoch": 0.4793814432989691, + "grad_norm": 0.9757519931508962, + "kl": 0.0283203125, + "learning_rate": 7.603092783505154e-07, + "loss": 0.0011, + "reward": 1.8052751421928406, + "reward_std": 0.05359087511897087, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8209001123905182, + "step": 558 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.125, + "epoch": 0.4802405498281787, + "grad_norm": 1.657372250071319, + "kl": 0.03369140625, + "learning_rate": 7.598797250859106e-07, + "loss": 0.0013, + "reward": 1.6079121828079224, + "reward_std": 0.22965000569820404, + "rewards/format_reward_gen": 0.9140625, + "rewards/llm_reward": 0.6938497424125671, + "step": 559 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.34375, + "epoch": 0.48109965635738833, + "grad_norm": 1.3334146432657494, + "kl": 0.0301513671875, + "learning_rate": 7.594501718213058e-07, + "loss": 0.0012, + "reward": 1.724092185497284, + "reward_std": 0.12364434078335762, + "rewards/format_reward_gen": 0.9609375, + "rewards/llm_reward": 0.7631546556949615, + "step": 560 + }, + { + "clip_ratio": 0.0, + "completion_length": 107.3515625, + "epoch": 0.48195876288659795, + "grad_norm": 1.3103330552745027, + "kl": 0.03033447265625, + "learning_rate": 7.59020618556701e-07, + "loss": 0.0012, + "reward": 1.7695825695991516, + "reward_std": 0.16746646538376808, + "rewards/format_reward_gen": 0.9609375, + "rewards/llm_reward": 0.8086450695991516, + "step": 561 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.21875, + "epoch": 0.48281786941580757, + "grad_norm": 3.4007488158636634, + "kl": 0.0413818359375, + "learning_rate": 7.585910652920962e-07, + "loss": 0.0017, + "reward": 1.8109328150749207, + "reward_std": 0.17322580516338348, + "rewards/format_reward_gen": 0.9609375, + "rewards/llm_reward": 0.8499953150749207, + "step": 562 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.859375, + "epoch": 0.4836769759450172, + "grad_norm": 4.0862178779047085, + "kl": 0.02703857421875, + "learning_rate": 7.581615120274913e-07, + "loss": 0.0011, + "reward": 1.7667149305343628, + "reward_std": 0.08758777007460594, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7667149305343628, + "step": 563 + }, + { + "clip_ratio": 0.0, + "completion_length": 122.5859375, + "epoch": 0.4845360824742268, + "grad_norm": 1.073396750076447, + "kl": 0.0284423828125, + "learning_rate": 7.577319587628865e-07, + "loss": 0.0011, + "reward": 1.7671363949775696, + "reward_std": 0.10686640068888664, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.774948924779892, + "step": 564 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.7109375, + "epoch": 0.4853951890034364, + "grad_norm": 1.6082673896426019, + "kl": 0.02496337890625, + "learning_rate": 7.573024054982817e-07, + "loss": 0.001, + "reward": 1.7323017120361328, + "reward_std": 0.11195755004882812, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7479267120361328, + "step": 565 + }, + { + "clip_ratio": 0.0, + "completion_length": 122.03125, + "epoch": 0.48625429553264604, + "grad_norm": 1.3078742427842147, + "kl": 0.02801513671875, + "learning_rate": 7.568728522336769e-07, + "loss": 0.0011, + "reward": 1.7409408688545227, + "reward_std": 0.11844996362924576, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7487533390522003, + "step": 566 + }, + { + "clip_ratio": 0.0, + "completion_length": 118.53125, + "epoch": 0.48711340206185566, + "grad_norm": 1.0128900337930118, + "kl": 0.0277099609375, + "learning_rate": 7.564432989690721e-07, + "loss": 0.0011, + "reward": 1.7667198181152344, + "reward_std": 0.09644139185547829, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7745323181152344, + "step": 567 + }, + { + "clip_ratio": 0.0, + "completion_length": 101.28125, + "epoch": 0.4879725085910653, + "grad_norm": 1.2158578490452436, + "kl": 0.0233154296875, + "learning_rate": 7.560137457044673e-07, + "loss": 0.0009, + "reward": 1.8185061812400818, + "reward_std": 0.07172945514321327, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8185062408447266, + "step": 568 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.78125, + "epoch": 0.4888316151202749, + "grad_norm": 1.1584871999787036, + "kl": 0.0294189453125, + "learning_rate": 7.555841924398625e-07, + "loss": 0.0012, + "reward": 1.6857864260673523, + "reward_std": 0.0634692870080471, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.6935989856719971, + "step": 569 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.8984375, + "epoch": 0.4896907216494845, + "grad_norm": 1.5831208673085706, + "kl": 0.02850341796875, + "learning_rate": 7.551546391752577e-07, + "loss": 0.0011, + "reward": 1.7977246046066284, + "reward_std": 0.17054561525583267, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.805537074804306, + "step": 570 + }, + { + "clip_ratio": 0.0, + "completion_length": 120.5078125, + "epoch": 0.49054982817869414, + "grad_norm": 0.7928407917060947, + "kl": 0.0238037109375, + "learning_rate": 7.547250859106528e-07, + "loss": 0.001, + "reward": 1.7591076493263245, + "reward_std": 0.0366393206641078, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7669200897216797, + "step": 571 + }, + { + "clip_ratio": 0.0, + "completion_length": 134.4609375, + "epoch": 0.49140893470790376, + "grad_norm": 0.8346365718982691, + "kl": 0.02227783203125, + "learning_rate": 7.54295532646048e-07, + "loss": 0.0009, + "reward": 1.826495349407196, + "reward_std": 0.05793471448123455, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8264953792095184, + "step": 572 + }, + { + "clip_ratio": 0.0, + "completion_length": 114.6484375, + "epoch": 0.49226804123711343, + "grad_norm": 1.852024426080103, + "kl": 0.021484375, + "learning_rate": 7.538659793814432e-07, + "loss": 0.0009, + "reward": 1.8043755292892456, + "reward_std": 0.18020495027303696, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.804375559091568, + "step": 573 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.8203125, + "epoch": 0.49312714776632305, + "grad_norm": 26.47121113659565, + "kl": 0.02374267578125, + "learning_rate": 7.534364261168384e-07, + "loss": 0.0009, + "reward": 1.827117145061493, + "reward_std": 0.10837876796722412, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8271171748638153, + "step": 574 + }, + { + "clip_ratio": 0.0, + "completion_length": 124.234375, + "epoch": 0.49398625429553267, + "grad_norm": 1.1310521518717915, + "kl": 0.02288818359375, + "learning_rate": 7.530068728522336e-07, + "loss": 0.0009, + "reward": 1.797199308872223, + "reward_std": 0.10720982775092125, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8128242492675781, + "step": 575 + }, + { + "clip_ratio": 0.0, + "completion_length": 120.9453125, + "epoch": 0.4948453608247423, + "grad_norm": 1.8553971162459708, + "kl": 0.02252197265625, + "learning_rate": 7.525773195876288e-07, + "loss": 0.0009, + "reward": 1.79475736618042, + "reward_std": 0.13686570525169373, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7947573065757751, + "step": 576 + }, + { + "clip_ratio": 0.0, + "completion_length": 135.5703125, + "epoch": 0.4957044673539519, + "grad_norm": 1.1152662401016404, + "kl": 0.026611328125, + "learning_rate": 7.52147766323024e-07, + "loss": 0.0011, + "reward": 1.858827292919159, + "reward_std": 0.15026213228702545, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8666398227214813, + "step": 577 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.15625, + "epoch": 0.4965635738831615, + "grad_norm": 0.7945279843022707, + "kl": 0.0247802734375, + "learning_rate": 7.517182130584192e-07, + "loss": 0.001, + "reward": 1.8583158254623413, + "reward_std": 0.08704648166894913, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8739407956600189, + "step": 578 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.015625, + "epoch": 0.49742268041237114, + "grad_norm": 0.8545181390908309, + "kl": 0.02587890625, + "learning_rate": 7.512886597938143e-07, + "loss": 0.001, + "reward": 1.794776201248169, + "reward_std": 0.07284862734377384, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8025886416435242, + "step": 579 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.828125, + "epoch": 0.49828178694158076, + "grad_norm": 5.702979853199694, + "kl": 0.02679443359375, + "learning_rate": 7.508591065292095e-07, + "loss": 0.0011, + "reward": 1.7567557096481323, + "reward_std": 0.10015341639518738, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7567557394504547, + "step": 580 + }, + { + "clip_ratio": 0.0, + "completion_length": 143.1796875, + "epoch": 0.4991408934707904, + "grad_norm": 3.2034971522784796, + "kl": 0.0230712890625, + "learning_rate": 7.504295532646047e-07, + "loss": 0.0009, + "reward": 1.8063377737998962, + "reward_std": 0.11063193529844284, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8063377737998962, + "step": 581 + }, + { + "clip_ratio": 0.0, + "completion_length": 129.734375, + "epoch": 0.5, + "grad_norm": 0.7129779180772821, + "kl": 0.0211181640625, + "learning_rate": 7.5e-07, + "loss": 0.0008, + "reward": 1.8224888443946838, + "reward_std": 0.056552507914602757, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8303013443946838, + "step": 582 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.0625, + "epoch": 0.5008591065292096, + "grad_norm": 2.78986309606567, + "kl": 0.0185546875, + "learning_rate": 7.495704467353952e-07, + "loss": 0.0007, + "reward": 1.854720413684845, + "reward_std": 0.10461413115262985, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8547204732894897, + "step": 583 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.1484375, + "epoch": 0.5017182130584192, + "grad_norm": 1.0824827966215953, + "kl": 0.02435302734375, + "learning_rate": 7.491408934707904e-07, + "loss": 0.001, + "reward": 1.7764517068862915, + "reward_std": 0.10467999801039696, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.7920766472816467, + "step": 584 + }, + { + "clip_ratio": 0.0, + "completion_length": 128.765625, + "epoch": 0.5025773195876289, + "grad_norm": 1.252463130344048, + "kl": 0.02105712890625, + "learning_rate": 7.487113402061856e-07, + "loss": 0.0008, + "reward": 1.8154524564743042, + "reward_std": 0.1660311445593834, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.8310775756835938, + "step": 585 + }, + { + "clip_ratio": 0.0, + "completion_length": 119.734375, + "epoch": 0.5034364261168385, + "grad_norm": 0.9304730717779914, + "kl": 0.023681640625, + "learning_rate": 7.482817869415808e-07, + "loss": 0.0009, + "reward": 1.8320194482803345, + "reward_std": 0.07787203788757324, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8398319780826569, + "step": 586 + }, + { + "clip_ratio": 0.0, + "completion_length": 151.5703125, + "epoch": 0.5042955326460481, + "grad_norm": 5.260833868032736, + "kl": 0.0369873046875, + "learning_rate": 7.47852233676976e-07, + "loss": 0.0015, + "reward": 1.7589982748031616, + "reward_std": 0.14544661343097687, + "rewards/format_reward_gen": 0.984375, + "rewards/llm_reward": 0.774623304605484, + "step": 587 + }, + { + "clip_ratio": 0.0, + "completion_length": 132.953125, + "epoch": 0.5051546391752577, + "grad_norm": 1.2660610390516958, + "kl": 0.031005859375, + "learning_rate": 7.474226804123711e-07, + "loss": 0.0012, + "reward": 1.822842538356781, + "reward_std": 0.13162581250071526, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8306550085544586, + "step": 588 + }, + { + "clip_ratio": 0.0, + "completion_length": 158.7578125, + "epoch": 0.5060137457044673, + "grad_norm": 0.9382817085600501, + "kl": 0.0223388671875, + "learning_rate": 7.469931271477663e-07, + "loss": 0.0009, + "reward": 1.7842236161231995, + "reward_std": 0.11218691617250443, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7842236161231995, + "step": 589 + }, + { + "clip_ratio": 0.0, + "completion_length": 141.609375, + "epoch": 0.506872852233677, + "grad_norm": 1.5108030501592749, + "kl": 0.0247802734375, + "learning_rate": 7.465635738831615e-07, + "loss": 0.001, + "reward": 1.8379915952682495, + "reward_std": 0.11418896913528442, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.8458041250705719, + "step": 590 + }, + { + "clip_ratio": 0.0, + "completion_length": 131.6640625, + "epoch": 0.5077319587628866, + "grad_norm": 0.9756789130203551, + "kl": 0.02655029296875, + "learning_rate": 7.461340206185567e-07, + "loss": 0.0011, + "reward": 1.8145712018013, + "reward_std": 0.09588093496859074, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8145712018013, + "step": 591 + }, + { + "clip_ratio": 0.0, + "completion_length": 136.1953125, + "epoch": 0.5085910652920962, + "grad_norm": 1.78737635682799, + "kl": 0.02001953125, + "learning_rate": 7.457044673539519e-07, + "loss": 0.0008, + "reward": 1.7849004864692688, + "reward_std": 0.09064675122499466, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7927128970623016, + "step": 592 + }, + { + "clip_ratio": 0.0, + "completion_length": 130.0234375, + "epoch": 0.5094501718213058, + "grad_norm": 1.5323474545376934, + "kl": 0.0244140625, + "learning_rate": 7.452749140893471e-07, + "loss": 0.001, + "reward": 1.8163361549377441, + "reward_std": 0.06137897726148367, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.8163362145423889, + "step": 593 + }, + { + "clip_ratio": 0.0, + "completion_length": 146.65625, + "epoch": 0.5103092783505154, + "grad_norm": 1.167714380864763, + "kl": 0.02984619140625, + "learning_rate": 7.448453608247423e-07, + "loss": 0.0012, + "reward": 1.7876654863357544, + "reward_std": 0.1253710240125656, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7954780161380768, + "step": 594 + }, + { + "clip_ratio": 0.0, + "completion_length": 133.375, + "epoch": 0.511168384879725, + "grad_norm": 1.2346413570984562, + "kl": 0.0233154296875, + "learning_rate": 7.444158075601375e-07, + "loss": 0.0009, + "reward": 1.7417802214622498, + "reward_std": 0.1481439284980297, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7417803108692169, + "step": 595 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.828125, + "epoch": 0.5120274914089347, + "grad_norm": 1.4208055330522806, + "kl": 0.0283203125, + "learning_rate": 7.439862542955327e-07, + "loss": 0.0011, + "reward": 1.7719964981079102, + "reward_std": 0.15724200010299683, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7798090577125549, + "step": 596 + }, + { + "clip_ratio": 0.0, + "completion_length": 144.71875, + "epoch": 0.5128865979381443, + "grad_norm": 3.920082332866134, + "kl": 0.02557373046875, + "learning_rate": 7.435567010309278e-07, + "loss": 0.001, + "reward": 1.7663773894309998, + "reward_std": 0.08243504166603088, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7663773894309998, + "step": 597 + }, + { + "clip_ratio": 0.0, + "completion_length": 137.40625, + "epoch": 0.5137457044673539, + "grad_norm": 1.2860662352628833, + "kl": 0.02374267578125, + "learning_rate": 7.43127147766323e-07, + "loss": 0.0009, + "reward": 1.7066620588302612, + "reward_std": 0.06952378898859024, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7144745290279388, + "step": 598 + }, + { + "clip_ratio": 0.0, + "completion_length": 140.2265625, + "epoch": 0.5146048109965635, + "grad_norm": 0.9580097497182469, + "kl": 0.021728515625, + "learning_rate": 7.426975945017182e-07, + "loss": 0.0009, + "reward": 1.7517217993736267, + "reward_std": 0.09551467373967171, + "rewards/format_reward_gen": 1.0, + "rewards/llm_reward": 0.7517217695713043, + "step": 599 + }, + { + "clip_ratio": 0.0, + "completion_length": 153.5625, + "epoch": 0.5154639175257731, + "grad_norm": 1.8465816930490027, + "kl": 0.0545654296875, + "learning_rate": 7.422680412371134e-07, + "loss": 0.0022, + "reward": 1.711679220199585, + "reward_std": 0.10108717158436775, + "rewards/format_reward_gen": 0.9921875, + "rewards/llm_reward": 0.7194916903972626, + "step": 600 + } + ], + "logging_steps": 1.0, + "max_steps": 2328, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}