{
  "best_global_step": 1000,
  "best_metric": 1.8821634650230408,
  "best_model_checkpoint": "/root/chenxr/ms-swift/output/GRPO_LOOK_AGAIN/v4-20250523-142343/checkpoint-1000",
  "epoch": 9.997304582210242,
  "eval_steps": 200,
  "global_step": 1850,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 595.33203125,
      "completions/min_length": 308.5,
      "epoch": 0.005390835579514825,
      "grad_norm": 2.576979390695997,
      "kl": 0.0,
      "learning_rate": 5.2631578947368416e-08,
      "loss": 0.08764700591564178,
      "memory(GiB)": 51.78,
      "reward": 1.4608121514320374,
      "reward_std": 0.7097527980804443,
      "rewards/Format/mean": 0.80078125,
      "rewards/Format/std": 0.40095221996307373,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.330063596367836,
      "rewards/ReflectionBalance/mean": 0.5311246812343597,
      "rewards/ReflectionBalance/std": 0.47368767857551575,
      "step": 1,
      "train_speed(iter/s)": 0.006534
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 582.75390625,
      "completions/min_length": 294.5,
      "epoch": 0.01078167115902965,
      "grad_norm": 2.382263282520165,
      "kl": 0.0,
      "learning_rate": 1.0526315789473683e-07,
      "loss": 0.058551955968141556,
      "memory(GiB)": 75.26,
      "reward": 1.4739943742752075,
      "reward_std": 0.6780318915843964,
      "rewards/Format/mean": 0.79296875,
      "rewards/Format/std": 0.4060489982366562,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.36627575755119324,
      "rewards/ReflectionBalance/mean": 0.5208692848682404,
      "rewards/ReflectionBalance/std": 0.5207047313451767,
      "step": 2,
      "train_speed(iter/s)": 0.007034
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 601.28125,
      "completions/min_length": 280.0,
      "epoch": 0.016172506738544475,
      "grad_norm": 2.4170355809163757,
      "kl": 0.0074310302734375,
      "learning_rate": 1.5789473684210525e-07,
      "loss": 0.06825781613588333,
      "memory(GiB)": 75.26,
      "reward": 1.5351904034614563,
      "reward_std": 0.7064629793167114,
      "rewards/Format/mean": 0.8359375,
      "rewards/Format/std": 0.3704231530427933,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3538244515657425,
      "rewards/ReflectionBalance/mean": 0.5430029630661011,
      "rewards/ReflectionBalance/std": 0.4752453714609146,
      "step": 3,
      "train_speed(iter/s)": 0.007075
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 577.8515625,
      "completions/min_length": 296.5,
      "epoch": 0.0215633423180593,
      "grad_norm": 2.4513983712861287,
      "kl": 0.0069427490234375,
      "learning_rate": 2.1052631578947366e-07,
      "loss": 0.03562547639012337,
      "memory(GiB)": 75.26,
      "reward": 1.5991138219833374,
      "reward_std": 0.6182156801223755,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.36045311391353607,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41857391595840454,
      "rewards/ReflectionBalance/mean": 0.5288012623786926,
      "rewards/ReflectionBalance/std": 0.48531265556812286,
      "step": 4,
      "train_speed(iter/s)": 0.007195
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 624.69140625,
      "completions/min_length": 274.0,
      "epoch": 0.026954177897574125,
      "grad_norm": 2.37145912975243,
      "kl": 0.0071258544921875,
      "learning_rate": 2.631578947368421e-07,
      "loss": 0.07935070246458054,
      "memory(GiB)": 75.26,
      "reward": 1.2977687120437622,
      "reward_std": 0.7675691843032837,
      "rewards/Format/mean": 0.7890625,
      "rewards/Format/std": 0.4095771610736847,
      "rewards/MultiModalAccuracyORM/mean": 0.078125,
      "rewards/MultiModalAccuracyORM/std": 0.258706197142601,
      "rewards/ReflectionBalance/mean": 0.4305812120437622,
      "rewards/ReflectionBalance/std": 0.5378593504428864,
      "step": 5,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 565.34375,
      "completions/min_length": 264.5,
      "epoch": 0.03234501347708895,
      "grad_norm": 2.4248945358733414,
      "kl": 0.00909423828125,
      "learning_rate": 3.157894736842105e-07,
      "loss": 0.08406262844800949,
      "memory(GiB)": 75.26,
      "reward": 1.4389241933822632,
      "reward_std": 0.8191309571266174,
      "rewards/Format/mean": 0.80859375,
      "rewards/Format/std": 0.39238542318344116,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3764529526233673,
      "rewards/ReflectionBalance/mean": 0.4584554135799408,
      "rewards/ReflectionBalance/std": 0.5594810545444489,
      "step": 6,
      "train_speed(iter/s)": 0.007176
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 581.7421875,
      "completions/min_length": 269.0,
      "epoch": 0.03773584905660377,
      "grad_norm": 2.7088410306165702,
      "kl": 0.0082855224609375,
      "learning_rate": 3.684210526315789e-07,
      "loss": 0.04906905069947243,
      "memory(GiB)": 75.26,
      "reward": 1.5628197193145752,
      "reward_std": 0.6806899309158325,
      "rewards/Format/mean": 0.796875,
      "rewards/Format/std": 0.3996165096759796,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.40284374356269836,
      "rewards/ReflectionBalance/mean": 0.5628196597099304,
      "rewards/ReflectionBalance/std": 0.4514058977365494,
      "step": 7,
      "train_speed(iter/s)": 0.00717
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 633.75,
      "completions/min_length": 306.5,
      "epoch": 0.0431266846361186,
      "grad_norm": 2.459295492056518,
      "kl": 0.009185791015625,
      "learning_rate": 4.2105263157894733e-07,
      "loss": 0.0789623111486435,
      "memory(GiB)": 75.26,
      "reward": 1.4861979484558105,
      "reward_std": 0.6560050845146179,
      "rewards/Format/mean": 0.8203125,
      "rewards/Format/std": 0.37402482330799103,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.33569003641605377,
      "rewards/ReflectionBalance/mean": 0.5330729186534882,
      "rewards/ReflectionBalance/std": 0.45205916464328766,
      "step": 8,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 605.64453125,
      "completions/min_length": 258.5,
      "epoch": 0.04851752021563342,
      "grad_norm": 2.4374111437788084,
      "kl": 0.0073394775390625,
      "learning_rate": 4.7368421052631574e-07,
      "loss": 0.08665700256824493,
      "memory(GiB)": 75.26,
      "reward": 1.4180679321289062,
      "reward_std": 0.6747447550296783,
      "rewards/Format/mean": 0.81640625,
      "rewards/Format/std": 0.38597656786441803,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.3029807060956955,
      "rewards/ReflectionBalance/mean": 0.5000993013381958,
      "rewards/ReflectionBalance/std": 0.46240587532520294,
      "step": 9,
      "train_speed(iter/s)": 0.007044
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 587.97265625,
      "completions/min_length": 264.0,
      "epoch": 0.05390835579514825,
      "grad_norm": 2.604114093610406,
      "kl": 0.010040283203125,
      "learning_rate": 5.263157894736842e-07,
      "loss": 0.09458405524492264,
      "memory(GiB)": 75.26,
      "reward": 1.495658040046692,
      "reward_std": 0.6541000008583069,
      "rewards/Format/mean": 0.8203125,
      "rewards/Format/std": 0.3842122554779053,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3130832314491272,
      "rewards/ReflectionBalance/mean": 0.5659705400466919,
      "rewards/ReflectionBalance/std": 0.4479058086872101,
      "step": 10,
      "train_speed(iter/s)": 0.00706
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 594.8359375,
      "completions/min_length": 298.0,
      "epoch": 0.05929919137466307,
      "grad_norm": 2.5967041285780263,
      "kl": 0.010223388671875,
      "learning_rate": 5.789473684210526e-07,
      "loss": 0.09127083420753479,
      "memory(GiB)": 75.26,
      "reward": 1.412841796875,
      "reward_std": 0.7146667242050171,
      "rewards/Format/mean": 0.77734375,
      "rewards/Format/std": 0.4154975265264511,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.3021402209997177,
      "rewards/ReflectionBalance/mean": 0.5339355617761612,
      "rewards/ReflectionBalance/std": 0.47991666197776794,
      "step": 11,
      "train_speed(iter/s)": 0.007056
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 577.7890625,
      "completions/min_length": 298.5,
      "epoch": 0.0646900269541779,
      "grad_norm": 2.4622939280627922,
      "kl": 0.0070343017578125,
      "learning_rate": 6.31578947368421e-07,
      "loss": 0.07271189987659454,
      "memory(GiB)": 75.26,
      "reward": 1.401135802268982,
      "reward_std": 0.6699142754077911,
      "rewards/Format/mean": 0.8046875,
      "rewards/Format/std": 0.3935103267431259,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3110392540693283,
      "rewards/ReflectionBalance/mean": 0.48707324266433716,
      "rewards/ReflectionBalance/std": 0.5133813619613647,
      "step": 12,
      "train_speed(iter/s)": 0.00707
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 598.04296875,
      "completions/min_length": 287.0,
      "epoch": 0.07008086253369272,
      "grad_norm": 2.5454050454382795,
      "kl": 0.007659912109375,
      "learning_rate": 6.842105263157895e-07,
      "loss": 0.09292205423116684,
      "memory(GiB)": 75.26,
      "reward": 1.4248135685920715,
      "reward_std": 0.662456214427948,
      "rewards/Format/mean": 0.8046875,
      "rewards/Format/std": 0.3978758305311203,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3643587827682495,
      "rewards/ReflectionBalance/mean": 0.4638761430978775,
      "rewards/ReflectionBalance/std": 0.4759978801012039,
      "step": 13,
      "train_speed(iter/s)": 0.007076
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 556.8671875,
      "completions/min_length": 296.5,
      "epoch": 0.07547169811320754,
      "grad_norm": 2.72421663127363,
      "kl": 0.0077667236328125,
      "learning_rate": 7.368421052631578e-07,
      "loss": 0.05722816288471222,
      "memory(GiB)": 75.26,
      "reward": 1.422330379486084,
      "reward_std": 0.7604877948760986,
      "rewards/Format/mean": 0.78125,
      "rewards/Format/std": 0.4145888388156891,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.3191515803337097,
      "rewards/ReflectionBalance/mean": 0.5238929241895676,
      "rewards/ReflectionBalance/std": 0.5091313123703003,
      "step": 14,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 541.88671875,
      "completions/min_length": 283.5,
      "epoch": 0.08086253369272237,
      "grad_norm": 2.5941146505373416,
      "kl": 0.0073394775390625,
      "learning_rate": 7.894736842105263e-07,
      "loss": 0.09058813750743866,
      "memory(GiB)": 75.26,
      "reward": 1.6432461142539978,
      "reward_std": 0.679021418094635,
      "rewards/Format/mean": 0.82421875,
      "rewards/Format/std": 0.3761351853609085,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4120787978172302,
      "rewards/ReflectionBalance/mean": 0.6041836142539978,
      "rewards/ReflectionBalance/std": 0.4352084547281265,
      "step": 15,
      "train_speed(iter/s)": 0.007145
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 596.0546875,
      "completions/min_length": 273.0,
      "epoch": 0.0862533692722372,
      "grad_norm": 2.427110617310291,
      "kl": 0.009368896484375,
      "learning_rate": 8.421052631578947e-07,
      "loss": 0.06289087980985641,
      "memory(GiB)": 75.26,
      "reward": 1.5345630049705505,
      "reward_std": 0.6162334680557251,
      "rewards/Format/mean": 0.80859375,
      "rewards/Format/std": 0.3946705758571625,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3481980115175247,
      "rewards/ReflectionBalance/mean": 0.5736255049705505,
      "rewards/ReflectionBalance/std": 0.450943261384964,
      "step": 16,
      "train_speed(iter/s)": 0.007129
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 570.03125,
      "completions/min_length": 274.5,
      "epoch": 0.09164420485175202,
      "grad_norm": 2.5623447116064906,
      "kl": 0.0101318359375,
      "learning_rate": 8.947368421052631e-07,
      "loss": 0.06114531308412552,
      "memory(GiB)": 75.26,
      "reward": 1.544982373714447,
      "reward_std": 0.5690030753612518,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3488178104162216,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3110392540693283,
      "rewards/ReflectionBalance/mean": 0.5762324333190918,
      "rewards/ReflectionBalance/std": 0.42977307736873627,
      "step": 17,
      "train_speed(iter/s)": 0.007143
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 567.12109375,
      "completions/min_length": 291.0,
      "epoch": 0.09703504043126684,
      "grad_norm": 2.5312649533386966,
      "kl": 0.00933837890625,
      "learning_rate": 9.473684210526315e-07,
      "loss": 0.050104379653930664,
      "memory(GiB)": 75.26,
      "reward": 1.5979236960411072,
      "reward_std": 0.6414499878883362,
      "rewards/Format/mean": 0.8359375,
      "rewards/Format/std": 0.3716365247964859,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4020542651414871,
      "rewards/ReflectionBalance/mean": 0.5471424460411072,
      "rewards/ReflectionBalance/std": 0.4390985071659088,
      "step": 18,
      "train_speed(iter/s)": 0.007139
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 610.734375,
      "completions/min_length": 273.0,
      "epoch": 0.10242587601078167,
      "grad_norm": 2.491307296000771,
      "kl": 0.010162353515625,
      "learning_rate": 1e-06,
      "loss": 0.07682099938392639,
      "memory(GiB)": 75.26,
      "reward": 1.5466822981834412,
      "reward_std": 0.7221653759479523,
      "rewards/Format/mean": 0.7890625,
      "rewards/Format/std": 0.405468225479126,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4206998348236084,
      "rewards/ReflectionBalance/mean": 0.5271510481834412,
      "rewards/ReflectionBalance/std": 0.4912317544221878,
      "step": 19,
      "train_speed(iter/s)": 0.007148
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 575.01171875,
      "completions/min_length": 227.0,
      "epoch": 0.1078167115902965,
      "grad_norm": 2.514475384711831,
      "kl": 0.008636474609375,
      "learning_rate": 9.999992640251751e-07,
      "loss": 0.07606859505176544,
      "memory(GiB)": 75.26,
      "reward": 1.4928263425827026,
      "reward_std": 0.5855598449707031,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3300945907831192,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3240884244441986,
      "rewards/ReflectionBalance/mean": 0.49282634258270264,
      "rewards/ReflectionBalance/std": 0.4728776514530182,
      "step": 20,
      "train_speed(iter/s)": 0.007169
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 571.44140625,
      "completions/min_length": 240.5,
      "epoch": 0.11320754716981132,
      "grad_norm": 2.5700684830325073,
      "kl": 0.010406494140625,
      "learning_rate": 9.99997056102867e-07,
      "loss": 0.07642433047294617,
      "memory(GiB)": 75.26,
      "reward": 1.5628766417503357,
      "reward_std": 0.6602072715759277,
      "rewards/Format/mean": 0.8125,
      "rewards/Format/std": 0.39132989943027496,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.5628766417503357,
      "rewards/ReflectionBalance/std": 0.4418746381998062,
      "step": 21,
      "train_speed(iter/s)": 0.007189
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 585.43359375,
      "completions/min_length": 253.5,
      "epoch": 0.11859838274932614,
      "grad_norm": 2.344141274957555,
      "kl": 0.007568359375,
      "learning_rate": 9.999933762395755e-07,
      "loss": 0.0839458480477333,
      "memory(GiB)": 75.26,
      "reward": 1.5905631184577942,
      "reward_std": 0.7205204665660858,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.36627575755119324,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.39705096185207367,
      "rewards/ReflectionBalance/mean": 0.535875678062439,
      "rewards/ReflectionBalance/std": 0.4491599053144455,
      "step": 22,
      "train_speed(iter/s)": 0.007157
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 543.3046875,
      "completions/min_length": 219.5,
      "epoch": 0.12398921832884097,
      "grad_norm": 3.109823116849284,
      "kl": 0.011322021484375,
      "learning_rate": 9.999882244461342e-07,
      "loss": 0.0833006352186203,
      "memory(GiB)": 75.26,
      "reward": 1.5974332094192505,
      "reward_std": 0.598694384098053,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3270204961299896,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36175550520420074,
      "rewards/ReflectionBalance/mean": 0.5544643998146057,
      "rewards/ReflectionBalance/std": 0.45583415031433105,
      "step": 23,
      "train_speed(iter/s)": 0.007152
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 549.171875,
      "completions/min_length": 200.5,
      "epoch": 0.1293800539083558,
      "grad_norm": 2.7277302752323376,
      "kl": 0.010894775390625,
      "learning_rate": 9.999816007377089e-07,
      "loss": 0.06091999635100365,
      "memory(GiB)": 75.26,
      "reward": 1.5567480325698853,
      "reward_std": 0.5653439462184906,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.35296520590782166,
      "rewards/ReflectionBalance/mean": 0.5176855176687241,
      "rewards/ReflectionBalance/std": 0.48428604006767273,
      "step": 24,
      "train_speed(iter/s)": 0.007142
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1003.0,
      "completions/mean_length": 536.234375,
      "completions/min_length": 295.0,
      "epoch": 0.1347708894878706,
      "grad_norm": 2.681964995957094,
      "kl": 0.009002685546875,
      "learning_rate": 9.999735051337993e-07,
      "loss": 0.08795583248138428,
      "memory(GiB)": 75.26,
      "reward": 1.7562971711158752,
      "reward_std": 0.5703249275684357,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32746291160583496,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.4149145632982254,
      "rewards/ReflectionBalance/mean": 0.6586409509181976,
      "rewards/ReflectionBalance/std": 0.38362522423267365,
      "step": 25,
      "train_speed(iter/s)": 0.007159
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 591.609375,
      "completions/min_length": 297.5,
      "epoch": 0.14016172506738545,
      "grad_norm": 2.603991738086859,
      "kl": 0.012115478515625,
      "learning_rate": 9.999639376582383e-07,
      "loss": 0.09100447595119476,
      "memory(GiB)": 75.26,
      "reward": 1.557416021823883,
      "reward_std": 0.636362224817276,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.36039385199546814,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3619322180747986,
      "rewards/ReflectionBalance/mean": 0.5535097718238831,
      "rewards/ReflectionBalance/std": 0.4339396059513092,
      "step": 26,
      "train_speed(iter/s)": 0.007179
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 605.8671875,
      "completions/min_length": 277.0,
      "epoch": 0.14555256064690028,
      "grad_norm": 2.549491554487536,
      "kl": 0.010498046875,
      "learning_rate": 9.999528983391914e-07,
      "loss": 0.12798286974430084,
      "memory(GiB)": 75.26,
      "reward": 1.542462170124054,
      "reward_std": 0.7209632992744446,
      "rewards/Format/mean": 0.8359375,
      "rewards/Format/std": 0.3704231530427933,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35675957798957825,
      "rewards/ReflectionBalance/mean": 0.5580872297286987,
      "rewards/ReflectionBalance/std": 0.4663889855146408,
      "step": 27,
      "train_speed(iter/s)": 0.007163
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 554.47265625,
      "completions/min_length": 253.5,
      "epoch": 0.1509433962264151,
      "grad_norm": 2.686046119412207,
      "kl": 0.011138916015625,
      "learning_rate": 9.999403872091572e-07,
      "loss": 0.06318516284227371,
      "memory(GiB)": 75.26,
      "reward": 1.5001006126403809,
      "reward_std": 0.6487970352172852,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.36039385199546814,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3265802264213562,
      "rewards/ReflectionBalance/mean": 0.5274443477392197,
      "rewards/ReflectionBalance/std": 0.4433763027191162,
      "step": 28,
      "train_speed(iter/s)": 0.007144
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 548.5078125,
      "completions/min_length": 279.5,
      "epoch": 0.15633423180592992,
      "grad_norm": 2.53649526767254,
      "kl": 0.0120849609375,
      "learning_rate": 9.99926404304967e-07,
      "loss": 0.06439143419265747,
      "memory(GiB)": 75.26,
      "reward": 1.6034176349639893,
      "reward_std": 0.5614951252937317,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.36627575755119324,
      "rewards/ReflectionBalance/mean": 0.5370113849639893,
      "rewards/ReflectionBalance/std": 0.4505663961172104,
      "step": 29,
      "train_speed(iter/s)": 0.007152
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 545.71484375,
      "completions/min_length": 239.5,
      "epoch": 0.16172506738544473,
      "grad_norm": 3.362378360469104,
      "kl": 0.02294921875,
      "learning_rate": 9.999109496677857e-07,
      "loss": 0.08645878732204437,
      "memory(GiB)": 75.26,
      "reward": 1.753075361251831,
      "reward_std": 0.5186206698417664,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31333550810813904,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41983383893966675,
      "rewards/ReflectionBalance/mean": 0.635887861251831,
      "rewards/ReflectionBalance/std": 0.34823407232761383,
      "step": 30,
      "train_speed(iter/s)": 0.007164
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 583.54296875,
      "completions/min_length": 298.0,
      "epoch": 0.16711590296495957,
      "grad_norm": 4.514782066110854,
      "kl": 0.018646240234375,
      "learning_rate": 9.998940233431095e-07,
      "loss": 0.09804210811853409,
      "memory(GiB)": 75.26,
      "reward": 1.7066685557365417,
      "reward_std": 0.6191928386688232,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31333550810813904,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41441819071769714,
      "rewards/ReflectionBalance/mean": 0.5933873653411865,
      "rewards/ReflectionBalance/std": 0.41488321125507355,
      "step": 31,
      "train_speed(iter/s)": 0.007178
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 531.0078125,
      "completions/min_length": 259.5,
      "epoch": 0.1725067385444744,
      "grad_norm": 2.5801121815806494,
      "kl": 0.009735107421875,
      "learning_rate": 9.998756253807684e-07,
      "loss": 0.07538656145334244,
      "memory(GiB)": 75.26,
      "reward": 1.6479092836380005,
      "reward_std": 0.5551980435848236,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3948954641819,
      "rewards/ReflectionBalance/mean": 0.5541593432426453,
      "rewards/ReflectionBalance/std": 0.43488429486751556,
      "step": 32,
      "train_speed(iter/s)": 0.007181
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 526.26171875,
      "completions/min_length": 262.5,
      "epoch": 0.1778975741239892,
      "grad_norm": 2.7038049106943953,
      "kl": 0.012298583984375,
      "learning_rate": 9.998557558349236e-07,
      "loss": 0.06017240881919861,
      "memory(GiB)": 75.26,
      "reward": 1.6717175245285034,
      "reward_std": 0.5337714552879333,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3630705028772354,
      "rewards/ReflectionBalance/mean": 0.6053112745285034,
      "rewards/ReflectionBalance/std": 0.38594989478588104,
      "step": 33,
      "train_speed(iter/s)": 0.007165
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 577.81640625,
      "completions/min_length": 277.0,
      "epoch": 0.18328840970350405,
      "grad_norm": 2.64056328473055,
      "kl": 0.013153076171875,
      "learning_rate": 9.998344147640693e-07,
      "loss": 0.07015877962112427,
      "memory(GiB)": 75.26,
      "reward": 1.6091487407684326,
      "reward_std": 0.5828424692153931,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3389271944761276,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3711828291416168,
      "rewards/ReflectionBalance/mean": 0.577898770570755,
      "rewards/ReflectionBalance/std": 0.41389377415180206,
      "step": 34,
      "train_speed(iter/s)": 0.007149
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 534.26953125,
      "completions/min_length": 276.5,
      "epoch": 0.18867924528301888,
      "grad_norm": 2.7627202689823385,
      "kl": 0.0133056640625,
      "learning_rate": 9.998116022310315e-07,
      "loss": 0.07463390380144119,
      "memory(GiB)": 75.26,
      "reward": 1.6474170088768005,
      "reward_std": 0.5859660357236862,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3630705028772354,
      "rewards/ReflectionBalance/mean": 0.5849170088768005,
      "rewards/ReflectionBalance/std": 0.38658659160137177,
      "step": 35,
      "train_speed(iter/s)": 0.007154
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 574.69921875,
      "completions/min_length": 252.0,
      "epoch": 0.1940700808625337,
      "grad_norm": 2.605559510528025,
      "kl": 0.012481689453125,
      "learning_rate": 9.99787318302968e-07,
      "loss": 0.09027635306119919,
      "memory(GiB)": 75.26,
      "reward": 1.4899238348007202,
      "reward_std": 0.6087012887001038,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32612966001033783,
      "rewards/MultiModalAccuracyORM/mean": 0.0859375,
      "rewards/MultiModalAccuracyORM/std": 0.2721100151538849,
      "rewards/ReflectionBalance/mean": 0.5250800848007202,
      "rewards/ReflectionBalance/std": 0.4670971632003784,
      "step": 36,
      "train_speed(iter/s)": 0.007167
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 545.9609375,
      "completions/min_length": 222.0,
      "epoch": 0.19946091644204852,
      "grad_norm": 2.8321256457220767,
      "kl": 0.0152587890625,
      "learning_rate": 9.99761563051368e-07,
      "loss": 0.07366214692592621,
      "memory(GiB)": 75.26,
      "reward": 1.6574755907058716,
      "reward_std": 0.5531894862651825,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.27558503299951553,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.35190291702747345,
      "rewards/ReflectionBalance/mean": 0.5988818109035492,
      "rewards/ReflectionBalance/std": 0.3908299654722214,
      "step": 37,
      "train_speed(iter/s)": 0.007168
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 529.32421875,
      "completions/min_length": 236.5,
      "epoch": 0.20485175202156333,
      "grad_norm": 2.4436759791984466,
      "kl": 0.012451171875,
      "learning_rate": 9.997343365520528e-07,
      "loss": 0.05735373497009277,
      "memory(GiB)": 75.26,
      "reward": 1.612046718597412,
      "reward_std": 0.5002427995204926,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.09375,
      "rewards/MultiModalAccuracyORM/std": 0.275717556476593,
      "rewards/ReflectionBalance/mean": 0.5964217185974121,
      "rewards/ReflectionBalance/std": 0.41718414425849915,
      "step": 38,
      "train_speed(iter/s)": 0.007162
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 556.03515625,
      "completions/min_length": 261.0,
      "epoch": 0.21024258760107817,
      "grad_norm": 2.691309502828634,
      "kl": 0.01708984375,
      "learning_rate": 9.99705638885174e-07,
      "loss": 0.0796165019273758,
      "memory(GiB)": 75.26,
      "reward": 1.7353515625,
      "reward_std": 0.5863654315471649,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.405468225479126,
      "rewards/ReflectionBalance/mean": 0.6025390625,
      "rewards/ReflectionBalance/std": 0.39295725524425507,
      "step": 39,
      "train_speed(iter/s)": 0.007158
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1002.0,
      "completions/mean_length": 543.5859375,
      "completions/min_length": 260.5,
      "epoch": 0.215633423180593,
      "grad_norm": 2.6313449425486994,
      "kl": 0.01153564453125,
      "learning_rate": 9.99675470135215e-07,
      "loss": 0.05663401260972023,
      "memory(GiB)": 75.26,
      "reward": 1.6236230731010437,
      "reward_std": 0.5937604308128357,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.26326169073581696,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3375232368707657,
      "rewards/ReflectionBalance/mean": 0.5728417932987213,
      "rewards/ReflectionBalance/std": 0.4279767721891403,
      "step": 40,
      "train_speed(iter/s)": 0.007158
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 527.4140625,
      "completions/min_length": 282.5,
      "epoch": 0.2210242587601078,
      "grad_norm": 2.6112048828881433,
      "kl": 0.014190673828125,
      "learning_rate": 9.996438303909894e-07,
      "loss": 0.08760400116443634,
      "memory(GiB)": 75.26,
      "reward": 1.6119580268859863,
      "reward_std": 0.49953463673591614,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.0859375,
      "rewards/MultiModalAccuracyORM/std": 0.2810242921113968,
      "rewards/ReflectionBalance/mean": 0.6080517470836639,
      "rewards/ReflectionBalance/std": 0.41364288330078125,
      "step": 41,
      "train_speed(iter/s)": 0.007163
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 547.9375,
      "completions/min_length": 237.5,
      "epoch": 0.22641509433962265,
      "grad_norm": 2.6064392421726406,
      "kl": 0.0147705078125,
      "learning_rate": 9.996107197456416e-07,
      "loss": 0.055845845490694046,
      "memory(GiB)": 75.26,
      "reward": 1.6543285846710205,
      "reward_std": 0.4986225366592407,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2488820105791092,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.298691526055336,
      "rewards/ReflectionBalance/mean": 0.6230785846710205,
      "rewards/ReflectionBalance/std": 0.3957352787256241,
      "step": 42,
      "train_speed(iter/s)": 0.007167
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1014.5,
      "completions/mean_length": 519.5625,
      "completions/min_length": 264.0,
      "epoch": 0.23180592991913745,
      "grad_norm": 2.7577887931079164,
      "kl": 0.01519775390625,
      "learning_rate": 9.995761382966457e-07,
      "loss": 0.08760404586791992,
      "memory(GiB)": 75.26,
      "reward": 1.6570544242858887,
      "reward_std": 0.5731455087661743,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3643587827682495,
      "rewards/ReflectionBalance/mean": 0.5945543646812439,
      "rewards/ReflectionBalance/std": 0.3995678722858429,
      "step": 43,
      "train_speed(iter/s)": 0.007175
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 576.6875,
      "completions/min_length": 291.0,
      "epoch": 0.2371967654986523,
      "grad_norm": 11.14873720081309,
      "kl": 0.0244140625,
      "learning_rate": 9.995400861458062e-07,
      "loss": 0.11227934062480927,
      "memory(GiB)": 75.26,
      "reward": 1.6093571186065674,
      "reward_std": 0.5719938576221466,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35872696340084076,
      "rewards/ReflectionBalance/mean": 0.5624821186065674,
      "rewards/ReflectionBalance/std": 0.42660994827747345,
      "step": 44,
      "train_speed(iter/s)": 0.007183
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1012.0,
      "completions/mean_length": 533.00390625,
      "completions/min_length": 268.5,
      "epoch": 0.24258760107816713,
      "grad_norm": 2.723464702368424,
      "kl": 0.015228271484375,
      "learning_rate": 9.99502563399257e-07,
      "loss": 0.06651771813631058,
      "memory(GiB)": 75.26,
      "reward": 1.68726646900177,
      "reward_std": 0.5043202638626099,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27079907804727554,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.3341423198580742,
      "rewards/ReflectionBalance/mean": 0.6208601593971252,
      "rewards/ReflectionBalance/std": 0.364407479763031,
      "step": 45,
      "train_speed(iter/s)": 0.007195
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 532.09765625,
      "completions/min_length": 274.5,
      "epoch": 0.24797843665768193,
      "grad_norm": 2.614369810904399,
      "kl": 0.01409912109375,
      "learning_rate": 9.994635701674613e-07,
      "loss": 0.059676893055438995,
      "memory(GiB)": 75.26,
      "reward": 1.7489760518074036,
      "reward_std": 0.5268251597881317,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4225567579269409,
      "rewards/ReflectionBalance/mean": 0.608351081609726,
      "rewards/ReflectionBalance/std": 0.4404994249343872,
      "step": 46,
      "train_speed(iter/s)": 0.007198
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 988.0,
      "completions/mean_length": 529.15234375,
      "completions/min_length": 257.0,
      "epoch": 0.25336927223719674,
      "grad_norm": 2.652387330810387,
      "kl": 0.015899658203125,
      "learning_rate": 9.99423106565211e-07,
      "loss": 0.05569852888584137,
      "memory(GiB)": 75.26,
      "reward": 1.8550137877464294,
      "reward_std": 0.47932133078575134,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4323427081108093,
      "rewards/ReflectionBalance/mean": 0.6714200973510742,
      "rewards/ReflectionBalance/std": 0.34484580159187317,
      "step": 47,
      "train_speed(iter/s)": 0.007204
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 982.0,
      "completions/mean_length": 488.8046875,
      "completions/min_length": 214.5,
      "epoch": 0.2587601078167116,
      "grad_norm": 2.882005358220201,
      "kl": 0.01519775390625,
      "learning_rate": 9.993811727116273e-07,
      "loss": 0.06770214438438416,
      "memory(GiB)": 75.26,
      "reward": 1.7983328700065613,
      "reward_std": 0.4938793033361435,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.39705096185207367,
      "rewards/ReflectionBalance/mean": 0.6577078402042389,
      "rewards/ReflectionBalance/std": 0.33989596366882324,
      "step": 48,
      "train_speed(iter/s)": 0.007214
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1016.5,
      "completions/mean_length": 529.33203125,
      "completions/min_length": 245.0,
      "epoch": 0.2641509433962264,
      "grad_norm": 9.601673420893698,
      "kl": 0.04241943359375,
      "learning_rate": 9.993377687301588e-07,
      "loss": 0.10285373777151108,
      "memory(GiB)": 75.26,
      "reward": 1.7496874928474426,
      "reward_std": 0.5711759626865387,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.253920242190361,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3968885838985443,
      "rewards/ReflectionBalance/mean": 0.6285937428474426,
      "rewards/ReflectionBalance/std": 0.40187886357307434,
      "step": 49,
      "train_speed(iter/s)": 0.007214
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 520.58203125,
      "completions/min_length": 259.5,
      "epoch": 0.2695417789757412,
      "grad_norm": 2.871689627571954,
      "kl": 0.016571044921875,
      "learning_rate": 9.99292894748583e-07,
      "loss": 0.07097083330154419,
      "memory(GiB)": 75.26,
      "reward": 1.6617772579193115,
      "reward_std": 0.5117383450269699,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3488178104162216,
      "rewards/ReflectionBalance/mean": 0.6031836271286011,
      "rewards/ReflectionBalance/std": 0.39070047438144684,
      "step": 50,
      "train_speed(iter/s)": 0.007222
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 523.05078125,
      "completions/min_length": 239.5,
      "epoch": 0.2749326145552561,
      "grad_norm": 2.734915236868847,
      "kl": 0.017242431640625,
      "learning_rate": 9.992465508990035e-07,
      "loss": 0.07420282065868378,
      "memory(GiB)": 75.26,
      "reward": 1.7969650626182556,
      "reward_std": 0.4684544950723648,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3636700212955475,
      "rewards/ReflectionBalance/mean": 0.6954025030136108,
      "rewards/ReflectionBalance/std": 0.29814252257347107,
      "step": 51,
      "train_speed(iter/s)": 0.007226
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 545.34375,
      "completions/min_length": 301.5,
      "epoch": 0.2803234501347709,
      "grad_norm": 2.519026125835849,
      "kl": 0.015533447265625,
      "learning_rate": 9.991987373178528e-07,
      "loss": 0.050643160939216614,
      "memory(GiB)": 75.26,
      "reward": 1.6989575624465942,
      "reward_std": 0.5202546864748001,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2801405265927315,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.38336627185344696,
      "rewards/ReflectionBalance/mean": 0.6013012826442719,
      "rewards/ReflectionBalance/std": 0.39197322726249695,
      "step": 52,
      "train_speed(iter/s)": 0.007232
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 525.9765625,
      "completions/min_length": 264.5,
      "epoch": 0.2857142857142857,
      "grad_norm": 2.5714206682571303,
      "kl": 0.0174560546875,
      "learning_rate": 9.991494541458889e-07,
      "loss": 0.07727980613708496,
      "memory(GiB)": 75.26,
      "reward": 1.7929589748382568,
      "reward_std": 0.5656134486198425,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41983383893966675,
      "rewards/ReflectionBalance/mean": 0.6484277248382568,
      "rewards/ReflectionBalance/std": 0.3472103327512741,
      "step": 53,
      "train_speed(iter/s)": 0.007232
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 523.23828125,
      "completions/min_length": 272.0,
      "epoch": 0.29110512129380056,
      "grad_norm": 2.7606982329234167,
      "kl": 0.018524169921875,
      "learning_rate": 9.990987015281966e-07,
      "loss": 0.06712295114994049,
      "memory(GiB)": 75.26,
      "reward": 1.71115642786026,
      "reward_std": 0.5094357430934906,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23443959653377533,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35737672448158264,
      "rewards/ReflectionBalance/mean": 0.61740642786026,
      "rewards/ReflectionBalance/std": 0.39012572169303894,
      "step": 54,
      "train_speed(iter/s)": 0.007233
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 540.94921875,
      "completions/min_length": 283.5,
      "epoch": 0.29649595687331537,
      "grad_norm": 3.2060461584551194,
      "kl": 0.0225830078125,
      "learning_rate": 9.990464796141864e-07,
      "loss": 0.06270969659090042,
      "memory(GiB)": 75.26,
      "reward": 1.7088735699653625,
      "reward_std": 0.5917553305625916,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.28448406606912613,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.4122922718524933,
      "rewards/ReflectionBalance/mean": 0.5838735401630402,
      "rewards/ReflectionBalance/std": 0.4211901128292084,
      "step": 55,
      "train_speed(iter/s)": 0.007234
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 530.9921875,
      "completions/min_length": 256.5,
      "epoch": 0.3018867924528302,
      "grad_norm": 2.5626060418204175,
      "kl": 0.01531982421875,
      "learning_rate": 9.989927885575944e-07,
      "loss": 0.06542704254388809,
      "memory(GiB)": 75.26,
      "reward": 1.6949970722198486,
      "reward_std": 0.36432217061519623,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.07421875,
      "rewards/MultiModalAccuracyORM/std": 0.260444276034832,
      "rewards/ReflectionBalance/mean": 0.6637470126152039,
      "rewards/ReflectionBalance/std": 0.31397657096385956,
      "step": 56,
      "train_speed(iter/s)": 0.007226
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 548.61328125,
      "completions/min_length": 275.0,
      "epoch": 0.30727762803234504,
      "grad_norm": 2.509997989115723,
      "kl": 0.014495849609375,
      "learning_rate": 9.989376285164815e-07,
      "loss": 0.08522529900074005,
      "memory(GiB)": 75.26,
      "reward": 1.6855634450912476,
      "reward_std": 0.5033027827739716,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2566775679588318,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.31812146306037903,
      "rewards/ReflectionBalance/mean": 0.6425946950912476,
      "rewards/ReflectionBalance/std": 0.36812634766101837,
      "step": 57,
      "train_speed(iter/s)": 0.00723
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 499.41796875,
      "completions/min_length": 259.0,
      "epoch": 0.31266846361185985,
      "grad_norm": 2.622659254841244,
      "kl": 0.014556884765625,
      "learning_rate": 9.988809996532336e-07,
      "loss": 0.060026615858078,
      "memory(GiB)": 75.26,
      "reward": 1.8088199496269226,
      "reward_std": 0.4815021753311157,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21220162510871887,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3948954641819,
      "rewards/ReflectionBalance/mean": 0.660382479429245,
      "rewards/ReflectionBalance/std": 0.3546471446752548,
      "step": 58,
      "train_speed(iter/s)": 0.007236
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 521.16015625,
      "completions/min_length": 289.0,
      "epoch": 0.31805929919137466,
      "grad_norm": 2.667751348916741,
      "kl": 0.01385498046875,
      "learning_rate": 9.988229021345602e-07,
      "loss": 0.08099701255559921,
      "memory(GiB)": 75.26,
      "reward": 1.7442985773086548,
      "reward_std": 0.42395099997520447,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21138381212949753,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3405093401670456,
      "rewards/ReflectionBalance/mean": 0.6583609879016876,
      "rewards/ReflectionBalance/std": 0.35230398178100586,
      "step": 59,
      "train_speed(iter/s)": 0.00724
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 488.6796875,
      "completions/min_length": 228.0,
      "epoch": 0.32345013477088946,
      "grad_norm": 2.917291836633914,
      "kl": 0.015533447265625,
      "learning_rate": 9.987633361314945e-07,
      "loss": 0.09062056988477707,
      "memory(GiB)": 75.26,
      "reward": 1.856981635093689,
      "reward_std": 0.5096233487129211,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23196382820606232,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3996165096759796,
      "rewards/ReflectionBalance/mean": 0.7124503552913666,
      "rewards/ReflectionBalance/std": 0.3078223466873169,
      "step": 60,
      "train_speed(iter/s)": 0.007232
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 544.39453125,
      "completions/min_length": 273.5,
      "epoch": 0.3288409703504043,
      "grad_norm": 2.4962097576535447,
      "kl": 0.01580810546875,
      "learning_rate": 9.98702301819393e-07,
      "loss": 0.054244209080934525,
      "memory(GiB)": 75.26,
      "reward": 1.6833567023277283,
      "reward_std": 0.47740231454372406,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.305100217461586,
      "rewards/ReflectionBalance/mean": 0.6208567023277283,
      "rewards/ReflectionBalance/std": 0.3722357600927353,
      "step": 61,
      "train_speed(iter/s)": 0.007224
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 520.54296875,
      "completions/min_length": 254.5,
      "epoch": 0.33423180592991913,
      "grad_norm": 4.291198193904349,
      "kl": 0.01812744140625,
      "learning_rate": 9.986397993779344e-07,
      "loss": 0.08077535033226013,
      "memory(GiB)": 75.26,
      "reward": 1.7721281051635742,
      "reward_std": 0.5012955814599991,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24301259219646454,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3591410219669342,
      "rewards/ReflectionBalance/mean": 0.6744718551635742,
      "rewards/ReflectionBalance/std": 0.34510597586631775,
      "step": 62,
      "train_speed(iter/s)": 0.007225
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 942.0,
      "completions/mean_length": 501.72265625,
      "completions/min_length": 261.0,
      "epoch": 0.33962264150943394,
      "grad_norm": 4.0306888436262005,
      "kl": 0.0181884765625,
      "learning_rate": 9.985758289911196e-07,
      "loss": 0.04869752377271652,
      "memory(GiB)": 75.26,
      "reward": 1.7551366686820984,
      "reward_std": 0.4754926860332489,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.35190291702747345,
      "rewards/ReflectionBalance/mean": 0.6770117282867432,
      "rewards/ReflectionBalance/std": 0.32854829728603363,
      "step": 63,
      "train_speed(iter/s)": 0.007235
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 545.98828125,
      "completions/min_length": 292.0,
      "epoch": 0.3450134770889488,
      "grad_norm": 2.5601493712557004,
      "kl": 0.01751708984375,
      "learning_rate": 9.985103908472712e-07,
      "loss": 0.09221744537353516,
      "memory(GiB)": 75.26,
      "reward": 1.6444857120513916,
      "reward_std": 0.5105410218238831,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.3082972764968872,
      "rewards/ReflectionBalance/mean": 0.6132356822490692,
      "rewards/ReflectionBalance/std": 0.39150404930114746,
      "step": 64,
      "train_speed(iter/s)": 0.007233
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1000.5,
      "completions/mean_length": 496.3671875,
      "completions/min_length": 246.5,
      "epoch": 0.3504043126684636,
      "grad_norm": 2.919252244215987,
      "kl": 0.020751953125,
      "learning_rate": 9.984434851390321e-07,
      "loss": 0.0627211481332779,
      "memory(GiB)": 75.26,
      "reward": 1.7763630747795105,
      "reward_std": 0.4762306958436966,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3734995424747467,
      "rewards/ReflectionBalance/mean": 0.6708943545818329,
      "rewards/ReflectionBalance/std": 0.3140830248594284,
      "step": 65,
      "train_speed(iter/s)": 0.007225
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 556.42578125,
      "completions/min_length": 302.5,
      "epoch": 0.3557951482479784,
      "grad_norm": 2.463080556483974,
      "kl": 0.01806640625,
      "learning_rate": 9.983751120633663e-07,
      "loss": 0.09403274953365326,
      "memory(GiB)": 75.26,
      "reward": 1.7304719686508179,
      "reward_std": 0.4794701486825943,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.339918777346611,
      "rewards/ReflectionBalance/mean": 0.6757844388484955,
      "rewards/ReflectionBalance/std": 0.29089654982089996,
      "step": 66,
      "train_speed(iter/s)": 0.007227
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 507.703125,
      "completions/min_length": 257.0,
      "epoch": 0.3611859838274933,
      "grad_norm": 2.674119161982221,
      "kl": 0.01971435546875,
      "learning_rate": 9.983052718215572e-07,
      "loss": 0.078293576836586,
      "memory(GiB)": 75.26,
      "reward": 1.7860628366470337,
      "reward_std": 0.4243137836456299,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3597298264503479,
      "rewards/ReflectionBalance/mean": 0.6884065568447113,
      "rewards/ReflectionBalance/std": 0.321468248963356,
      "step": 67,
      "train_speed(iter/s)": 0.007232
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 518.7890625,
      "completions/min_length": 254.0,
      "epoch": 0.3665768194070081,
      "grad_norm": 2.6310914641146255,
      "kl": 0.018310546875,
      "learning_rate": 9.982339646192075e-07,
      "loss": 0.08578591048717499,
      "memory(GiB)": 75.26,
      "reward": 1.720004916191101,
      "reward_std": 0.47176042199134827,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3270204961299896,
      "rewards/ReflectionBalance/mean": 0.6848486363887787,
      "rewards/ReflectionBalance/std": 0.3109336793422699,
      "step": 68,
      "train_speed(iter/s)": 0.007231
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1007.5,
      "completions/mean_length": 519.1796875,
      "completions/min_length": 267.0,
      "epoch": 0.3719676549865229,
      "grad_norm": 2.810630819944814,
      "kl": 0.018798828125,
      "learning_rate": 9.98161190666238e-07,
      "loss": 0.06687115132808685,
      "memory(GiB)": 75.26,
      "reward": 1.6955868005752563,
      "reward_std": 0.4233112037181854,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22826264798641205,
      "rewards/MultiModalAccuracyORM/mean": 0.08984375,
      "rewards/MultiModalAccuracyORM/std": 0.2863408625125885,
      "rewards/ReflectionBalance/mean": 0.6604304909706116,
      "rewards/ReflectionBalance/std": 0.33968085050582886,
      "step": 69,
      "train_speed(iter/s)": 0.007234
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 529.8359375,
      "completions/min_length": 301.0,
      "epoch": 0.37735849056603776,
      "grad_norm": 3.271523797094722,
      "kl": 0.024169921875,
      "learning_rate": 9.980869501768886e-07,
      "loss": 0.0764276459813118,
      "memory(GiB)": 75.26,
      "reward": 1.81620454788208,
      "reward_std": 0.5017227083444595,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.27558503299951553,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4067413955926895,
      "rewards/ReflectionBalance/mean": 0.6951107978820801,
      "rewards/ReflectionBalance/std": 0.3073228746652603,
      "step": 70,
      "train_speed(iter/s)": 0.007232
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 968.0,
      "completions/mean_length": 526.046875,
      "completions/min_length": 288.0,
      "epoch": 0.38274932614555257,
      "grad_norm": 2.6358061200923086,
      "kl": 0.02099609375,
      "learning_rate": 9.980112433697149e-07,
      "loss": 0.07633523643016815,
      "memory(GiB)": 75.26,
      "reward": 1.778160035610199,
      "reward_std": 0.4150947481393814,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3704231530427933,
      "rewards/ReflectionBalance/mean": 0.6648787260055542,
      "rewards/ReflectionBalance/std": 0.31563329696655273,
      "step": 71,
      "train_speed(iter/s)": 0.00724
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 996.0,
      "completions/mean_length": 520.5234375,
      "completions/min_length": 281.0,
      "epoch": 0.3881401617250674,
      "grad_norm": 2.663503361333192,
      "kl": 0.0230712890625,
      "learning_rate": 9.979340704675911e-07,
      "loss": 0.05977418273687363,
      "memory(GiB)": 75.26,
      "reward": 1.7736095190048218,
      "reward_std": 0.45015859603881836,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.20856640115380287,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39068229496479034,
      "rewards/ReflectionBalance/mean": 0.6407970190048218,
      "rewards/ReflectionBalance/std": 0.36635541915893555,
      "step": 72,
      "train_speed(iter/s)": 0.007244
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 497.859375,
      "completions/min_length": 263.0,
      "epoch": 0.3935309973045822,
      "grad_norm": 2.7078132635939465,
      "kl": 0.02490234375,
      "learning_rate": 9.978554316977057e-07,
      "loss": 0.0586530826985836,
      "memory(GiB)": 75.26,
      "reward": 1.8216007351875305,
      "reward_std": 0.5247361212968826,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.4002292603254318,
      "rewards/ReflectionBalance/mean": 0.6966007351875305,
      "rewards/ReflectionBalance/std": 0.3028703033924103,
      "step": 73,
      "train_speed(iter/s)": 0.007253
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 983.0,
      "completions/mean_length": 518.91796875,
      "completions/min_length": 203.0,
      "epoch": 0.39892183288409705,
      "grad_norm": 11.676978851277585,
      "kl": 0.0760498046875,
      "learning_rate": 9.97775327291564e-07,
      "loss": 0.04942583665251732,
      "memory(GiB)": 75.26,
      "reward": 1.7341903448104858,
      "reward_std": 0.4788230359554291,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3497565984725952,
      "rewards/ReflectionBalance/mean": 0.6599715054035187,
      "rewards/ReflectionBalance/std": 0.32149793207645416,
      "step": 74,
      "train_speed(iter/s)": 0.007256
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 508.30859375,
      "completions/min_length": 221.5,
      "epoch": 0.40431266846361186,
      "grad_norm": 2.6682865201453434,
      "kl": 0.0223388671875,
      "learning_rate": 9.976937574849845e-07,
      "loss": 0.05844493210315704,
      "memory(GiB)": 75.26,
      "reward": 1.849054992198944,
      "reward_std": 0.4313090741634369,
      "rewards/Format/mean": 0.97265625,
      "rewards/Format/std": 0.1595059372484684,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3846261203289032,
      "rewards/ReflectionBalance/mean": 0.6928050220012665,
      "rewards/ReflectionBalance/std": 0.31061410903930664,
      "step": 75,
      "train_speed(iter/s)": 0.007252
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 923.5,
      "completions/mean_length": 496.11328125,
      "completions/min_length": 252.0,
      "epoch": 0.40970350404312667,
      "grad_norm": 62.14199957475747,
      "kl": 0.1868896484375,
      "learning_rate": 9.976107225181012e-07,
      "loss": 0.024208927527070045,
      "memory(GiB)": 75.26,
      "reward": 1.8310779333114624,
      "reward_std": 0.48015131056308746,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.39750634133815765,
      "rewards/ReflectionBalance/mean": 0.7099842429161072,
      "rewards/ReflectionBalance/std": 0.3080308884382248,
      "step": 76,
      "train_speed(iter/s)": 0.007262
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 537.15625,
      "completions/min_length": 253.5,
      "epoch": 0.41509433962264153,
      "grad_norm": 2.780347074583763,
      "kl": 0.02154541015625,
      "learning_rate": 9.975262226353603e-07,
      "loss": 0.06912977993488312,
      "memory(GiB)": 75.26,
      "reward": 1.769864559173584,
      "reward_std": 0.46872979402542114,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23443959653377533,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.36039385199546814,
      "rewards/ReflectionBalance/mean": 0.676114559173584,
      "rewards/ReflectionBalance/std": 0.3520418703556061,
      "step": 77,
      "train_speed(iter/s)": 0.007259
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 555.23046875,
      "completions/min_length": 283.5,
      "epoch": 0.42048517520215634,
      "grad_norm": 2.6415941946128423,
      "kl": 0.0240478515625,
      "learning_rate": 9.97440258085521e-07,
      "loss": 0.09614720940589905,
      "memory(GiB)": 75.26,
      "reward": 1.7925699353218079,
      "reward_std": 0.4685550481081009,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24081215262413025,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3613582104444504,
      "rewards/ReflectionBalance/mean": 0.6871012449264526,
      "rewards/ReflectionBalance/std": 0.331355482339859,
      "step": 78,
      "train_speed(iter/s)": 0.007258
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 518.36328125,
      "completions/min_length": 263.0,
      "epoch": 0.42587601078167114,
      "grad_norm": 2.5256049505370917,
      "kl": 0.021484375,
      "learning_rate": 9.973528291216541e-07,
      "loss": 0.07941921055316925,
      "memory(GiB)": 75.26,
      "reward": 1.883103847503662,
      "reward_std": 0.46904510259628296,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24081215262413025,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42852601408958435,
      "rewards/ReflectionBalance/mean": 0.7034163475036621,
      "rewards/ReflectionBalance/std": 0.2896035313606262,
      "step": 79,
      "train_speed(iter/s)": 0.007258
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 514.25390625,
      "completions/min_length": 276.5,
      "epoch": 0.431266846361186,
      "grad_norm": 2.604916690339567,
      "kl": 0.02154541015625,
      "learning_rate": 9.972639360011422e-07,
      "loss": 0.06120653077960014,
      "memory(GiB)": 75.26,
      "reward": 1.7458886504173279,
      "reward_std": 0.45483991503715515,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24301259219646454,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3130832314491272,
      "rewards/ReflectionBalance/mean": 0.6990136802196503,
      "rewards/ReflectionBalance/std": 0.306050568819046,
      "step": 80,
      "train_speed(iter/s)": 0.007263
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 502.3125,
      "completions/min_length": 229.0,
      "epoch": 0.4366576819407008,
      "grad_norm": 2.481760277275023,
      "kl": 0.01983642578125,
      "learning_rate": 9.971735789856775e-07,
      "loss": 0.07438698410987854,
      "memory(GiB)": 75.26,
      "reward": 1.8907164931297302,
      "reward_std": 0.5020117163658142,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.23365110158920288,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.41966044902801514,
      "rewards/ReflectionBalance/mean": 0.7227477431297302,
      "rewards/ReflectionBalance/std": 0.3179563879966736,
      "step": 81,
      "train_speed(iter/s)": 0.007263
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 947.5,
      "completions/mean_length": 506.015625,
      "completions/min_length": 281.0,
      "epoch": 0.4420485175202156,
      "grad_norm": 3.437639057853365,
      "kl": 0.034423828125,
      "learning_rate": 9.970817583412616e-07,
      "loss": 0.04757340997457504,
      "memory(GiB)": 75.26,
      "reward": 1.9075040817260742,
      "reward_std": 0.39709872007369995,
      "rewards/Format/mean": 0.96875,
      "rewards/Format/std": 0.17319392412900925,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3851911425590515,
      "rewards/ReflectionBalance/mean": 0.7434415817260742,
      "rewards/ReflectionBalance/std": 0.2548302263021469,
      "step": 82,
      "train_speed(iter/s)": 0.007274
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 527.2421875,
      "completions/min_length": 281.5,
      "epoch": 0.4474393530997305,
      "grad_norm": 2.5460739288078993,
      "kl": 0.0228271484375,
      "learning_rate": 9.969884743382055e-07,
      "loss": 0.05713076889514923,
      "memory(GiB)": 75.26,
      "reward": 1.8534326553344727,
      "reward_std": 0.44412219524383545,
      "rewards/Format/mean": 0.96484375,
      "rewards/Format/std": 0.1845906376838684,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4094643145799637,
      "rewards/ReflectionBalance/mean": 0.6776513457298279,
      "rewards/ReflectionBalance/std": 0.33451075851917267,
      "step": 83,
      "train_speed(iter/s)": 0.007274
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 500.27734375,
      "completions/min_length": 266.0,
      "epoch": 0.4528301886792453,
      "grad_norm": 2.6754832271858215,
      "kl": 0.024169921875,
      "learning_rate": 9.968937272511278e-07,
      "loss": 0.0808253288269043,
      "memory(GiB)": 75.26,
      "reward": 1.8384754061698914,
      "reward_std": 0.5245428085327148,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22826264798641205,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3973046988248825,
      "rewards/ReflectionBalance/mean": 0.6939441561698914,
      "rewards/ReflectionBalance/std": 0.3471072167158127,
      "step": 84,
      "train_speed(iter/s)": 0.007275
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1003.5,
      "completions/mean_length": 508.14453125,
      "completions/min_length": 257.0,
      "epoch": 0.4582210242587601,
      "grad_norm": 2.5534458733239203,
      "kl": 0.0208740234375,
      "learning_rate": 9.967975173589545e-07,
      "loss": 0.08311194181442261,
      "memory(GiB)": 75.26,
      "reward": 1.8944709300994873,
      "reward_std": 0.4415066987276077,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.19744773209095,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.3815378099679947,
      "rewards/ReflectionBalance/mean": 0.7147835195064545,
      "rewards/ReflectionBalance/std": 0.31376951932907104,
      "step": 85,
      "train_speed(iter/s)": 0.007271
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 523.35546875,
      "completions/min_length": 289.0,
      "epoch": 0.4636118598382749,
      "grad_norm": 2.8146377306893453,
      "kl": 0.0289306640625,
      "learning_rate": 9.96699844944918e-07,
      "loss": 0.06714411824941635,
      "memory(GiB)": 75.26,
      "reward": 1.6908219456672668,
      "reward_std": 0.4545360505580902,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2280246615409851,
      "rewards/MultiModalAccuracyORM/mean": 0.08203125,
      "rewards/MultiModalAccuracyORM/std": 0.27079907804727554,
      "rewards/ReflectionBalance/mean": 0.6673844158649445,
      "rewards/ReflectionBalance/std": 0.3484017997980118,
      "step": 86,
      "train_speed(iter/s)": 0.00727
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1015.5,
      "completions/mean_length": 527.6875,
      "completions/min_length": 233.0,
      "epoch": 0.46900269541778977,
      "grad_norm": 2.651289659078652,
      "kl": 0.02679443359375,
      "learning_rate": 9.966007102965555e-07,
      "loss": 0.07094302773475647,
      "memory(GiB)": 75.26,
      "reward": 1.7704458832740784,
      "reward_std": 0.49292050302028656,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.19343896210193634,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3538244515657425,
      "rewards/ReflectionBalance/mean": 0.6532584726810455,
      "rewards/ReflectionBalance/std": 0.3594873398542404,
      "step": 87,
      "train_speed(iter/s)": 0.00727
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 907.0,
      "completions/mean_length": 497.12109375,
      "completions/min_length": 232.5,
      "epoch": 0.4743935309973046,
      "grad_norm": 2.7489903101202726,
      "kl": 0.02581787109375,
      "learning_rate": 9.965001137057102e-07,
      "loss": 0.047264497727155685,
      "memory(GiB)": 75.26,
      "reward": 1.7928833365440369,
      "reward_std": 0.45421938598155975,
      "rewards/Format/mean": 0.96875,
      "rewards/Format/std": 0.1746762990951538,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3488178104162216,
      "rewards/ReflectionBalance/mean": 0.6835083067417145,
      "rewards/ReflectionBalance/std": 0.3235289752483368,
      "step": 88,
      "train_speed(iter/s)": 0.007278
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 890.0,
      "completions/mean_length": 496.3359375,
      "completions/min_length": 266.0,
      "epoch": 0.4797843665768194,
      "grad_norm": 2.761506039875543,
      "kl": 0.02764892578125,
      "learning_rate": 9.963980554685273e-07,
      "loss": 0.060527246445417404,
      "memory(GiB)": 75.26,
      "reward": 1.8426793813705444,
      "reward_std": 0.4268501400947571,
      "rewards/Format/mean": 0.96484375,
      "rewards/Format/std": 0.17638477310538292,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.37864522635936737,
      "rewards/ReflectionBalance/mean": 0.698148101568222,
      "rewards/ReflectionBalance/std": 0.3303479850292206,
      "step": 89,
      "train_speed(iter/s)": 0.007289
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 524.9375,
      "completions/min_length": 277.5,
      "epoch": 0.48517520215633425,
      "grad_norm": 2.3875582502429116,
      "kl": 0.0225830078125,
      "learning_rate": 9.962945358854572e-07,
      "loss": 0.06071839481592178,
      "memory(GiB)": 75.26,
      "reward": 1.9915930032730103,
      "reward_std": 0.4713473618030548,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.19007275998592377,
      "rewards/MultiModalAccuracyORM/mean": 0.3203125,
      "rewards/MultiModalAccuracyORM/std": 0.4657035768032074,
      "rewards/ReflectionBalance/mean": 0.7103429138660431,
      "rewards/ReflectionBalance/std": 0.30471573770046234,
      "step": 90,
      "train_speed(iter/s)": 0.007285
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 484.87890625,
      "completions/min_length": 254.0,
      "epoch": 0.49056603773584906,
      "grad_norm": 4.0093983951611305,
      "kl": 0.03167724609375,
      "learning_rate": 9.9618955526125e-07,
      "loss": 0.07547543942928314,
      "memory(GiB)": 75.26,
      "reward": 1.8374414443969727,
      "reward_std": 0.5489976704120636,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.4131145179271698,
      "rewards/ReflectionBalance/mean": 0.7085351645946503,
      "rewards/ReflectionBalance/std": 0.32018448412418365,
      "step": 91,
      "train_speed(iter/s)": 0.007286
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 494.59375,
      "completions/min_length": 257.0,
      "epoch": 0.49595687331536387,
      "grad_norm": 2.9044050204354366,
      "kl": 0.02838134765625,
      "learning_rate": 9.96083113904959e-07,
      "loss": 0.06173965334892273,
      "memory(GiB)": 75.26,
      "reward": 1.7667993307113647,
      "reward_std": 0.4383925199508667,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23443959653377533,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.29920530319213867,
      "rewards/ReflectionBalance/mean": 0.7042993009090424,
      "rewards/ReflectionBalance/std": 0.3150889128446579,
      "step": 92,
      "train_speed(iter/s)": 0.007284
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 506.52734375,
      "completions/min_length": 288.0,
      "epoch": 0.5013477088948787,
      "grad_norm": 2.576593598220569,
      "kl": 0.02642822265625,
      "learning_rate": 9.959752121299363e-07,
      "loss": 0.06433319300413132,
      "memory(GiB)": 75.26,
      "reward": 1.7882665991783142,
      "reward_std": 0.3941054195165634,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20146947354078293,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3749772012233734,
      "rewards/ReflectionBalance/mean": 0.6632665991783142,
      "rewards/ReflectionBalance/std": 0.30864517390727997,
      "step": 93,
      "train_speed(iter/s)": 0.007279
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 954.0,
      "completions/mean_length": 499.86328125,
      "completions/min_length": 175.0,
      "epoch": 0.5067385444743935,
      "grad_norm": 6.625933597734688,
      "kl": 0.03607177734375,
      "learning_rate": 9.958658502538338e-07,
      "loss": 0.045587435364723206,
      "memory(GiB)": 75.26,
      "reward": 1.8157389163970947,
      "reward_std": 0.46012310683727264,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23196382820606232,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.6868326663970947,
      "rewards/ReflectionBalance/std": 0.32346363365650177,
      "step": 94,
      "train_speed(iter/s)": 0.00728
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 504.9453125,
      "completions/min_length": 276.0,
      "epoch": 0.5121293800539084,
      "grad_norm": 3.0731044993272834,
      "kl": 0.03857421875,
      "learning_rate": 9.95755028598602e-07,
      "loss": 0.08180917054414749,
      "memory(GiB)": 75.26,
      "reward": 1.7215991616249084,
      "reward_std": 0.42363759875297546,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.19744773209095,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.31232215464115143,
      "rewards/ReflectionBalance/mean": 0.6551928520202637,
      "rewards/ReflectionBalance/std": 0.3643551915884018,
      "step": 95,
      "train_speed(iter/s)": 0.007282
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 495.203125,
      "completions/min_length": 267.5,
      "epoch": 0.5175202156334232,
      "grad_norm": 90.4681375185632,
      "kl": 0.21734619140625,
      "learning_rate": 9.956427474904888e-07,
      "loss": 0.05293312668800354,
      "memory(GiB)": 75.26,
      "reward": 1.80235356092453,
      "reward_std": 0.4157355725765228,
      "rewards/Format/mean": 0.96484375,
      "rewards/Format/std": 0.17638477310538292,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.339918777346611,
      "rewards/ReflectionBalance/mean": 0.7046972513198853,
      "rewards/ReflectionBalance/std": 0.3154076188802719,
      "step": 96,
      "train_speed(iter/s)": 0.007281
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 539.55078125,
      "completions/min_length": 271.5,
      "epoch": 0.522911051212938,
      "grad_norm": 2.6242267468811704,
      "kl": 0.0308837890625,
      "learning_rate": 9.955290072600386e-07,
      "loss": 0.056684061884880066,
      "memory(GiB)": 75.26,
      "reward": 1.895814597606659,
      "reward_std": 0.501386746764183,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4217475354671478,
      "rewards/ReflectionBalance/mean": 0.6614396572113037,
      "rewards/ReflectionBalance/std": 0.33315137028694153,
      "step": 97,
      "train_speed(iter/s)": 0.007276
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 922.0,
      "completions/mean_length": 473.1015625,
      "completions/min_length": 273.0,
      "epoch": 0.5283018867924528,
      "grad_norm": 2.7599634662429264,
      "kl": 0.0296630859375,
      "learning_rate": 9.95413808242091e-07,
      "loss": 0.04050455242395401,
      "memory(GiB)": 75.26,
      "reward": 1.937968909740448,
      "reward_std": 0.4003048241138458,
      "rewards/Format/mean": 0.98828125,
      "rewards/Format/std": 0.10644762217998505,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.424868568778038,
      "rewards/ReflectionBalance/mean": 0.715312659740448,
      "rewards/ReflectionBalance/std": 0.2969643473625183,
      "step": 98,
      "train_speed(iter/s)": 0.007287
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1014.0,
      "completions/mean_length": 516.05078125,
      "completions/min_length": 276.0,
      "epoch": 0.5336927223719676,
      "grad_norm": 2.4764564106115614,
      "kl": 0.028564453125,
      "learning_rate": 9.952971507757802e-07,
      "loss": 0.06992937624454498,
      "memory(GiB)": 75.26,
      "reward": 1.8167411088943481,
      "reward_std": 0.44749945402145386,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.19007275998592377,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36045311391353607,
      "rewards/ReflectionBalance/mean": 0.6995535492897034,
      "rewards/ReflectionBalance/std": 0.34221331775188446,
      "step": 99,
      "train_speed(iter/s)": 0.007289
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 532.06640625,
      "completions/min_length": 273.0,
      "epoch": 0.5390835579514824,
      "grad_norm": 2.525093615734264,
      "kl": 0.02984619140625,
      "learning_rate": 9.95179035204534e-07,
      "loss": 0.08889700472354889,
      "memory(GiB)": 75.26,
      "reward": 1.7149935364723206,
      "reward_std": 0.48205533623695374,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2255912721157074,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.29434798657894135,
      "rewards/ReflectionBalance/mean": 0.6720247566699982,
      "rewards/ReflectionBalance/std": 0.32762520015239716,
      "step": 100,
      "train_speed(iter/s)": 0.007281
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 517.953125,
      "completions/min_length": 260.5,
      "epoch": 0.5444743935309974,
      "grad_norm": 2.7594378171363316,
      "kl": 0.034912109375,
      "learning_rate": 9.95059461876073e-07,
      "loss": 0.06853577494621277,
      "memory(GiB)": 75.26,
      "reward": 1.7385188937187195,
      "reward_std": 0.4232315868139267,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.31088338047266006,
      "rewards/ReflectionBalance/mean": 0.6643001437187195,
      "rewards/ReflectionBalance/std": 0.3354874700307846,
      "step": 101,
      "train_speed(iter/s)": 0.00728
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 548.90234375,
      "completions/min_length": 310.0,
      "epoch": 0.5498652291105122,
      "grad_norm": 3.8871691045655927,
      "kl": 0.042724609375,
      "learning_rate": 9.949384311424089e-07,
      "loss": 0.08052141964435577,
      "memory(GiB)": 75.26,
      "reward": 1.662149965763092,
      "reward_std": 0.48296093940734863,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2280246615409851,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.28836025297641754,
      "rewards/ReflectionBalance/mean": 0.6191812753677368,
      "rewards/ReflectionBalance/std": 0.37641869485378265,
      "step": 102,
      "train_speed(iter/s)": 0.007272
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 493.8671875,
      "completions/min_length": 276.5,
      "epoch": 0.555256064690027,
      "grad_norm": 2.651980367590372,
      "kl": 0.0328369140625,
      "learning_rate": 9.94815943359844e-07,
      "loss": 0.07148145139217377,
      "memory(GiB)": 75.26,
      "reward": 1.8344134092330933,
      "reward_std": 0.39429205656051636,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.19744773209095,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38201163709163666,
      "rewards/ReflectionBalance/mean": 0.6976946592330933,
      "rewards/ReflectionBalance/std": 0.3063305467367172,
      "step": 103,
      "train_speed(iter/s)": 0.00727
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1011.0,
      "completions/mean_length": 500.2734375,
      "completions/min_length": 239.5,
      "epoch": 0.5606469002695418,
      "grad_norm": 2.503891881535184,
      "kl": 0.02899169921875,
      "learning_rate": 9.9469199888897e-07,
      "loss": 0.06032980978488922,
      "memory(GiB)": 75.26,
      "reward": 1.819568693637848,
      "reward_std": 0.4125630408525467,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.1905922330915928,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35872696340084076,
      "rewards/ReflectionBalance/mean": 0.7101936936378479,
      "rewards/ReflectionBalance/std": 0.29253700375556946,
      "step": 104,
      "train_speed(iter/s)": 0.007268
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 482.12890625,
      "completions/min_length": 245.0,
      "epoch": 0.5660377358490566,
      "grad_norm": 2.6535164446884627,
      "kl": 0.0299072265625,
      "learning_rate": 9.945665980946669e-07,
      "loss": 0.08102186769247055,
      "memory(GiB)": 75.26,
      "reward": 1.859497845172882,
      "reward_std": 0.44738559424877167,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4120787978172302,
      "rewards/ReflectionBalance/mean": 0.7032478749752045,
      "rewards/ReflectionBalance/std": 0.289259597659111,
      "step": 105,
      "train_speed(iter/s)": 0.007274
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 554.1796875,
      "completions/min_length": 233.0,
      "epoch": 0.5714285714285714,
      "grad_norm": 2.428646178690926,
      "kl": 0.03204345703125,
      "learning_rate": 9.944397413461021e-07,
      "loss": 0.05015728250145912,
      "memory(GiB)": 75.26,
      "reward": 1.7092097997665405,
      "reward_std": 0.38970354199409485,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.078125,
      "rewards/MultiModalAccuracyORM/std": 0.258706197142601,
      "rewards/ReflectionBalance/mean": 0.6818660199642181,
      "rewards/ReflectionBalance/std": 0.338120236992836,
      "step": 106,
      "train_speed(iter/s)": 0.007274
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 989.0,
      "completions/mean_length": 543.95703125,
      "completions/min_length": 271.5,
      "epoch": 0.5768194070080862,
      "grad_norm": 2.425307299835807,
      "kl": 0.0343017578125,
      "learning_rate": 9.94311429016729e-07,
      "loss": 0.09082530438899994,
      "memory(GiB)": 75.26,
      "reward": 1.8193864822387695,
      "reward_std": 0.46968382596969604,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24081215262413025,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.37818245589733124,
      "rewards/ReflectionBalance/mean": 0.7100113928318024,
      "rewards/ReflectionBalance/std": 0.311079278588295,
      "step": 107,
      "train_speed(iter/s)": 0.007275
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 531.359375,
      "completions/min_length": 260.0,
      "epoch": 0.5822102425876011,
      "grad_norm": 2.7687339510780387,
      "kl": 0.0322265625,
      "learning_rate": 9.941816614842862e-07,
      "loss": 0.05211513862013817,
      "memory(GiB)": 75.26,
      "reward": 1.8218416571617126,
      "reward_std": 0.5128048062324524,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3890078365802765,
      "rewards/ReflectionBalance/mean": 0.6694979071617126,
      "rewards/ReflectionBalance/std": 0.330733947455883,
      "step": 108,
      "train_speed(iter/s)": 0.007276
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 532.15625,
      "completions/min_length": 252.0,
      "epoch": 0.5876010781671159,
      "grad_norm": 2.5672369179785055,
      "kl": 0.0321044921875,
      "learning_rate": 9.940504391307965e-07,
      "loss": 0.06716234982013702,
      "memory(GiB)": 75.26,
      "reward": 1.8173925876617432,
      "reward_std": 0.3953797370195389,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.19744773209095,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3538244515657425,
      "rewards/ReflectionBalance/mean": 0.7041113376617432,
      "rewards/ReflectionBalance/std": 0.2554439902305603,
      "step": 109,
      "train_speed(iter/s)": 0.007279
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 521.09765625,
      "completions/min_length": 266.5,
      "epoch": 0.5929919137466307,
      "grad_norm": 2.2872679769604267,
      "kl": 0.0281982421875,
      "learning_rate": 9.939177623425647e-07,
      "loss": 0.051641762256622314,
      "memory(GiB)": 75.26,
      "reward": 1.8852799534797668,
      "reward_std": 0.4171188622713089,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24301259219646454,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42852601408958435,
      "rewards/ReflectionBalance/mean": 0.7055924534797668,
      "rewards/ReflectionBalance/std": 0.28353647887706757,
      "step": 110,
      "train_speed(iter/s)": 0.007274
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 966.0,
      "completions/mean_length": 496.62890625,
      "completions/min_length": 266.5,
      "epoch": 0.5983827493261455,
      "grad_norm": 2.5015123970254565,
      "kl": 0.031005859375,
      "learning_rate": 9.937836315101785e-07,
      "loss": 0.03866230696439743,
      "memory(GiB)": 75.26,
      "reward": 1.917551338672638,
      "reward_std": 0.3978566825389862,
      "rewards/Format/mean": 0.984375,
      "rewards/Format/std": 0.1201356090605259,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.40309572219848633,
      "rewards/ReflectionBalance/mean": 0.7027076780796051,
      "rewards/ReflectionBalance/std": 0.3227662742137909,
      "step": 111,
      "train_speed(iter/s)": 0.007273
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 537.3046875,
      "completions/min_length": 289.0,
      "epoch": 0.6037735849056604,
      "grad_norm": 2.3843692566996095,
      "kl": 0.0306396484375,
      "learning_rate": 9.936480470285054e-07,
      "loss": 0.07106906175613403,
      "memory(GiB)": 75.26,
      "reward": 1.8428059220314026,
      "reward_std": 0.4610760360956192,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.19343896210193634,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38837648928165436,
      "rewards/ReflectionBalance/mean": 0.6982747614383698,
      "rewards/ReflectionBalance/std": 0.32250475883483887,
      "step": 112,
      "train_speed(iter/s)": 0.007268
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 966.5,
      "completions/mean_length": 527.17578125,
      "completions/min_length": 274.0,
      "epoch": 0.6091644204851752,
      "grad_norm": 2.418458548940243,
      "kl": 0.035400390625,
      "learning_rate": 9.935110092966923e-07,
      "loss": 0.07103750109672546,
      "memory(GiB)": 75.26,
      "reward": 1.801806390285492,
      "reward_std": 0.42337630689144135,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3405093401670456,
      "rewards/ReflectionBalance/mean": 0.7119626998901367,
      "rewards/ReflectionBalance/std": 0.2727465555071831,
      "step": 113,
      "train_speed(iter/s)": 0.007269
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 970.0,
      "completions/mean_length": 526.04296875,
      "completions/min_length": 277.0,
      "epoch": 0.6145552560646901,
      "grad_norm": 2.392147962649887,
      "kl": 0.0328369140625,
      "learning_rate": 9.933725187181644e-07,
      "loss": 0.047923214733600616,
      "memory(GiB)": 75.26,
      "reward": 1.776918113231659,
      "reward_std": 0.41450418531894684,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21875878423452377,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3300945907831192,
      "rewards/ReflectionBalance/mean": 0.7026993632316589,
      "rewards/ReflectionBalance/std": 0.27047334611415863,
      "step": 114,
      "train_speed(iter/s)": 0.00727
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 522.55859375,
      "completions/min_length": 258.5,
      "epoch": 0.6199460916442049,
      "grad_norm": 2.634425843912101,
      "kl": 0.0386962890625,
      "learning_rate": 9.932325757006246e-07,
      "loss": 0.047136515378952026,
      "memory(GiB)": 75.26,
      "reward": 1.7823671102523804,
      "reward_std": 0.43827536702156067,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20146947354078293,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3174293115735054,
      "rewards/ReflectionBalance/mean": 0.7003358006477356,
      "rewards/ReflectionBalance/std": 0.296391099691391,
      "step": 115,
      "train_speed(iter/s)": 0.007267
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 516.6484375,
      "completions/min_length": 250.5,
      "epoch": 0.6253369272237197,
      "grad_norm": 2.4228581386194272,
      "kl": 0.0360107421875,
      "learning_rate": 9.930911806560502e-07,
      "loss": 0.06301569938659668,
      "memory(GiB)": 75.26,
      "reward": 1.8121304512023926,
      "reward_std": 0.40884095430374146,
      "rewards/Format/mean": 0.96875,
      "rewards/Format/std": 0.1746762990951538,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3460448384284973,
      "rewards/ReflectionBalance/mean": 0.7027555704116821,
      "rewards/ReflectionBalance/std": 0.2908548265695572,
      "step": 116,
      "train_speed(iter/s)": 0.007266
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 532.4765625,
      "completions/min_length": 265.0,
      "epoch": 0.6307277628032345,
      "grad_norm": 2.3659720828891415,
      "kl": 0.03564453125,
      "learning_rate": 9.929483340006948e-07,
      "loss": 0.05715256556868553,
      "memory(GiB)": 75.26,
      "reward": 1.8573755025863647,
      "reward_std": 0.4400773197412491,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.20884444564580917,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.7167505025863647,
      "rewards/ReflectionBalance/std": 0.2859955281019211,
      "step": 117,
      "train_speed(iter/s)": 0.007267
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 496.45703125,
      "completions/min_length": 236.0,
      "epoch": 0.6361185983827493,
      "grad_norm": 3.1042413157321045,
      "kl": 0.03857421875,
      "learning_rate": 9.92804036155084e-07,
      "loss": 0.08855443447828293,
      "memory(GiB)": 75.26,
      "reward": 1.869972288608551,
      "reward_std": 0.38912469148635864,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.2156769335269928,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.38974663615226746,
      "rewards/ReflectionBalance/mean": 0.7215348184108734,
      "rewards/ReflectionBalance/std": 0.2846880257129669,
      "step": 118,
      "train_speed(iter/s)": 0.007265
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1006.5,
      "completions/mean_length": 510.4140625,
      "completions/min_length": 263.5,
      "epoch": 0.6415094339622641,
      "grad_norm": 2.602507478861046,
      "kl": 0.0413818359375,
      "learning_rate": 9.926582875440163e-07,
      "loss": 0.04564861208200455,
      "memory(GiB)": 75.26,
      "reward": 1.8369030952453613,
      "reward_std": 0.44171082973480225,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.19343896210193634,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3774644732475281,
      "rewards/ReflectionBalance/mean": 0.7040906548500061,
      "rewards/ReflectionBalance/std": 0.3105480968952179,
      "step": 119,
      "train_speed(iter/s)": 0.007263
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 992.0,
      "completions/mean_length": 523.44921875,
      "completions/min_length": 310.0,
      "epoch": 0.6469002695417789,
      "grad_norm": 2.499294420556076,
      "kl": 0.036865234375,
      "learning_rate": 9.925110885965609e-07,
      "loss": 0.07180796563625336,
      "memory(GiB)": 75.26,
      "reward": 1.7475504279136658,
      "reward_std": 0.3358819782733917,
      "rewards/Format/mean": 0.98046875,
      "rewards/Format/std": 0.13153232261538506,
      "rewards/MultiModalAccuracyORM/mean": 0.05859375,
      "rewards/MultiModalAccuracyORM/std": 0.2280246615409851,
      "rewards/ReflectionBalance/mean": 0.7084879279136658,
      "rewards/ReflectionBalance/std": 0.3283933848142624,
      "step": 120,
      "train_speed(iter/s)": 0.007261
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 522.6328125,
      "completions/min_length": 261.5,
      "epoch": 0.6522911051212938,
      "grad_norm": 2.429262434436656,
      "kl": 0.036376953125,
      "learning_rate": 9.92362439746057e-07,
      "loss": 0.059392333030700684,
      "memory(GiB)": 75.26,
      "reward": 1.7388931512832642,
      "reward_std": 0.4270232617855072,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21220162510871887,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3270204961299896,
      "rewards/ReflectionBalance/mean": 0.6646745204925537,
      "rewards/ReflectionBalance/std": 0.344754159450531,
      "step": 121,
      "train_speed(iter/s)": 0.007259
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 525.453125,
      "completions/min_length": 254.0,
      "epoch": 0.6576819407008087,
      "grad_norm": 124.70572664585856,
      "kl": 0.2757568359375,
      "learning_rate": 9.922123414301112e-07,
      "loss": 0.060740333050489426,
      "memory(GiB)": 75.26,
      "reward": 1.8766624927520752,
      "reward_std": 0.42697368562221527,
      "rewards/Format/mean": 0.96875,
      "rewards/Format/std": 0.1746762990951538,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3853980004787445,
      "rewards/ReflectionBalance/mean": 0.7204125821590424,
      "rewards/ReflectionBalance/std": 0.2710004299879074,
      "step": 122,
      "train_speed(iter/s)": 0.00726
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 522.25,
      "completions/min_length": 242.0,
      "epoch": 0.6630727762803235,
      "grad_norm": 2.5882390896351968,
      "kl": 0.0394287109375,
      "learning_rate": 9.920607940905983e-07,
      "loss": 0.0679333284497261,
      "memory(GiB)": 75.26,
      "reward": 1.8972849249839783,
      "reward_std": 0.47602346539497375,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.265625,
      "rewards/MultiModalAccuracyORM/std": 0.4343355745077133,
      "rewards/ReflectionBalance/mean": 0.6863473355770111,
      "rewards/ReflectionBalance/std": 0.3259919285774231,
      "step": 123,
      "train_speed(iter/s)": 0.007257
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1005.0,
      "completions/mean_length": 505.05078125,
      "completions/min_length": 270.0,
      "epoch": 0.6684636118598383,
      "grad_norm": 2.6297580800724587,
      "kl": 0.0390625,
      "learning_rate": 9.919077981736586e-07,
      "loss": 0.03863496333360672,
      "memory(GiB)": 75.26,
      "reward": 1.8976187705993652,
      "reward_std": 0.41352391242980957,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.21662794798612595,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4065109044313431,
      "rewards/ReflectionBalance/mean": 0.74527508020401,
      "rewards/ReflectionBalance/std": 0.2683292031288147,
      "step": 124,
      "train_speed(iter/s)": 0.00726
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 510.046875,
      "completions/min_length": 248.5,
      "epoch": 0.6738544474393531,
      "grad_norm": 2.5705812901102094,
      "kl": 0.0399169921875,
      "learning_rate": 9.917533541296962e-07,
      "loss": 0.07674337923526764,
      "memory(GiB)": 75.26,
      "reward": 1.9571890830993652,
      "reward_std": 0.5406909286975861,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.20428021997213364,
      "rewards/MultiModalAccuracyORM/mean": 0.2890625,
      "rewards/MultiModalAccuracyORM/std": 0.4530448317527771,
      "rewards/ReflectionBalance/mean": 0.71500164270401,
      "rewards/ReflectionBalance/std": 0.33675652742385864,
      "step": 125,
      "train_speed(iter/s)": 0.007266
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 990.0,
      "completions/mean_length": 506.9609375,
      "completions/min_length": 268.5,
      "epoch": 0.6792452830188679,
      "grad_norm": 4.562668380786877,
      "kl": 0.039306640625,
      "learning_rate": 9.915974624133788e-07,
      "loss": 0.09044257551431656,
      "memory(GiB)": 75.26,
      "reward": 1.8936432600021362,
      "reward_std": 0.4798816740512848,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20146947354078293,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41328008472919464,
      "rewards/ReflectionBalance/mean": 0.7178620100021362,
      "rewards/ReflectionBalance/std": 0.329559788107872,
      "step": 126,
      "train_speed(iter/s)": 0.007269
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 523.13671875,
      "completions/min_length": 254.5,
      "epoch": 0.6846361185983828,
      "grad_norm": 2.63971978016663,
      "kl": 0.0416259765625,
      "learning_rate": 9.914401234836362e-07,
      "loss": 0.05946851521730423,
      "memory(GiB)": 75.26,
      "reward": 1.7422285079956055,
      "reward_std": 0.39812250435352325,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.08203125,
      "rewards/MultiModalAccuracyORM/std": 0.27079907804727554,
      "rewards/ReflectionBalance/mean": 0.7031660079956055,
      "rewards/ReflectionBalance/std": 0.3124641925096512,
      "step": 127,
      "train_speed(iter/s)": 0.007267
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 533.171875,
      "completions/min_length": 278.0,
      "epoch": 0.6900269541778976,
      "grad_norm": 2.424356104088641,
      "kl": 0.0411376953125,
      "learning_rate": 9.912813378036584e-07,
      "loss": 0.06573596596717834,
      "memory(GiB)": 75.26,
      "reward": 1.8438280820846558,
      "reward_std": 0.4078674614429474,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22204948961734772,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3787541687488556,
      "rewards/ReflectionBalance/mean": 0.7266406118869781,
      "rewards/ReflectionBalance/std": 0.28269290924072266,
      "step": 128,
      "train_speed(iter/s)": 0.00726
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 522.1875,
      "completions/min_length": 272.0,
      "epoch": 0.6954177897574124,
      "grad_norm": 2.4049225200226667,
      "kl": 0.041015625,
      "learning_rate": 9.911211058408942e-07,
      "loss": 0.049022965133190155,
      "memory(GiB)": 75.26,
      "reward": 1.891539216041565,
      "reward_std": 0.36699289083480835,
      "rewards/Format/mean": 0.96484375,
      "rewards/Format/std": 0.1845906376838684,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39068229496479034,
      "rewards/ReflectionBalance/mean": 0.7391954958438873,
      "rewards/ReflectionBalance/std": 0.255997434258461,
      "step": 129,
      "train_speed(iter/s)": 0.007262
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 523.59375,
      "completions/min_length": 259.0,
      "epoch": 0.7008086253369272,
      "grad_norm": 2.466457286453367,
      "kl": 0.038330078125,
      "learning_rate": 9.909594280670506e-07,
      "loss": 0.08324560523033142,
      "memory(GiB)": 75.26,
      "reward": 1.832780122756958,
      "reward_std": 0.49508780241012573,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.2156769335269928,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3787541687488556,
      "rewards/ReflectionBalance/mean": 0.711686372756958,
      "rewards/ReflectionBalance/std": 0.33561091125011444,
      "step": 130,
      "train_speed(iter/s)": 0.007262
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 986.0,
      "completions/mean_length": 526.5390625,
      "completions/min_length": 275.0,
      "epoch": 0.706199460916442,
      "grad_norm": 2.5347077528418764,
      "kl": 0.0435791015625,
      "learning_rate": 9.907963049580904e-07,
      "loss": 0.04103754833340645,
      "memory(GiB)": 75.26,
      "reward": 1.7806856632232666,
      "reward_std": 0.4077284634113312,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3410066068172455,
      "rewards/ReflectionBalance/mean": 0.6869356334209442,
      "rewards/ReflectionBalance/std": 0.2891972064971924,
      "step": 131,
      "train_speed(iter/s)": 0.007263
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 549.48046875,
      "completions/min_length": 248.5,
      "epoch": 0.7115902964959568,
      "grad_norm": 4.225396391641813,
      "kl": 0.045166015625,
      "learning_rate": 9.90631736994232e-07,
      "loss": 0.08308522403240204,
      "memory(GiB)": 75.26,
      "reward": 1.8005964756011963,
      "reward_std": 0.3964887708425522,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.194504976272583,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.339918777346611,
      "rewards/ReflectionBalance/mean": 0.7068465054035187,
      "rewards/ReflectionBalance/std": 0.3257136046886444,
      "step": 132,
      "train_speed(iter/s)": 0.007258
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1014.0,
      "completions/mean_length": 528.72265625,
      "completions/min_length": 282.0,
      "epoch": 0.7169811320754716,
      "grad_norm": 2.3764498299355585,
      "kl": 0.043701171875,
      "learning_rate": 9.904657246599465e-07,
      "loss": 0.023471739143133163,
      "memory(GiB)": 75.26,
      "reward": 1.8210994005203247,
      "reward_std": 0.39819763600826263,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.1969647891819477,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.3338889628648758,
      "rewards/ReflectionBalance/mean": 0.7390680909156799,
      "rewards/ReflectionBalance/std": 0.27204541862010956,
      "step": 133,
      "train_speed(iter/s)": 0.007257
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 568.05859375,
      "completions/min_length": 272.5,
      "epoch": 0.7223719676549866,
      "grad_norm": 2.266559831369006,
      "kl": 0.040771484375,
      "learning_rate": 9.902982684439579e-07,
      "loss": 0.05516345053911209,
      "memory(GiB)": 75.26,
      "reward": 1.7726178169250488,
      "reward_std": 0.4472382217645645,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3437131494283676,
      "rewards/ReflectionBalance/mean": 0.6905866265296936,
      "rewards/ReflectionBalance/std": 0.33134905993938446,
      "step": 134,
      "train_speed(iter/s)": 0.007251
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 574.33203125,
      "completions/min_length": 285.0,
      "epoch": 0.7277628032345014,
      "grad_norm": 2.261726670315607,
      "kl": 0.0430908203125,
      "learning_rate": 9.901293688392401e-07,
      "loss": 0.07126966118812561,
      "memory(GiB)": 75.26,
      "reward": 1.7347480058670044,
      "reward_std": 0.4827611446380615,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2487918585538864,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3300945907831192,
      "rewards/ReflectionBalance/mean": 0.6839666962623596,
      "rewards/ReflectionBalance/std": 0.33294424414634705,
      "step": 135,
      "train_speed(iter/s)": 0.007246
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 516.23828125,
      "completions/min_length": 247.0,
      "epoch": 0.7331536388140162,
      "grad_norm": 3.5173600938543923,
      "kl": 0.0479736328125,
      "learning_rate": 9.899590263430167e-07,
      "loss": 0.07372212409973145,
      "memory(GiB)": 75.26,
      "reward": 1.751655399799347,
      "reward_std": 0.3666611462831497,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.0703125,
      "rewards/MultiModalAccuracyORM/std": 0.2548178359866142,
      "rewards/ReflectionBalance/mean": 0.7321241497993469,
      "rewards/ReflectionBalance/std": 0.26173239946365356,
      "step": 136,
      "train_speed(iter/s)": 0.007247
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 493.4453125,
      "completions/min_length": 238.0,
      "epoch": 0.738544474393531,
      "grad_norm": 9.696899919051507,
      "kl": 0.0457763671875,
      "learning_rate": 9.897872414567587e-07,
      "loss": 0.05223765969276428,
      "memory(GiB)": 75.26,
      "reward": 1.8114017844200134,
      "reward_std": 0.32482780516147614,
      "rewards/Format/mean": 0.984375,
      "rewards/Format/std": 0.1201356090605259,
      "rewards/MultiModalAccuracyORM/mean": 0.08203125,
      "rewards/MultiModalAccuracyORM/std": 0.27079907804727554,
      "rewards/ReflectionBalance/mean": 0.7449954450130463,
      "rewards/ReflectionBalance/std": 0.25622280687093735,
      "step": 137,
      "train_speed(iter/s)": 0.007249
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1016.0,
      "completions/mean_length": 496.91796875,
      "completions/min_length": 258.5,
      "epoch": 0.7439353099730458,
      "grad_norm": 2.4826992651936908,
      "kl": 0.046142578125,
      "learning_rate": 9.896140146861838e-07,
      "loss": 0.04647716507315636,
      "memory(GiB)": 75.26,
      "reward": 1.8228803873062134,
      "reward_std": 0.3217468112707138,
      "rewards/Format/mean": 0.98046875,
      "rewards/Format/std": 0.13153232261538506,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.33116789162158966,
      "rewards/ReflectionBalance/mean": 0.717411607503891,
      "rewards/ReflectionBalance/std": 0.28832660615444183,
      "step": 138,
      "train_speed(iter/s)": 0.007245
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 542.421875,
      "completions/min_length": 247.5,
      "epoch": 0.7493261455525606,
      "grad_norm": 2.6156944224896685,
      "kl": 0.0504150390625,
      "learning_rate": 9.894393465412538e-07,
      "loss": 0.05658886581659317,
      "memory(GiB)": 75.26,
      "reward": 1.8509293794631958,
      "reward_std": 0.4287025034427643,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20146947354078293,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39110109210014343,
      "rewards/ReflectionBalance/mean": 0.702491819858551,
      "rewards/ReflectionBalance/std": 0.2757936045527458,
      "step": 139,
      "train_speed(iter/s)": 0.007247
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 540.5390625,
      "completions/min_length": 267.5,
      "epoch": 0.7547169811320755,
      "grad_norm": 2.374371830698942,
      "kl": 0.04150390625,
      "learning_rate": 9.892632375361743e-07,
      "loss": 0.07994107902050018,
      "memory(GiB)": 75.26,
      "reward": 1.8126057982444763,
      "reward_std": 0.43652716279029846,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.19744773209095,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.35190291702747345,
      "rewards/ReflectionBalance/mean": 0.7110432982444763,
      "rewards/ReflectionBalance/std": 0.29203906655311584,
      "step": 140,
      "train_speed(iter/s)": 0.007247
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 527.578125,
      "completions/min_length": 272.0,
      "epoch": 0.7601078167115903,
      "grad_norm": 2.417016910376276,
      "kl": 0.04296875,
      "learning_rate": 9.890856881893925e-07,
      "loss": 0.0422799214720726,
      "memory(GiB)": 75.26,
      "reward": 1.8612421751022339,
      "reward_std": 0.337089404463768,
      "rewards/Format/mean": 0.984375,
      "rewards/Format/std": 0.12450689822435379,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.35643893480300903,
      "rewards/ReflectionBalance/mean": 0.7167109251022339,
      "rewards/ReflectionBalance/std": 0.288787379860878,
      "step": 141,
      "train_speed(iter/s)": 0.007245
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 956.5,
      "completions/mean_length": 513.07421875,
      "completions/min_length": 272.5,
      "epoch": 0.7654986522911051,
      "grad_norm": 2.3679540425474723,
      "kl": 0.04443359375,
      "learning_rate": 9.889066990235958e-07,
      "loss": 0.059715721756219864,
      "memory(GiB)": 75.26,
      "reward": 1.8664844036102295,
      "reward_std": 0.4286072105169296,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21138381212949753,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.38954539597034454,
      "rewards/ReflectionBalance/mean": 0.7219531536102295,
      "rewards/ReflectionBalance/std": 0.3038695156574249,
      "step": 142,
      "train_speed(iter/s)": 0.007251
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 524.6640625,
      "completions/min_length": 252.0,
      "epoch": 0.77088948787062,
      "grad_norm": 2.355373471287977,
      "kl": 0.0440673828125,
      "learning_rate": 9.887262705657104e-07,
      "loss": 0.0647086352109909,
      "memory(GiB)": 75.26,
      "reward": 1.8572739958763123,
      "reward_std": 0.4884045422077179,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21875878423452377,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38864077627658844,
      "rewards/ReflectionBalance/mean": 0.7244614362716675,
      "rewards/ReflectionBalance/std": 0.285247340798378,
      "step": 143,
      "train_speed(iter/s)": 0.007249
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 555.35546875,
      "completions/min_length": 296.5,
      "epoch": 0.7762803234501348,
      "grad_norm": 2.471292996324568,
      "kl": 0.0518798828125,
      "learning_rate": 9.885444033468993e-07,
      "loss": 0.06025347113609314,
      "memory(GiB)": 75.26,
      "reward": 1.8132990598678589,
      "reward_std": 0.45233288407325745,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21875878423452377,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3749772012233734,
      "rewards/ReflectionBalance/mean": 0.6961116194725037,
      "rewards/ReflectionBalance/std": 0.3277191072702408,
      "step": 144,
      "train_speed(iter/s)": 0.00725
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 997.5,
      "completions/mean_length": 520.15234375,
      "completions/min_length": 290.5,
      "epoch": 0.7816711590296496,
      "grad_norm": 2.4259268793917266,
      "kl": 0.0540771484375,
      "learning_rate": 9.883610979025614e-07,
      "loss": 0.04816053807735443,
      "memory(GiB)": 75.26,
      "reward": 1.8285534381866455,
      "reward_std": 0.3492733985185623,
      "rewards/Format/mean": 0.96875,
      "rewards/Format/std": 0.1746762990951538,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3098885118961334,
      "rewards/ReflectionBalance/mean": 0.7230846285820007,
      "rewards/ReflectionBalance/std": 0.2862326353788376,
      "step": 145,
      "train_speed(iter/s)": 0.007252
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 544.03515625,
      "completions/min_length": 259.0,
      "epoch": 0.7870619946091644,
      "grad_norm": 2.273549461790945,
      "kl": 0.0482177734375,
      "learning_rate": 9.881763547723292e-07,
      "loss": 0.09595300257205963,
      "memory(GiB)": 75.26,
      "reward": 1.838685691356659,
      "reward_std": 0.42961448431015015,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.38181672990322113,
      "rewards/ReflectionBalance/mean": 0.7214982211589813,
      "rewards/ReflectionBalance/std": 0.2940831184387207,
      "step": 146,
      "train_speed(iter/s)": 0.007245
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 541.79296875,
      "completions/min_length": 257.0,
      "epoch": 0.7924528301886793,
      "grad_norm": 2.467909021960107,
      "kl": 0.056396484375,
      "learning_rate": 9.879901745000684e-07,
      "loss": 0.06027314066886902,
      "memory(GiB)": 75.26,
      "reward": 1.7789193391799927,
      "reward_std": 0.41698458790779114,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21220162510871887,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3375232368707657,
      "rewards/ReflectionBalance/mean": 0.6929817497730255,
      "rewards/ReflectionBalance/std": 0.295562744140625,
      "step": 147,
      "train_speed(iter/s)": 0.007245
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 547.69921875,
      "completions/min_length": 284.5,
      "epoch": 0.7978436657681941,
      "grad_norm": 2.4077619807071082,
      "kl": 0.0592041015625,
      "learning_rate": 9.878025576338745e-07,
      "loss": 0.04321632534265518,
      "memory(GiB)": 75.26,
      "reward": 1.7941755652427673,
      "reward_std": 0.4593184292316437,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2524137645959854,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.35190291702747345,
      "rewards/ReflectionBalance/mean": 0.7199568450450897,
      "rewards/ReflectionBalance/std": 0.25139304995536804,
      "step": 148,
      "train_speed(iter/s)": 0.007244
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 512.875,
      "completions/min_length": 272.5,
      "epoch": 0.8032345013477089,
      "grad_norm": 2.4223861003998075,
      "kl": 0.05322265625,
      "learning_rate": 9.876135047260728e-07,
      "loss": 0.036728546023368835,
      "memory(GiB)": 75.26,
      "reward": 1.9216273427009583,
      "reward_std": 0.3766011744737625,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.19007275998592377,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.41096827387809753,
      "rewards/ReflectionBalance/mean": 0.7458461225032806,
      "rewards/ReflectionBalance/std": 0.25930534303188324,
      "step": 149,
      "train_speed(iter/s)": 0.007239
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 529.88671875,
      "completions/min_length": 257.0,
      "epoch": 0.8086253369272237,
      "grad_norm": 2.274808110826725,
      "kl": 0.0552978515625,
      "learning_rate": 9.874230163332163e-07,
      "loss": 0.0716937780380249,
      "memory(GiB)": 75.26,
      "reward": 1.784624695777893,
      "reward_std": 0.4029228687286377,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.3077666610479355,
      "rewards/ReflectionBalance/mean": 0.7455621957778931,
      "rewards/ReflectionBalance/std": 0.2711905241012573,
      "step": 150,
      "train_speed(iter/s)": 0.007239
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 522.62890625,
      "completions/min_length": 264.5,
      "epoch": 0.8140161725067385,
      "grad_norm": 2.38045789309645,
      "kl": 0.0518798828125,
      "learning_rate": 9.87231093016083e-07,
      "loss": 0.06613266468048096,
      "memory(GiB)": 75.26,
      "reward": 1.9189192652702332,
      "reward_std": 0.43947093188762665,
      "rewards/Format/mean": 0.96875,
      "rewards/Format/std": 0.1746762990951538,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.423653244972229,
      "rewards/ReflectionBalance/mean": 0.7157942652702332,
      "rewards/ReflectionBalance/std": 0.3112916350364685,
      "step": 151,
      "train_speed(iter/s)": 0.00724
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 538.6953125,
      "completions/min_length": 237.0,
      "epoch": 0.8194070080862533,
      "grad_norm": 2.3975316515752048,
      "kl": 0.054931640625,
      "learning_rate": 9.870377353396765e-07,
      "loss": 0.07492934167385101,
      "memory(GiB)": 75.26,
      "reward": 1.8027400970458984,
      "reward_std": 0.4112595319747925,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35538628697395325,
      "rewards/ReflectionBalance/mean": 0.7050837874412537,
      "rewards/ReflectionBalance/std": 0.2922307550907135,
      "step": 152,
      "train_speed(iter/s)": 0.007238
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 557.39453125,
      "completions/min_length": 300.0,
      "epoch": 0.8247978436657682,
      "grad_norm": 2.3078966204851654,
      "kl": 0.06396484375,
      "learning_rate": 9.86842943873222e-07,
      "loss": 0.06120332330465317,
      "memory(GiB)": 75.26,
      "reward": 1.867418646812439,
      "reward_std": 0.47704167664051056,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23443959653377533,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.4145888388156891,
      "rewards/ReflectionBalance/mean": 0.7072623670101166,
      "rewards/ReflectionBalance/std": 0.3004324585199356,
      "step": 153,
      "train_speed(iter/s)": 0.007235
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 566.06640625,
      "completions/min_length": 294.0,
      "epoch": 0.8301886792452831,
      "grad_norm": 2.242836020054921,
      "kl": 0.0606689453125,
      "learning_rate": 9.866467191901659e-07,
      "loss": 0.0848696231842041,
      "memory(GiB)": 75.26,
      "reward": 1.7856167554855347,
      "reward_std": 0.46216243505477905,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2566775679588318,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.3338889628648758,
      "rewards/ReflectionBalance/mean": 0.7270230948925018,
      "rewards/ReflectionBalance/std": 0.2833152413368225,
      "step": 154,
      "train_speed(iter/s)": 0.007234
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 527.71875,
      "completions/min_length": 244.5,
      "epoch": 0.8355795148247979,
      "grad_norm": 2.497938803698393,
      "kl": 0.062255859375,
      "learning_rate": 9.86449061868174e-07,
      "loss": 0.0554848313331604,
      "memory(GiB)": 75.26,
      "reward": 1.8672205209732056,
      "reward_std": 0.4291422963142395,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23196382820606232,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3761351853609085,
      "rewards/ReflectionBalance/mean": 0.7500330209732056,
      "rewards/ReflectionBalance/std": 0.24969593435525894,
      "step": 155,
      "train_speed(iter/s)": 0.007232
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 981.0,
      "completions/mean_length": 501.33984375,
      "completions/min_length": 240.0,
      "epoch": 0.8409703504043127,
      "grad_norm": 49.809267001597966,
      "kl": 0.1053466796875,
      "learning_rate": 9.862499724891297e-07,
      "loss": 0.019649073481559753,
      "memory(GiB)": 75.26,
      "reward": 1.7932943105697632,
      "reward_std": 0.3883666396141052,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21875878423452377,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.3082972764968872,
      "rewards/ReflectionBalance/mean": 0.7386067807674408,
      "rewards/ReflectionBalance/std": 0.2553929015994072,
      "step": 156,
      "train_speed(iter/s)": 0.007235
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 510.6953125,
      "completions/min_length": 264.0,
      "epoch": 0.8463611859838275,
      "grad_norm": 39471.16303215642,
      "kl": 82.5274658203125,
      "learning_rate": 9.86049451639132e-07,
      "loss": 4.186208724975586,
      "memory(GiB)": 75.26,
      "reward": 1.8744717836380005,
      "reward_std": 0.37302207946777344,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.382988765835762,
      "rewards/ReflectionBalance/mean": 0.7416592836380005,
      "rewards/ReflectionBalance/std": 0.2591061145067215,
      "step": 157,
      "train_speed(iter/s)": 0.007237
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 533.15234375,
      "completions/min_length": 265.0,
      "epoch": 0.8517520215633423,
      "grad_norm": 2.3867872106569408,
      "kl": 0.06494140625,
      "learning_rate": 9.858474999084939e-07,
      "loss": 0.05635862052440643,
      "memory(GiB)": 75.26,
      "reward": 1.7909361720085144,
      "reward_std": 0.4968418776988983,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35358233749866486,
      "rewards/ReflectionBalance/mean": 0.7049987018108368,
      "rewards/ReflectionBalance/std": 0.2969968914985657,
      "step": 158,
      "train_speed(iter/s)": 0.007237
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 994.5,
      "completions/mean_length": 536.2734375,
      "completions/min_length": 261.0,
      "epoch": 0.8571428571428571,
      "grad_norm": 2.4353869799698344,
      "kl": 0.07080078125,
      "learning_rate": 9.856441178917412e-07,
      "loss": 0.05109911784529686,
      "memory(GiB)": 75.26,
      "reward": 1.8697957396507263,
      "reward_std": 0.5364424884319305,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24356544017791748,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4270860105752945,
      "rewards/ReflectionBalance/mean": 0.6979207396507263,
      "rewards/ReflectionBalance/std": 0.3220529705286026,
      "step": 159,
      "train_speed(iter/s)": 0.00724
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 529.84765625,
      "completions/min_length": 282.0,
      "epoch": 0.862533692722372,
      "grad_norm": 2.4512421161602225,
      "kl": 0.0667724609375,
      "learning_rate": 9.854393061876097e-07,
      "loss": 0.06813901662826538,
      "memory(GiB)": 75.26,
      "reward": 1.7535673379898071,
      "reward_std": 0.45131799578666687,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22826264798641205,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.2918590381741524,
      "rewards/ReflectionBalance/mean": 0.7105986177921295,
      "rewards/ReflectionBalance/std": 0.3051357567310333,
      "step": 160,
      "train_speed(iter/s)": 0.00724
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 966.5,
      "completions/mean_length": 536.43359375,
      "completions/min_length": 297.5,
      "epoch": 0.8679245283018868,
      "grad_norm": 2.3730650568767353,
      "kl": 0.066162109375,
      "learning_rate": 9.852330653990452e-07,
      "loss": 0.026312701404094696,
      "memory(GiB)": 75.26,
      "reward": 1.8915584087371826,
      "reward_std": 0.3815688341856003,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20146947354078293,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3786114752292633,
      "rewards/ReflectionBalance/mean": 0.762652188539505,
      "rewards/ReflectionBalance/std": 0.24570902436971664,
      "step": 161,
      "train_speed(iter/s)": 0.007243
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 534.734375,
      "completions/min_length": 266.0,
      "epoch": 0.8733153638814016,
      "grad_norm": 2.2930770405138183,
      "kl": 0.069091796875,
      "learning_rate": 9.85025396133199e-07,
      "loss": 0.053162239491939545,
      "memory(GiB)": 75.26,
      "reward": 1.8305073380470276,
      "reward_std": 0.3784237653017044,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24301259219646454,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.33148839324712753,
      "rewards/ReflectionBalance/mean": 0.7406635880470276,
      "rewards/ReflectionBalance/std": 0.23897293210029602,
      "step": 162,
      "train_speed(iter/s)": 0.007243
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 511.9609375,
      "completions/min_length": 240.5,
      "epoch": 0.8787061994609164,
      "grad_norm": 2.5417472495838607,
      "kl": 0.06982421875,
      "learning_rate": 9.848162990014288e-07,
      "loss": 0.09088842570781708,
      "memory(GiB)": 75.26,
      "reward": 1.8022525906562805,
      "reward_std": 0.4178864061832428,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21065277606248856,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3490002751350403,
      "rewards/ReflectionBalance/mean": 0.7124088406562805,
      "rewards/ReflectionBalance/std": 0.30794723331928253,
      "step": 163,
      "train_speed(iter/s)": 0.007242
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 492.48828125,
      "completions/min_length": 228.0,
      "epoch": 0.8840970350404312,
      "grad_norm": 2.484074275122516,
      "kl": 0.0634765625,
      "learning_rate": 9.846057746192957e-07,
      "loss": 0.06331992149353027,
      "memory(GiB)": 75.26,
      "reward": 1.9215380549430847,
      "reward_std": 0.44639089703559875,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4094643145799637,
      "rewards/ReflectionBalance/mean": 0.7535693645477295,
      "rewards/ReflectionBalance/std": 0.2832320109009743,
      "step": 164,
      "train_speed(iter/s)": 0.007242
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 531.41015625,
      "completions/min_length": 295.5,
      "epoch": 0.889487870619946,
      "grad_norm": 2.4558709906524196,
      "kl": 0.080810546875,
      "learning_rate": 9.843938236065618e-07,
      "loss": 0.06329581141471863,
      "memory(GiB)": 75.26,
      "reward": 1.8918852806091309,
      "reward_std": 0.49117377400398254,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4225567579269409,
      "rewards/ReflectionBalance/mean": 0.7161040008068085,
      "rewards/ReflectionBalance/std": 0.30901142954826355,
      "step": 165,
      "train_speed(iter/s)": 0.007244
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 519.4296875,
      "completions/min_length": 284.5,
      "epoch": 0.894878706199461,
      "grad_norm": 2.499110002233806,
      "kl": 0.0693359375,
      "learning_rate": 9.841804465871902e-07,
      "loss": 0.07048840820789337,
      "memory(GiB)": 75.26,
      "reward": 1.7886016964912415,
      "reward_std": 0.4514082223176956,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.33636194467544556,
      "rewards/ReflectionBalance/mean": 0.7221954762935638,
      "rewards/ReflectionBalance/std": 0.28301844000816345,
      "step": 166,
      "train_speed(iter/s)": 0.007245
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 985.0,
      "completions/mean_length": 520.44140625,
      "completions/min_length": 263.0,
      "epoch": 0.9002695417789758,
      "grad_norm": 2.4947346131284256,
      "kl": 0.0791015625,
      "learning_rate": 9.839656441893408e-07,
      "loss": 0.03291773423552513,
      "memory(GiB)": 75.26,
      "reward": 1.8011277318000793,
      "reward_std": 0.41115498542785645,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23443959653377533,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.2999713644385338,
      "rewards/ReflectionBalance/mean": 0.7542527318000793,
      "rewards/ReflectionBalance/std": 0.2401454895734787,
      "step": 167,
      "train_speed(iter/s)": 0.007243
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 543.55859375,
      "completions/min_length": 293.0,
      "epoch": 0.9056603773584906,
      "grad_norm": 2.3863803858881494,
      "kl": 0.08203125,
      "learning_rate": 9.837494170453705e-07,
      "loss": 0.07109785825014114,
      "memory(GiB)": 75.26,
      "reward": 1.8066160082817078,
      "reward_std": 0.45132267475128174,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3538244515657425,
      "rewards/ReflectionBalance/mean": 0.7089597582817078,
      "rewards/ReflectionBalance/std": 0.3272208273410797,
      "step": 168,
      "train_speed(iter/s)": 0.007245
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 505.8984375,
      "completions/min_length": 218.0,
      "epoch": 0.9110512129380054,
      "grad_norm": 2.4485460478276604,
      "kl": 0.079833984375,
      "learning_rate": 9.835317657918302e-07,
      "loss": 0.06619758903980255,
      "memory(GiB)": 75.26,
      "reward": 1.913867712020874,
      "reward_std": 0.4443027377128601,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21065277606248856,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.420418843626976,
      "rewards/ReflectionBalance/mean": 0.7263676822185516,
      "rewards/ReflectionBalance/std": 0.3014582693576813,
      "step": 169,
      "train_speed(iter/s)": 0.007247
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 940.0,
      "completions/mean_length": 502.19140625,
      "completions/min_length": 282.0,
      "epoch": 0.9164420485175202,
      "grad_norm": 2.421580656961874,
      "kl": 0.080810546875,
      "learning_rate": 9.833126910694632e-07,
      "loss": 0.029241811484098434,
      "memory(GiB)": 75.26,
      "reward": 1.8327381610870361,
      "reward_std": 0.40365883708000183,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21220162510871887,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3425614684820175,
      "rewards/ReflectionBalance/mean": 0.7428943812847137,
      "rewards/ReflectionBalance/std": 0.2540576532483101,
      "step": 170,
      "train_speed(iter/s)": 0.007251
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 943.5,
      "completions/mean_length": 484.8046875,
      "completions/min_length": 243.5,
      "epoch": 0.921832884097035,
      "grad_norm": 2.7610490648597223,
      "kl": 0.09033203125,
      "learning_rate": 9.830921935232035e-07,
      "loss": 0.0749974399805069,
      "memory(GiB)": 75.26,
      "reward": 1.8435319066047668,
      "reward_std": 0.40985214710235596,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3597298264503479,
      "rewards/ReflectionBalance/mean": 0.7341568768024445,
      "rewards/ReflectionBalance/std": 0.2553749084472656,
      "step": 171,
      "train_speed(iter/s)": 0.007255
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1009.5,
      "completions/mean_length": 532.890625,
      "completions/min_length": 268.0,
      "epoch": 0.9272237196765498,
      "grad_norm": 2.2149329140978966,
      "kl": 0.071044921875,
      "learning_rate": 9.828702738021734e-07,
      "loss": 0.07008306682109833,
      "memory(GiB)": 75.26,
      "reward": 1.8536732196807861,
      "reward_std": 0.43080076575279236,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3778642416000366,
      "rewards/ReflectionBalance/mean": 0.7521106898784637,
      "rewards/ReflectionBalance/std": 0.22783784568309784,
      "step": 172,
      "train_speed(iter/s)": 0.007254
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 500.953125,
      "completions/min_length": 259.5,
      "epoch": 0.9326145552560647,
      "grad_norm": 10.038576037729735,
      "kl": 0.112548828125,
      "learning_rate": 9.826469325596826e-07,
      "loss": 0.08076193183660507,
      "memory(GiB)": 75.26,
      "reward": 1.8973006010055542,
      "reward_std": 0.38879016041755676,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21138381212949753,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3978758305311203,
      "rewards/ReflectionBalance/mean": 0.7488631308078766,
      "rewards/ReflectionBalance/std": 0.2616365849971771,
      "step": 173,
      "train_speed(iter/s)": 0.007253
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 515.453125,
      "completions/min_length": 283.0,
      "epoch": 0.9380053908355795,
      "grad_norm": 2.2378634277567815,
      "kl": 0.079345703125,
      "learning_rate": 9.824221704532252e-07,
      "loss": 0.06828932464122772,
      "memory(GiB)": 75.26,
      "reward": 1.894848644733429,
      "reward_std": 0.46455904841423035,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2255912721157074,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4020542651414871,
      "rewards/ReflectionBalance/mean": 0.734692394733429,
      "rewards/ReflectionBalance/std": 0.29635605216026306,
      "step": 174,
      "train_speed(iter/s)": 0.007254
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 988.5,
      "completions/mean_length": 505.3125,
      "completions/min_length": 280.0,
      "epoch": 0.9433962264150944,
      "grad_norm": 2.4673550212727644,
      "kl": 0.093505859375,
      "learning_rate": 9.821959881444781e-07,
      "loss": 0.0357230044901371,
      "memory(GiB)": 75.26,
      "reward": 1.889153242111206,
      "reward_std": 0.4363860487937927,
      "rewards/Format/mean": 0.96875,
      "rewards/Format/std": 0.17319392412900925,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4091253876686096,
      "rewards/ReflectionBalance/mean": 0.7094658017158508,
      "rewards/ReflectionBalance/std": 0.317457839846611,
      "step": 175,
      "train_speed(iter/s)": 0.007254
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 528.640625,
      "completions/min_length": 265.5,
      "epoch": 0.9487870619946092,
      "grad_norm": 11.038613244608241,
      "kl": 0.111572265625,
      "learning_rate": 9.81968386299299e-07,
      "loss": 0.07201281189918518,
      "memory(GiB)": 75.26,
      "reward": 1.8332430124282837,
      "reward_std": 0.527408093214035,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24356544017791748,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4053537845611572,
      "rewards/ReflectionBalance/mean": 0.6926180124282837,
      "rewards/ReflectionBalance/std": 0.3445834368467331,
      "step": 176,
      "train_speed(iter/s)": 0.007252
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 971.0,
      "completions/mean_length": 523.09765625,
      "completions/min_length": 323.0,
      "epoch": 0.954177897574124,
      "grad_norm": 2.248083588500742,
      "kl": 0.091064453125,
      "learning_rate": 9.817393655877253e-07,
      "loss": 0.04471620172262192,
      "memory(GiB)": 75.26,
      "reward": 1.9328864812850952,
      "reward_std": 0.42396748065948486,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.18375974521040916,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.42950694262981415,
      "rewards/ReflectionBalance/mean": 0.7258553206920624,
      "rewards/ReflectionBalance/std": 0.3116491585969925,
      "step": 177,
      "train_speed(iter/s)": 0.007248
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 996.5,
      "completions/mean_length": 517.58203125,
      "completions/min_length": 258.0,
      "epoch": 0.9595687331536388,
      "grad_norm": 2.3534774286555655,
      "kl": 0.09619140625,
      "learning_rate": 9.815089266839704e-07,
      "loss": 0.04761804640293121,
      "memory(GiB)": 75.26,
      "reward": 1.7119881510734558,
      "reward_std": 0.452044814825058,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.2767152339220047,
      "rewards/ReflectionBalance/mean": 0.6612068712711334,
      "rewards/ReflectionBalance/std": 0.3563062399625778,
      "step": 178,
      "train_speed(iter/s)": 0.007247
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 513.4765625,
      "completions/min_length": 264.5,
      "epoch": 0.9649595687331537,
      "grad_norm": 2.4244572089193888,
      "kl": 0.09814453125,
      "learning_rate": 9.812770702664235e-07,
      "loss": 0.06687559187412262,
      "memory(GiB)": 75.26,
      "reward": 1.836255669593811,
      "reward_std": 0.44230224192142487,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23196382820606232,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3842122554779053,
      "rewards/ReflectionBalance/mean": 0.7151619493961334,
      "rewards/ReflectionBalance/std": 0.31923795491456985,
      "step": 179,
      "train_speed(iter/s)": 0.007248
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 513.828125,
      "completions/min_length": 240.0,
      "epoch": 0.9703504043126685,
      "grad_norm": 2.462317594663071,
      "kl": 0.105712890625,
      "learning_rate": 9.810437970176466e-07,
      "loss": 0.07568500936031342,
      "memory(GiB)": 75.26,
      "reward": 1.7697303295135498,
      "reward_std": 0.4810917228460312,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.32267695665359497,
      "rewards/ReflectionBalance/mean": 0.7033240497112274,
      "rewards/ReflectionBalance/std": 0.3017711639404297,
      "step": 180,
      "train_speed(iter/s)": 0.007247
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 532.3671875,
      "completions/min_length": 277.0,
      "epoch": 0.9757412398921833,
      "grad_norm": 2.551384154592592,
      "kl": 0.11328125,
      "learning_rate": 9.808091076243727e-07,
      "loss": 0.10799656063318253,
      "memory(GiB)": 75.26,
      "reward": 1.7477343082427979,
      "reward_std": 0.5180933922529221,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3752707988023758,
      "rewards/ReflectionBalance/mean": 0.669609397649765,
      "rewards/ReflectionBalance/std": 0.3693419396877289,
      "step": 181,
      "train_speed(iter/s)": 0.007248
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 502.9140625,
      "completions/min_length": 251.5,
      "epoch": 0.9811320754716981,
      "grad_norm": 2.443558874358976,
      "kl": 0.09765625,
      "learning_rate": 9.805730027775035e-07,
      "loss": 0.045511502772569656,
      "memory(GiB)": 75.26,
      "reward": 1.8968433141708374,
      "reward_std": 0.420606330037117,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21138381212949753,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.39205317199230194,
      "rewards/ReflectionBalance/mean": 0.7444996237754822,
      "rewards/ReflectionBalance/std": 0.23709890246391296,
      "step": 182,
      "train_speed(iter/s)": 0.007249
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 502.7578125,
      "completions/min_length": 253.5,
      "epoch": 0.9865229110512129,
      "grad_norm": 2.4857241381342075,
      "kl": 0.122314453125,
      "learning_rate": 9.80335483172108e-07,
      "loss": 0.05613870173692703,
      "memory(GiB)": 75.26,
      "reward": 1.9255340099334717,
      "reward_std": 0.47790688276290894,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.19744773209095,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4416199326515198,
      "rewards/ReflectionBalance/mean": 0.6911590397357941,
      "rewards/ReflectionBalance/std": 0.31917044520378113,
      "step": 183,
      "train_speed(iter/s)": 0.007245
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 506.65234375,
      "completions/min_length": 268.0,
      "epoch": 0.9919137466307277,
      "grad_norm": 2.5364421267889545,
      "kl": 0.1083984375,
      "learning_rate": 9.8009654950742e-07,
      "loss": 0.09427884221076965,
      "memory(GiB)": 75.26,
      "reward": 1.7851139307022095,
      "reward_std": 0.46098044514656067,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22826264798641205,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3422359675168991,
      "rewards/ReflectionBalance/mean": 0.6991764008998871,
      "rewards/ReflectionBalance/std": 0.3454058766365051,
      "step": 184,
      "train_speed(iter/s)": 0.007242
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 513.5859375,
      "completions/min_length": 286.0,
      "epoch": 0.9973045822102425,
      "grad_norm": 2.5230124209380937,
      "kl": 0.11865234375,
      "learning_rate": 9.79856202486836e-07,
      "loss": 0.11313892155885696,
      "memory(GiB)": 75.26,
      "reward": 1.8334274291992188,
      "reward_std": 0.4821489453315735,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3613582104444504,
      "rewards/ReflectionBalance/mean": 0.720146119594574,
      "rewards/ReflectionBalance/std": 0.3034694194793701,
      "step": 185,
      "train_speed(iter/s)": 0.007242
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 503.828125,
      "completions/min_length": 283.5,
      "epoch": 1.005390835579515,
      "grad_norm": 2.6087934337405745,
      "kl": 0.132568359375,
      "learning_rate": 9.796144428179138e-07,
      "loss": 0.08995895087718964,
      "memory(GiB)": 75.26,
      "reward": 1.863552212715149,
      "reward_std": 0.5117660760879517,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.253920242190361,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4087653011083603,
      "rewards/ReflectionBalance/mean": 0.6994897425174713,
      "rewards/ReflectionBalance/std": 0.3316012918949127,
      "step": 186,
      "train_speed(iter/s)": 0.007239
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 544.203125,
      "completions/min_length": 291.5,
      "epoch": 1.0107816711590296,
      "grad_norm": 2.3889919037203167,
      "kl": 0.14111328125,
      "learning_rate": 9.793712712123693e-07,
      "loss": 0.09745481610298157,
      "memory(GiB)": 75.26,
      "reward": 1.9139489531517029,
      "reward_std": 0.45713528990745544,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21138381212949753,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.42359209060668945,
      "rewards/ReflectionBalance/mean": 0.7147301435470581,
      "rewards/ReflectionBalance/std": 0.34098130464553833,
      "step": 187,
      "train_speed(iter/s)": 0.007237
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 528.77734375,
      "completions/min_length": 255.5,
      "epoch": 1.0161725067385445,
      "grad_norm": 2.6503298679893255,
      "kl": 0.1630859375,
      "learning_rate": 9.791266883860752e-07,
      "loss": 0.10180152207612991,
      "memory(GiB)": 75.26,
      "reward": 1.8453474044799805,
      "reward_std": 0.45987196266651154,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.38125884532928467,
      "rewards/ReflectionBalance/mean": 0.7555037140846252,
      "rewards/ReflectionBalance/std": 0.2639937102794647,
      "step": 188,
      "train_speed(iter/s)": 0.007237
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 496.41796875,
      "completions/min_length": 253.5,
      "epoch": 1.0215633423180592,
      "grad_norm": 2.670096978059776,
      "kl": 0.1376953125,
      "learning_rate": 9.788806950590588e-07,
      "loss": 0.06386512517929077,
      "memory(GiB)": 75.26,
      "reward": 1.79653799533844,
      "reward_std": 0.4433436393737793,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3493855148553848,
      "rewards/ReflectionBalance/mean": 0.7066943347454071,
      "rewards/ReflectionBalance/std": 0.3211069107055664,
      "step": 189,
      "train_speed(iter/s)": 0.007235
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 503.46875,
      "completions/min_length": 281.0,
      "epoch": 1.0269541778975741,
      "grad_norm": 2.577233779468943,
      "kl": 0.152587890625,
      "learning_rate": 9.786332919554993e-07,
      "loss": 0.1181679517030716,
      "memory(GiB)": 75.26,
      "reward": 1.8749934434890747,
      "reward_std": 0.47867944836616516,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24678732454776764,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.40950682759284973,
      "rewards/ReflectionBalance/mean": 0.7148372530937195,
      "rewards/ReflectionBalance/std": 0.32841774821281433,
      "step": 190,
      "train_speed(iter/s)": 0.007235
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 527.5703125,
      "completions/min_length": 291.5,
      "epoch": 1.0323450134770888,
      "grad_norm": 2.559262513299179,
      "kl": 0.1845703125,
      "learning_rate": 9.78384479803727e-07,
      "loss": 0.11288633942604065,
      "memory(GiB)": 75.26,
      "reward": 1.7590236067771912,
      "reward_std": 0.494504913687706,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2897924929857254,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3716365247964859,
      "rewards/ReflectionBalance/mean": 0.6887110769748688,
      "rewards/ReflectionBalance/std": 0.3172583431005478,
      "step": 191,
      "train_speed(iter/s)": 0.007236
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 503.79296875,
      "completions/min_length": 270.0,
      "epoch": 1.0377358490566038,
      "grad_norm": 2.6149317664598217,
      "kl": 0.16015625,
      "learning_rate": 9.781342593362199e-07,
      "loss": 0.10778997838497162,
      "memory(GiB)": 75.26,
      "reward": 1.8016740083694458,
      "reward_std": 0.502606600522995,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35259076952934265,
      "rewards/ReflectionBalance/mean": 0.7157364785671234,
      "rewards/ReflectionBalance/std": 0.3465382307767868,
      "step": 192,
      "train_speed(iter/s)": 0.007237
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 539.69921875,
      "completions/min_length": 265.5,
      "epoch": 1.0431266846361187,
      "grad_norm": 2.602269455508196,
      "kl": 0.193359375,
      "learning_rate": 9.778826312896014e-07,
      "loss": 0.11238988488912582,
      "memory(GiB)": 75.26,
      "reward": 1.7929720282554626,
      "reward_std": 0.55313640832901,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31518664956092834,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4094643145799637,
      "rewards/ReflectionBalance/mean": 0.6953157484531403,
      "rewards/ReflectionBalance/std": 0.3608507961034775,
      "step": 193,
      "train_speed(iter/s)": 0.007236
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 521.5546875,
      "completions/min_length": 257.0,
      "epoch": 1.0485175202156334,
      "grad_norm": 3.7039832433561615,
      "kl": 0.2041015625,
      "learning_rate": 9.776295964046392e-07,
      "loss": 0.12562313675880432,
      "memory(GiB)": 75.26,
      "reward": 1.648020327091217,
      "reward_std": 0.6278850734233856,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.330063596367836,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.3077666610479355,
      "rewards/ReflectionBalance/mean": 0.671457827091217,
      "rewards/ReflectionBalance/std": 0.39231257140636444,
      "step": 194,
      "train_speed(iter/s)": 0.007235
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 486.94140625,
      "completions/min_length": 223.5,
      "epoch": 1.0539083557951483,
      "grad_norm": 2.666298694975938,
      "kl": 0.169921875,
      "learning_rate": 9.77375155426243e-07,
      "loss": 0.11147800087928772,
      "memory(GiB)": 75.26,
      "reward": 1.9007796049118042,
      "reward_std": 0.5001051723957062,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.4313432276248932,
      "rewards/ReflectionBalance/mean": 0.7015608847141266,
      "rewards/ReflectionBalance/std": 0.32842494547367096,
      "step": 195,
      "train_speed(iter/s)": 0.007235
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 526.87109375,
      "completions/min_length": 255.0,
      "epoch": 1.059299191374663,
      "grad_norm": 3.229548914911873,
      "kl": 0.23486328125,
      "learning_rate": 9.771193091034608e-07,
      "loss": 0.13719408214092255,
      "memory(GiB)": 75.26,
      "reward": 1.7400349378585815,
      "reward_std": 0.5498025119304657,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.29329850524663925,
      "rewards/ReflectionBalance/mean": 0.7087849974632263,
      "rewards/ReflectionBalance/std": 0.33162256330251694,
      "step": 196,
      "train_speed(iter/s)": 0.007234
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 548.65625,
      "completions/min_length": 253.5,
      "epoch": 1.064690026954178,
      "grad_norm": 3.1574201056820477,
      "kl": 0.30078125,
      "learning_rate": 9.768620581894789e-07,
      "loss": 0.1608009934425354,
      "memory(GiB)": 75.26,
      "reward": 1.5840535759925842,
      "reward_std": 0.6378822922706604,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.3678421527147293,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.34447427093982697,
      "rewards/ReflectionBalance/mean": 0.6074910163879395,
      "rewards/ReflectionBalance/std": 0.44820185005664825,
      "step": 197,
      "train_speed(iter/s)": 0.007233
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 526.55078125,
      "completions/min_length": 253.5,
      "epoch": 1.0700808625336928,
      "grad_norm": 2.5412758046837154,
      "kl": 0.2509765625,
      "learning_rate": 9.766034034416178e-07,
      "loss": 0.12408459186553955,
      "memory(GiB)": 75.26,
      "reward": 1.7266291975975037,
      "reward_std": 0.48145586252212524,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3208130896091461,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.34826576709747314,
      "rewards/ReflectionBalance/mean": 0.6953792572021484,
      "rewards/ReflectionBalance/std": 0.34042881429195404,
      "step": 198,
      "train_speed(iter/s)": 0.007231
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 589.53515625,
      "completions/min_length": 246.0,
      "epoch": 1.0754716981132075,
      "grad_norm": 3.256683922455533,
      "kl": 0.3740234375,
      "learning_rate": 9.76343345621331e-07,
      "loss": 0.1815955638885498,
      "memory(GiB)": 75.26,
      "reward": 1.5430525541305542,
      "reward_std": 0.6777904331684113,
      "rewards/Format/mean": 0.80078125,
      "rewards/Format/std": 0.395822137594223,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36624228954315186,
      "rewards/ReflectionBalance/mean": 0.5782088339328766,
      "rewards/ReflectionBalance/std": 0.47777119278907776,
      "step": 199,
      "train_speed(iter/s)": 0.00723
    },
    {
      "epoch": 1.0808625336927224,
      "grad_norm": 13.103120537767056,
      "learning_rate": 9.760818854942027e-07,
      "loss": 0.18363191187381744,
      "memory(GiB)": 75.26,
      "step": 200,
      "train_speed(iter/s)": 0.007232
    },
    {
      "epoch": 1.0808625336927224,
      "eval_clip_ratio": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 1029.25,
      "eval_completions/mean_length": 591.64453125,
      "eval_completions/min_length": 282.0,
      "eval_kl": 0.37890625,
      "eval_loss": 0.16774871945381165,
      "eval_reward": 1.5091862082481384,
      "eval_reward_std": 0.6190158873796463,
      "eval_rewards/Format/mean": 0.802734375,
      "eval_rewards/Format/std": 0.3937984108924866,
      "eval_rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "eval_rewards/MultiModalAccuracyORM/std": 0.31158997118473053,
      "eval_rewards/ReflectionBalance/mean": 0.5853580683469772,
      "eval_rewards/ReflectionBalance/std": 0.44587429612874985,
      "eval_runtime": 249.4171,
      "eval_samples_per_second": 0.237,
      "eval_steps_per_second": 0.004,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 583.416015625,
      "completions/min_length": 278.25,
      "epoch": 1.0862533692722371,
      "grad_norm": 2.8866767703411003,
      "kl": 0.3935546875,
      "learning_rate": 9.758190238299452e-07,
      "loss": 0.15724384784698486,
      "memory(GiB)": 75.26,
      "reward": 1.593218833208084,
      "reward_std": 0.6550646722316742,
      "rewards/Format/mean": 0.818359375,
      "rewards/Format/std": 0.38249245285987854,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.330863893032074,
      "rewards/ReflectionBalance/mean": 0.6264219433069229,
      "rewards/ReflectionBalance/std": 0.4157736971974373,
      "step": 201,
      "train_speed(iter/s)": 0.007156
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1033.0,
      "completions/mean_length": 539.7109375,
      "completions/min_length": 228.5,
      "epoch": 1.091644204851752,
      "grad_norm": 3.510022272496078,
      "kl": 0.3671875,
      "learning_rate": 9.755547614023967e-07,
      "loss": 0.1978098452091217,
      "memory(GiB)": 75.26,
      "reward": 1.5747796297073364,
      "reward_std": 0.7395844161510468,
      "rewards/Format/mean": 0.78515625,
      "rewards/Format/std": 0.41230008006095886,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3820948451757431,
      "rewards/ReflectionBalance/mean": 0.6138421297073364,
      "rewards/ReflectionBalance/std": 0.4375159293413162,
      "step": 202,
      "train_speed(iter/s)": 0.007155
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.5,
      "completions/mean_length": 603.765625,
      "completions/min_length": 228.0,
      "epoch": 1.0970350404312668,
      "grad_norm": 2.997749467161737,
      "kl": 0.4248046875,
      "learning_rate": 9.752890989895192e-07,
      "loss": 0.20261260867118835,
      "memory(GiB)": 75.26,
      "reward": 1.4277982711791992,
      "reward_std": 0.7095491588115692,
      "rewards/Format/mean": 0.70703125,
      "rewards/Format/std": 0.44167305529117584,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.31891990453004837,
      "rewards/ReflectionBalance/mean": 0.5879545211791992,
      "rewards/ReflectionBalance/std": 0.43435585498809814,
      "step": 203,
      "train_speed(iter/s)": 0.007157
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 621.81640625,
      "completions/min_length": 288.0,
      "epoch": 1.1024258760107817,
      "grad_norm": 3.201894217176067,
      "kl": 0.4853515625,
      "learning_rate": 9.75022037373396e-07,
      "loss": 0.2241160124540329,
      "memory(GiB)": 75.26,
      "reward": 1.4859263896942139,
      "reward_std": 0.7323862910270691,
      "rewards/Format/mean": 0.7265625,
      "rewards/Format/std": 0.447474867105484,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.4131145179271698,
      "rewards/ReflectionBalance/mean": 0.5367076992988586,
      "rewards/ReflectionBalance/std": 0.48203320801258087,
      "step": 204,
      "train_speed(iter/s)": 0.007156
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 595.3515625,
      "completions/min_length": 273.5,
      "epoch": 1.1078167115902966,
      "grad_norm": 3.688896675505514,
      "kl": 0.53125,
      "learning_rate": 9.747535773402299e-07,
      "loss": 0.20696067810058594,
      "memory(GiB)": 75.26,
      "reward": 1.4636532664299011,
      "reward_std": 0.7407130002975464,
      "rewards/Format/mean": 0.72265625,
      "rewards/Format/std": 0.4468510299921036,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3619322180747986,
      "rewards/ReflectionBalance/mean": 0.5847469866275787,
      "rewards/ReflectionBalance/std": 0.45219479501247406,
      "step": 205,
      "train_speed(iter/s)": 0.007157
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.0,
      "completions/mean_length": 643.6875,
      "completions/min_length": 270.0,
      "epoch": 1.1132075471698113,
      "grad_norm": 3.4187731529230625,
      "kl": 0.6083984375,
      "learning_rate": 9.744837196803397e-07,
      "loss": 0.23515257239341736,
      "memory(GiB)": 75.26,
      "reward": 1.2525933980941772,
      "reward_std": 0.7038880288600922,
      "rewards/Format/mean": 0.6875,
      "rewards/Format/std": 0.4634045511484146,
      "rewards/MultiModalAccuracyORM/mean": 0.08984375,
      "rewards/MultiModalAccuracyORM/std": 0.2850065380334854,
      "rewards/ReflectionBalance/mean": 0.47524967789649963,
      "rewards/ReflectionBalance/std": 0.49864552915096283,
      "step": 206,
      "train_speed(iter/s)": 0.007158
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 661.7421875,
      "completions/min_length": 303.0,
      "epoch": 1.1185983827493262,
      "grad_norm": 4.181812838523217,
      "kl": 0.625,
      "learning_rate": 9.742124651881594e-07,
      "loss": 0.24888494610786438,
      "memory(GiB)": 75.26,
      "reward": 1.2660480737686157,
      "reward_std": 0.7660779654979706,
      "rewards/Format/mean": 0.63671875,
      "rewards/Format/std": 0.4828174412250519,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3473474234342575,
      "rewards/ReflectionBalance/mean": 0.4887044131755829,
      "rewards/ReflectionBalance/std": 0.5153256058692932,
      "step": 207,
      "train_speed(iter/s)": 0.007158
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 611.5,
      "completions/min_length": 250.5,
      "epoch": 1.123989218328841,
      "grad_norm": 34.31263344597327,
      "kl": 1.40234375,
      "learning_rate": 9.739398146622353e-07,
      "loss": 0.28949856758117676,
      "memory(GiB)": 75.26,
      "reward": 1.4224417209625244,
      "reward_std": 0.7999645471572876,
      "rewards/Format/mean": 0.703125,
      "rewards/Format/std": 0.45835477113723755,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3704231530427933,
      "rewards/ReflectionBalance/mean": 0.5552542209625244,
      "rewards/ReflectionBalance/std": 0.46990200877189636,
      "step": 208,
      "train_speed(iter/s)": 0.007155
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 597.4765625,
      "completions/min_length": 260.0,
      "epoch": 1.1293800539083558,
      "grad_norm": 9.9475918703191,
      "kl": 0.658203125,
      "learning_rate": 9.736657689052224e-07,
      "loss": 0.2656392455101013,
      "memory(GiB)": 75.26,
      "reward": 1.4611408710479736,
      "reward_std": 0.7794257998466492,
      "rewards/Format/mean": 0.703125,
      "rewards/Format/std": 0.45867621898651123,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3978758305311203,
      "rewards/ReflectionBalance/mean": 0.5627034604549408,
      "rewards/ReflectionBalance/std": 0.4439594894647598,
      "step": 209,
      "train_speed(iter/s)": 0.007155
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 678.16015625,
      "completions/min_length": 218.0,
      "epoch": 1.1347708894878705,
      "grad_norm": 5.399577309947014,
      "kl": 0.84765625,
      "learning_rate": 9.733903287238845e-07,
      "loss": 0.26466429233551025,
      "memory(GiB)": 75.26,
      "reward": 1.1599414348602295,
      "reward_std": 0.8583858013153076,
      "rewards/Format/mean": 0.5625,
      "rewards/Format/std": 0.49796485900878906,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3723055571317673,
      "rewards/ReflectionBalance/mean": 0.4294726699590683,
      "rewards/ReflectionBalance/std": 0.5250942260026932,
      "step": 210,
      "train_speed(iter/s)": 0.007155
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 707.7890625,
      "completions/min_length": 228.0,
      "epoch": 1.1401617250673854,
      "grad_norm": 6.581700343018197,
      "kl": 0.9453125,
      "learning_rate": 9.731134949290891e-07,
      "loss": 0.3131682276725769,
      "memory(GiB)": 75.26,
      "reward": 1.023771196603775,
      "reward_std": 0.8570093810558319,
      "rewards/Format/mean": 0.53125,
      "rewards/Format/std": 0.5007366687059402,
      "rewards/MultiModalAccuracyORM/mean": 0.07421875,
      "rewards/MultiModalAccuracyORM/std": 0.253920242190361,
      "rewards/ReflectionBalance/mean": 0.41830240190029144,
      "rewards/ReflectionBalance/std": 0.5127230733633041,
      "step": 211,
      "train_speed(iter/s)": 0.007157
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 717.7421875,
      "completions/min_length": 261.5,
      "epoch": 1.1455525606469004,
      "grad_norm": 27.522986625968105,
      "kl": 1.88671875,
      "learning_rate": 9.728352683358073e-07,
      "loss": 0.34156718850135803,
      "memory(GiB)": 75.26,
      "reward": 0.9609863758087158,
      "reward_std": 0.8982530236244202,
      "rewards/Format/mean": 0.48828125,
      "rewards/Format/std": 0.5014433264732361,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.2961650490760803,
      "rewards/ReflectionBalance/mean": 0.37504883110523224,
      "rewards/ReflectionBalance/std": 0.5367146283388138,
      "step": 212,
      "train_speed(iter/s)": 0.007159
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 736.04296875,
      "completions/min_length": 293.0,
      "epoch": 1.150943396226415,
      "grad_norm": 132.50800193484815,
      "kl": 3.9609375,
      "learning_rate": 9.725556497631106e-07,
      "loss": 0.41803795099258423,
      "memory(GiB)": 75.26,
      "reward": 1.006316751241684,
      "reward_std": 0.8081718385219574,
      "rewards/Format/mean": 0.453125,
      "rewards/Format/std": 0.49819907546043396,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3526121824979782,
      "rewards/ReflectionBalance/mean": 0.40866048634052277,
      "rewards/ReflectionBalance/std": 0.4697639048099518,
      "step": 213,
      "train_speed(iter/s)": 0.007159
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 722.6015625,
      "completions/min_length": 206.5,
      "epoch": 1.15633423180593,
      "grad_norm": 5.077275788428287,
      "kl": 1.28125,
      "learning_rate": 9.722746400341673e-07,
      "loss": 0.21924901008605957,
      "memory(GiB)": 75.26,
      "reward": 0.9236279129981995,
      "reward_std": 0.6602786183357239,
      "rewards/Format/mean": 0.44140625,
      "rewards/Format/std": 0.4939642697572708,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.3142324984073639,
      "rewards/ReflectionBalance/mean": 0.36503419280052185,
      "rewards/ReflectionBalance/std": 0.4792015701532364,
      "step": 214,
      "train_speed(iter/s)": 0.007158
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 800.90234375,
      "completions/min_length": 295.0,
      "epoch": 1.1617250673854447,
      "grad_norm": 17.9787734134307,
      "kl": 2.30859375,
      "learning_rate": 9.71992239976242e-07,
      "loss": 0.30830347537994385,
      "memory(GiB)": 75.26,
      "reward": 0.7396883368492126,
      "reward_std": 0.7065556049346924,
      "rewards/Format/mean": 0.34765625,
      "rewards/Format/std": 0.4740851819515228,
      "rewards/MultiModalAccuracyORM/mean": 0.06640625,
      "rewards/MultiModalAccuracyORM/std": 0.2389676719903946,
      "rewards/ReflectionBalance/mean": 0.32562581449747086,
      "rewards/ReflectionBalance/std": 0.4650345891714096,
      "step": 215,
      "train_speed(iter/s)": 0.007158
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 741.703125,
      "completions/min_length": 250.0,
      "epoch": 1.1671159029649596,
      "grad_norm": 6.749936010426161,
      "kl": 1.70703125,
      "learning_rate": 9.717084504206921e-07,
      "loss": 0.2794554531574249,
      "memory(GiB)": 75.26,
      "reward": 0.9132405817508698,
      "reward_std": 0.743252158164978,
      "rewards/Format/mean": 0.421875,
      "rewards/Format/std": 0.4947807043790817,
      "rewards/MultiModalAccuracyORM/mean": 0.08984375,
      "rewards/MultiModalAccuracyORM/std": 0.2869994640350342,
      "rewards/ReflectionBalance/mean": 0.40152183175086975,
      "rewards/ReflectionBalance/std": 0.48775866627693176,
      "step": 216,
      "train_speed(iter/s)": 0.007156
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 749.65234375,
      "completions/min_length": 288.0,
      "epoch": 1.1725067385444743,
      "grad_norm": 6.311224790390792,
      "kl": 2.4609375,
      "learning_rate": 9.714232722029653e-07,
      "loss": 0.32953792810440063,
      "memory(GiB)": 75.26,
      "reward": 0.8589046597480774,
      "reward_std": 0.9073879718780518,
      "rewards/Format/mean": 0.38671875,
      "rewards/Format/std": 0.48876263201236725,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.34734736382961273,
      "rewards/ReflectionBalance/mean": 0.315935879945755,
      "rewards/ReflectionBalance/std": 0.5487177670001984,
      "step": 217,
      "train_speed(iter/s)": 0.007155
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 746.31640625,
      "completions/min_length": 252.0,
      "epoch": 1.1778975741239892,
      "grad_norm": 8.248909928529253,
      "kl": 3.140625,
      "learning_rate": 9.711367061625974e-07,
      "loss": 0.35587725043296814,
      "memory(GiB)": 75.26,
      "reward": 0.7843074202537537,
      "reward_std": 0.6982082724571228,
      "rewards/Format/mean": 0.35546875,
      "rewards/Format/std": 0.47911877930164337,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.3077666610479355,
      "rewards/ReflectionBalance/mean": 0.32336993515491486,
      "rewards/ReflectionBalance/std": 0.481586217880249,
      "step": 218,
      "train_speed(iter/s)": 0.007154
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 792.90234375,
      "completions/min_length": 281.5,
      "epoch": 1.1832884097035041,
      "grad_norm": 10.85468630412591,
      "kl": 3.3828125,
      "learning_rate": 9.708487531432106e-07,
      "loss": 0.329852819442749,
      "memory(GiB)": 75.26,
      "reward": 0.6606429219245911,
      "reward_std": 0.8185202181339264,
      "rewards/Format/mean": 0.2578125,
      "rewards/Format/std": 0.4383239597082138,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.30740001052618027,
      "rewards/ReflectionBalance/mean": 0.28954915702342987,
      "rewards/ReflectionBalance/std": 0.48207852244377136,
      "step": 219,
      "train_speed(iter/s)": 0.007153
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 746.98828125,
      "completions/min_length": 283.0,
      "epoch": 1.1886792452830188,
      "grad_norm": 15.057819808890704,
      "kl": 4.03125,
      "learning_rate": 9.70559413992509e-07,
      "loss": 0.3424689471721649,
      "memory(GiB)": 75.26,
      "reward": 0.7640804052352905,
      "reward_std": 0.7905463576316833,
      "rewards/Format/mean": 0.3359375,
      "rewards/Format/std": 0.46682190895080566,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.32267695665359497,
      "rewards/ReflectionBalance/mean": 0.3109554052352905,
      "rewards/ReflectionBalance/std": 0.5193818211555481,
      "step": 220,
      "train_speed(iter/s)": 0.007153
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 785.89453125,
      "completions/min_length": 275.0,
      "epoch": 1.1940700808625337,
      "grad_norm": 13.312241267240106,
      "kl": 3.9140625,
      "learning_rate": 9.702686895622781e-07,
      "loss": 0.3668285012245178,
      "memory(GiB)": 75.26,
      "reward": 0.7398811876773834,
      "reward_std": 0.6932709217071533,
      "rewards/Format/mean": 0.30078125,
      "rewards/Format/std": 0.46037986874580383,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.32746291160583496,
      "rewards/ReflectionBalance/mean": 0.3180061876773834,
      "rewards/ReflectionBalance/std": 0.4611350744962692,
      "step": 221,
      "train_speed(iter/s)": 0.007152
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 761.46875,
      "completions/min_length": 244.0,
      "epoch": 1.1994609164420484,
      "grad_norm": 10.249213694360161,
      "kl": 4.03125,
      "learning_rate": 9.699765807083815e-07,
      "loss": 0.40676450729370117,
      "memory(GiB)": 75.26,
      "reward": 0.709433913230896,
      "reward_std": 0.7399958372116089,
      "rewards/Format/mean": 0.3203125,
      "rewards/Format/std": 0.46835459768772125,
      "rewards/MultiModalAccuracyORM/mean": 0.0625,
      "rewards/MultiModalAccuracyORM/std": 0.24247010797262192,
      "rewards/ReflectionBalance/mean": 0.326621413230896,
      "rewards/ReflectionBalance/std": 0.42577917873859406,
      "step": 222,
      "train_speed(iter/s)": 0.007152
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 688.78515625,
      "completions/min_length": 245.5,
      "epoch": 1.2048517520215634,
      "grad_norm": 7.021875199611011,
      "kl": 3.0703125,
      "learning_rate": 9.696830882907579e-07,
      "loss": 0.4037257432937622,
      "memory(GiB)": 75.26,
      "reward": 1.0812548995018005,
      "reward_std": 0.8630607426166534,
      "rewards/Format/mean": 0.484375,
      "rewards/Format/std": 0.5001830011606216,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3643587827682495,
      "rewards/ReflectionBalance/mean": 0.44062989950180054,
      "rewards/ReflectionBalance/std": 0.4593242257833481,
      "step": 223,
      "train_speed(iter/s)": 0.007152
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 735.53125,
      "completions/min_length": 292.5,
      "epoch": 1.210242587601078,
      "grad_norm": 6.524655377706854,
      "kl": 3.109375,
      "learning_rate": 9.6938821317342e-07,
      "loss": 0.33837857842445374,
      "memory(GiB)": 75.26,
      "reward": 1.0418505668640137,
      "reward_std": 0.760410487651825,
      "rewards/Format/mean": 0.44921875,
      "rewards/Format/std": 0.49673235416412354,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39171716570854187,
      "rewards/ReflectionBalance/mean": 0.40513184666633606,
      "rewards/ReflectionBalance/std": 0.43368014693260193,
      "step": 224,
      "train_speed(iter/s)": 0.007152
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 732.6640625,
      "completions/min_length": 264.5,
      "epoch": 1.215633423180593,
      "grad_norm": 5.8593507764932165,
      "kl": 3.046875,
      "learning_rate": 9.690919562244498e-07,
      "loss": 0.35993313789367676,
      "memory(GiB)": 75.26,
      "reward": 0.9220311939716339,
      "reward_std": 0.7802919447422028,
      "rewards/Format/mean": 0.43359375,
      "rewards/Format/std": 0.4971243292093277,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.2961650490760803,
      "rewards/ReflectionBalance/mean": 0.3907812386751175,
      "rewards/ReflectionBalance/std": 0.47613662481307983,
      "step": 225,
      "train_speed(iter/s)": 0.00715
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 739.2265625,
      "completions/min_length": 207.5,
      "epoch": 1.221024258760108,
      "grad_norm": 5.090851245553306,
      "kl": 2.59375,
      "learning_rate": 9.687943183159982e-07,
      "loss": 0.30597203969955444,
      "memory(GiB)": 75.26,
      "reward": 1.0559781789779663,
      "reward_std": 0.7277742326259613,
      "rewards/Format/mean": 0.44921875,
      "rewards/Format/std": 0.49897994101047516,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.32164282351732254,
      "rewards/ReflectionBalance/mean": 0.4700406938791275,
      "rewards/ReflectionBalance/std": 0.42591311037540436,
      "step": 226,
      "train_speed(iter/s)": 0.00715
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 697.1484375,
      "completions/min_length": 260.0,
      "epoch": 1.2264150943396226,
      "grad_norm": 10.16901578256346,
      "kl": 2.01953125,
      "learning_rate": 9.684953003242813e-07,
      "loss": 0.35436561703681946,
      "memory(GiB)": 75.26,
      "reward": 1.213134765625,
      "reward_std": 0.7996749877929688,
      "rewards/Format/mean": 0.55078125,
      "rewards/Format/std": 0.4993534982204437,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3460448384284973,
      "rewards/ReflectionBalance/mean": 0.521728515625,
      "rewards/ReflectionBalance/std": 0.42496471107006073,
      "step": 227,
      "train_speed(iter/s)": 0.00715
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 731.875,
      "completions/min_length": 249.0,
      "epoch": 1.2318059299191375,
      "grad_norm": 5.02283021038662,
      "kl": 2.390625,
      "learning_rate": 9.68194903129578e-07,
      "loss": 0.3089757561683655,
      "memory(GiB)": 75.26,
      "reward": 1.0517122447490692,
      "reward_std": 0.7089298665523529,
      "rewards/Format/mean": 0.4375,
      "rewards/Format/std": 0.4977765679359436,
      "rewards/MultiModalAccuracyORM/mean": 0.09375,
      "rewards/MultiModalAccuracyORM/std": 0.29262590408325195,
      "rewards/ReflectionBalance/mean": 0.5204622447490692,
      "rewards/ReflectionBalance/std": 0.395825520157814,
      "step": 228,
      "train_speed(iter/s)": 0.00715
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 686.7421875,
      "completions/min_length": 290.5,
      "epoch": 1.2371967654986522,
      "grad_norm": 10.230604900329801,
      "kl": 2.125,
      "learning_rate": 9.67893127616227e-07,
      "loss": 0.36557120084762573,
      "memory(GiB)": 75.26,
      "reward": 1.2089624404907227,
      "reward_std": 0.8537536263465881,
      "rewards/Format/mean": 0.53125,
      "rewards/Format/std": 0.4999961107969284,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3526121824979782,
      "rewards/ReflectionBalance/mean": 0.5331811308860779,
      "rewards/ReflectionBalance/std": 0.4114348888397217,
      "step": 229,
      "train_speed(iter/s)": 0.007151
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 685.03125,
      "completions/min_length": 257.0,
      "epoch": 1.2425876010781671,
      "grad_norm": 23.552215259184685,
      "kl": 3.15625,
      "learning_rate": 9.675899746726256e-07,
      "loss": 0.4239997863769531,
      "memory(GiB)": 75.26,
      "reward": 1.0943546295166016,
      "reward_std": 0.9093301296234131,
      "rewards/Format/mean": 0.4609375,
      "rewards/Format/std": 0.4988819509744644,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.35296520590782166,
      "rewards/ReflectionBalance/mean": 0.48888590931892395,
      "rewards/ReflectionBalance/std": 0.42439456284046173,
      "step": 230,
      "train_speed(iter/s)": 0.007153
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 732.81640625,
      "completions/min_length": 279.0,
      "epoch": 1.2479784366576818,
      "grad_norm": 5.857447372908369,
      "kl": 3.5859375,
      "learning_rate": 9.672854451912252e-07,
      "loss": 0.3857918679714203,
      "memory(GiB)": 75.26,
      "reward": 0.8412149846553802,
      "reward_std": 0.7718622088432312,
      "rewards/Format/mean": 0.36328125,
      "rewards/Format/std": 0.47991423308849335,
      "rewards/MultiModalAccuracyORM/mean": 0.0625,
      "rewards/MultiModalAccuracyORM/std": 0.2379390001296997,
      "rewards/ReflectionBalance/mean": 0.41543374955654144,
      "rewards/ReflectionBalance/std": 0.4241185933351517,
      "step": 231,
      "train_speed(iter/s)": 0.007153
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 638.34765625,
      "completions/min_length": 253.0,
      "epoch": 1.2533692722371967,
      "grad_norm": 6.91299228296385,
      "kl": 3.5859375,
      "learning_rate": 9.6697954006853e-07,
      "loss": 0.3929609954357147,
      "memory(GiB)": 75.26,
      "reward": 1.2274934649467468,
      "reward_std": 0.7447052597999573,
      "rewards/Format/mean": 0.53515625,
      "rewards/Format/std": 0.5007068514823914,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3358502835035324,
      "rewards/ReflectionBalance/mean": 0.5478059947490692,
      "rewards/ReflectionBalance/std": 0.4195135533809662,
      "step": 232,
      "train_speed(iter/s)": 0.007152
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 693.53125,
      "completions/min_length": 255.0,
      "epoch": 1.2587601078167117,
      "grad_norm": 10.801182267349242,
      "kl": 3.90625,
      "learning_rate": 9.666722602050937e-07,
      "loss": 0.3575071096420288,
      "memory(GiB)": 75.26,
      "reward": 0.9142740964889526,
      "reward_std": 0.6781527400016785,
      "rewards/Format/mean": 0.40234375,
      "rewards/Format/std": 0.4763943701982498,
      "rewards/MultiModalAccuracyORM/mean": 0.07421875,
      "rewards/MultiModalAccuracyORM/std": 0.1784650683403015,
      "rewards/ReflectionBalance/mean": 0.43771156668663025,
      "rewards/ReflectionBalance/std": 0.4289555102586746,
      "step": 233,
      "train_speed(iter/s)": 0.007152
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 659.4453125,
      "completions/min_length": 271.0,
      "epoch": 1.2641509433962264,
      "grad_norm": 7.220909195450298,
      "kl": 3.9453125,
      "learning_rate": 9.663636065055176e-07,
      "loss": 0.40874478220939636,
      "memory(GiB)": 75.26,
      "reward": 1.0880175828933716,
      "reward_std": 0.7404800355434418,
      "rewards/Format/mean": 0.4609375,
      "rewards/Format/std": 0.4988819509744644,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.298691526055336,
      "rewards/ReflectionBalance/mean": 0.5255175530910492,
      "rewards/ReflectionBalance/std": 0.40371111035346985,
      "step": 234,
      "train_speed(iter/s)": 0.007154
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 659.0546875,
      "completions/min_length": 285.5,
      "epoch": 1.2695417789757413,
      "grad_norm": 6.128546726714799,
      "kl": 3.625,
      "learning_rate": 9.660535798784469e-07,
      "loss": 0.3982512652873993,
      "memory(GiB)": 75.26,
      "reward": 1.1703084707260132,
      "reward_std": 0.7289623916149139,
      "rewards/Format/mean": 0.51171875,
      "rewards/Format/std": 0.5016887485980988,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3270204961299896,
      "rewards/ReflectionBalance/mean": 0.5374959111213684,
      "rewards/ReflectionBalance/std": 0.40597233176231384,
      "step": 235,
      "train_speed(iter/s)": 0.007153
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 693.38671875,
      "completions/min_length": 250.0,
      "epoch": 1.2749326145552562,
      "grad_norm": 7.705407823615547,
      "kl": 3.6640625,
      "learning_rate": 9.657421812365686e-07,
      "loss": 0.3843494653701782,
      "memory(GiB)": 75.26,
      "reward": 0.9064747989177704,
      "reward_std": 0.70859694480896,
      "rewards/Format/mean": 0.41015625,
      "rewards/Format/std": 0.49377845227718353,
      "rewards/MultiModalAccuracyORM/mean": 0.07421875,
      "rewards/MultiModalAccuracyORM/std": 0.253920242190361,
      "rewards/ReflectionBalance/mean": 0.422099769115448,
      "rewards/ReflectionBalance/std": 0.4191877990961075,
      "step": 236,
      "train_speed(iter/s)": 0.007153
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 679.75,
      "completions/min_length": 221.0,
      "epoch": 1.280323450134771,
      "grad_norm": 5.73534320072691,
      "kl": 3.296875,
      "learning_rate": 9.654294114966095e-07,
      "loss": 0.35336339473724365,
      "memory(GiB)": 75.26,
      "reward": 1.0667903423309326,
      "reward_std": 0.7694992125034332,
      "rewards/Format/mean": 0.4765625,
      "rewards/Format/std": 0.5014128684997559,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3270204961299896,
      "rewards/ReflectionBalance/mean": 0.4691341072320938,
      "rewards/ReflectionBalance/std": 0.43495550751686096,
      "step": 237,
      "train_speed(iter/s)": 0.007153
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 697.94140625,
      "completions/min_length": 246.5,
      "epoch": 1.2857142857142856,
      "grad_norm": 6.218113589890933,
      "kl": 3.2890625,
      "learning_rate": 9.651152715793317e-07,
      "loss": 0.42697519063949585,
      "memory(GiB)": 75.26,
      "reward": 1.004503607749939,
      "reward_std": 0.8130412399768829,
      "rewards/Format/mean": 0.45703125,
      "rewards/Format/std": 0.4993480294942856,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3131764084100723,
      "rewards/ReflectionBalance/mean": 0.4341910779476166,
      "rewards/ReflectionBalance/std": 0.40710607171058655,
      "step": 238,
      "train_speed(iter/s)": 0.007153
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 711.4609375,
      "completions/min_length": 262.5,
      "epoch": 1.2911051212938005,
      "grad_norm": 5.177663901319421,
      "kl": 3.0625,
      "learning_rate": 9.647997624095316e-07,
      "loss": 0.3681216537952423,
      "memory(GiB)": 75.26,
      "reward": 0.9939860105514526,
      "reward_std": 0.7549453675746918,
      "rewards/Format/mean": 0.44921875,
      "rewards/Format/std": 0.4924577921628952,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.2767152339220047,
      "rewards/ReflectionBalance/mean": 0.44320476055145264,
      "rewards/ReflectionBalance/std": 0.4067220985889435,
      "step": 239,
      "train_speed(iter/s)": 0.007153
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 697.765625,
      "completions/min_length": 261.0,
      "epoch": 1.2964959568733154,
      "grad_norm": 5.874366591148422,
      "kl": 3.0234375,
      "learning_rate": 9.644828849160367e-07,
      "loss": 0.39667418599128723,
      "memory(GiB)": 75.26,
      "reward": 1.038902997970581,
      "reward_std": 0.7477164566516876,
      "rewards/Format/mean": 0.47265625,
      "rewards/Format/std": 0.4943803399801254,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.2800559103488922,
      "rewards/ReflectionBalance/mean": 0.46077799797058105,
      "rewards/ReflectionBalance/std": 0.40336379408836365,
      "step": 240,
      "train_speed(iter/s)": 0.007154
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 672.390625,
      "completions/min_length": 279.0,
      "epoch": 1.3018867924528301,
      "grad_norm": 8.966171727680138,
      "kl": 2.625,
      "learning_rate": 9.64164640031702e-07,
      "loss": 0.376760870218277,
      "memory(GiB)": 75.26,
      "reward": 1.1791308522224426,
      "reward_std": 0.7813467979431152,
      "rewards/Format/mean": 0.5234375,
      "rewards/Format/std": 0.49640798568725586,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.33063435554504395,
      "rewards/ReflectionBalance/mean": 0.522880882024765,
      "rewards/ReflectionBalance/std": 0.4048462510108948,
      "step": 241,
      "train_speed(iter/s)": 0.007152
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 663.30859375,
      "completions/min_length": 249.0,
      "epoch": 1.307277628032345,
      "grad_norm": 8.174377578694122,
      "kl": 3.1171875,
      "learning_rate": 9.638450286934088e-07,
      "loss": 0.42467349767684937,
      "memory(GiB)": 75.26,
      "reward": 1.2758528590202332,
      "reward_std": 0.7994622588157654,
      "rewards/Format/mean": 0.5546875,
      "rewards/Format/std": 0.49588608741760254,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4120787978172302,
      "rewards/ReflectionBalance/mean": 0.5063216388225555,
      "rewards/ReflectionBalance/std": 0.4102555960416794,
      "step": 242,
      "train_speed(iter/s)": 0.007152
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 694.71875,
      "completions/min_length": 257.5,
      "epoch": 1.31266846361186,
      "grad_norm": 6.029940677408426,
      "kl": 3.96875,
      "learning_rate": 9.635240518420603e-07,
      "loss": 0.42840030789375305,
      "memory(GiB)": 75.26,
      "reward": 0.9735367894172668,
      "reward_std": 0.7542374730110168,
      "rewards/Format/mean": 0.4609375,
      "rewards/Format/std": 0.5001830011606216,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.29794247448444366,
      "rewards/ReflectionBalance/mean": 0.41494303941726685,
      "rewards/ReflectionBalance/std": 0.4265829771757126,
      "step": 243,
      "train_speed(iter/s)": 0.007151
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 669.55859375,
      "completions/min_length": 214.0,
      "epoch": 1.3180592991913747,
      "grad_norm": 6.913616107904033,
      "kl": 4.3125,
      "learning_rate": 9.632017104225803e-07,
      "loss": 0.4567493796348572,
      "memory(GiB)": 75.26,
      "reward": 1.1395410299301147,
      "reward_std": 0.72019162774086,
      "rewards/Format/mean": 0.515625,
      "rewards/Format/std": 0.4895436465740204,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.344347283244133,
      "rewards/ReflectionBalance/mean": 0.48329100012779236,
      "rewards/ReflectionBalance/std": 0.40779300034046173,
      "step": 244,
      "train_speed(iter/s)": 0.007149
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 694.234375,
      "completions/min_length": 191.0,
      "epoch": 1.3234501347708894,
      "grad_norm": 6.0364130557975635,
      "kl": 4.1484375,
      "learning_rate": 9.628780053839093e-07,
      "loss": 0.43473437428474426,
      "memory(GiB)": 75.26,
      "reward": 1.0335806906223297,
      "reward_std": 0.7375575006008148,
      "rewards/Format/mean": 0.4921875,
      "rewards/Format/std": 0.481633260846138,
      "rewards/MultiModalAccuracyORM/mean": 0.078125,
      "rewards/MultiModalAccuracyORM/std": 0.26276856660842896,
      "rewards/ReflectionBalance/mean": 0.4632682204246521,
      "rewards/ReflectionBalance/std": 0.40529267489910126,
      "step": 245,
      "train_speed(iter/s)": 0.007146
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 680.21484375,
      "completions/min_length": 258.0,
      "epoch": 1.3288409703504043,
      "grad_norm": 6.5792067272321075,
      "kl": 4.2265625,
      "learning_rate": 9.625529376790025e-07,
      "loss": 0.4711546301841736,
      "memory(GiB)": 75.26,
      "reward": 1.134130835533142,
      "reward_std": 0.8200222849845886,
      "rewards/Format/mean": 0.5234375,
      "rewards/Format/std": 0.4983912855386734,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3131764084100723,
      "rewards/ReflectionBalance/mean": 0.4974121153354645,
      "rewards/ReflectionBalance/std": 0.4151216447353363,
      "step": 246,
      "train_speed(iter/s)": 0.007146
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 682.67578125,
      "completions/min_length": 277.5,
      "epoch": 1.3342318059299192,
      "grad_norm": 6.032007728502722,
      "kl": 4.25,
      "learning_rate": 9.622265082648264e-07,
      "loss": 0.4924547076225281,
      "memory(GiB)": 75.26,
      "reward": 1.1846582293510437,
      "reward_std": 0.805219829082489,
      "rewards/Format/mean": 0.52734375,
      "rewards/Format/std": 0.5011980831623077,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3643587827682495,
      "rewards/ReflectionBalance/mean": 0.5010644495487213,
      "rewards/ReflectionBalance/std": 0.40801022946834564,
      "step": 247,
      "train_speed(iter/s)": 0.007147
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 664.90234375,
      "completions/min_length": 232.0,
      "epoch": 1.3396226415094339,
      "grad_norm": 7.055160011133057,
      "kl": 3.5,
      "learning_rate": 9.618987181023566e-07,
      "loss": 0.4692852795124054,
      "memory(GiB)": 75.26,
      "reward": 1.1721760630607605,
      "reward_std": 0.7668159902095795,
      "rewards/Format/mean": 0.58203125,
      "rewards/Format/std": 0.4880688339471817,
      "rewards/MultiModalAccuracyORM/mean": 0.078125,
      "rewards/MultiModalAccuracyORM/std": 0.2657608464360237,
      "rewards/ReflectionBalance/mean": 0.5120198577642441,
      "rewards/ReflectionBalance/std": 0.4056391566991806,
      "step": 248,
      "train_speed(iter/s)": 0.007148
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 690.04296875,
      "completions/min_length": 235.5,
      "epoch": 1.3450134770889488,
      "grad_norm": 5.530818327682444,
      "kl": 3.734375,
      "learning_rate": 9.615695681565738e-07,
      "loss": 0.46923744678497314,
      "memory(GiB)": 75.26,
      "reward": 1.0760156214237213,
      "reward_std": 0.8225083947181702,
      "rewards/Format/mean": 0.53125,
      "rewards/Format/std": 0.4947807043790817,
      "rewards/MultiModalAccuracyORM/mean": 0.0703125,
      "rewards/MultiModalAccuracyORM/std": 0.2524137645959854,
      "rewards/ReflectionBalance/mean": 0.4744531214237213,
      "rewards/ReflectionBalance/std": 0.410795658826828,
      "step": 249,
      "train_speed(iter/s)": 0.007146
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 712.74609375,
      "completions/min_length": 227.5,
      "epoch": 1.3504043126684637,
      "grad_norm": 7.086373264787719,
      "kl": 4.421875,
      "learning_rate": 9.612390593964628e-07,
      "loss": 0.4972318410873413,
      "memory(GiB)": 75.26,
      "reward": 1.0526220798492432,
      "reward_std": 0.7702766358852386,
      "rewards/Format/mean": 0.484375,
      "rewards/Format/std": 0.5007366687059402,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35538628697395325,
      "rewards/ReflectionBalance/mean": 0.41980957984924316,
      "rewards/ReflectionBalance/std": 0.4125642627477646,
      "step": 250,
      "train_speed(iter/s)": 0.007145
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 674.6640625,
      "completions/min_length": 257.0,
      "epoch": 1.3557951482479784,
      "grad_norm": 12.088533239733419,
      "kl": 4.828125,
      "learning_rate": 9.609071927950076e-07,
      "loss": 0.559328556060791,
      "memory(GiB)": 75.26,
      "reward": 1.226474642753601,
      "reward_std": 0.9185117185115814,
      "rewards/Format/mean": 0.5390625,
      "rewards/Format/std": 0.5004304051399231,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41404466331005096,
      "rewards/ReflectionBalance/mean": 0.4686621129512787,
      "rewards/ReflectionBalance/std": 0.41977404057979584,
      "step": 251,
      "train_speed(iter/s)": 0.007144
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 800.53515625,
      "completions/min_length": 294.0,
      "epoch": 1.3611859838274933,
      "grad_norm": 23.308668834688895,
      "kl": 6.0,
      "learning_rate": 9.605739693291905e-07,
      "loss": 0.4945274889469147,
      "memory(GiB)": 75.26,
      "reward": 0.788134753704071,
      "reward_std": 0.6539475321769714,
      "rewards/Format/mean": 0.35546875,
      "rewards/Format/std": 0.4800991117954254,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3130832314491272,
      "rewards/ReflectionBalance/mean": 0.32329100370407104,
      "rewards/ReflectionBalance/std": 0.40646426379680634,
      "step": 252,
      "train_speed(iter/s)": 0.007142
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 772.98046875,
      "completions/min_length": 257.5,
      "epoch": 1.366576819407008,
      "grad_norm": 19.31902564263812,
      "kl": 5.984375,
      "learning_rate": 9.602393899799877e-07,
      "loss": 0.5423768758773804,
      "memory(GiB)": 75.26,
      "reward": 0.8470654189586639,
      "reward_std": 0.7948366403579712,
      "rewards/Format/mean": 0.39453125,
      "rewards/Format/std": 0.4905225783586502,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.2918590381741524,
      "rewards/ReflectionBalance/mean": 0.35487793385982513,
      "rewards/ReflectionBalance/std": 0.4138353019952774,
      "step": 253,
      "train_speed(iter/s)": 0.007142
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 750.1640625,
      "completions/min_length": 253.5,
      "epoch": 1.371967654986523,
      "grad_norm": 11.095023648386126,
      "kl": 4.78125,
      "learning_rate": 9.599034557323672e-07,
      "loss": 0.46057289838790894,
      "memory(GiB)": 75.26,
      "reward": 1.0482128858566284,
      "reward_std": 0.7262905836105347,
      "rewards/Format/mean": 0.453125,
      "rewards/Format/std": 0.4891436845064163,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3666747808456421,
      "rewards/ReflectionBalance/mean": 0.4232128858566284,
      "rewards/ReflectionBalance/std": 0.4281173646450043,
      "step": 254,
      "train_speed(iter/s)": 0.007142
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 734.50390625,
      "completions/min_length": 289.0,
      "epoch": 1.3773584905660377,
      "grad_norm": 5.31483903617705,
      "kl": 3.9765625,
      "learning_rate": 9.595661675752854e-07,
      "loss": 0.46435675024986267,
      "memory(GiB)": 75.26,
      "reward": 1.1491992473602295,
      "reward_std": 0.8997495770454407,
      "rewards/Format/mean": 0.4453125,
      "rewards/Format/std": 0.4983912855386734,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.4346839040517807,
      "rewards/ReflectionBalance/mean": 0.4304492324590683,
      "rewards/ReflectionBalance/std": 0.43250906467437744,
      "step": 255,
      "train_speed(iter/s)": 0.00714
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 742.42578125,
      "completions/min_length": 259.5,
      "epoch": 1.3827493261455526,
      "grad_norm": 7.4519791813388,
      "kl": 3.7578125,
      "learning_rate": 9.592275265016845e-07,
      "loss": 0.47984689474105835,
      "memory(GiB)": 75.26,
      "reward": 0.9389599859714508,
      "reward_std": 0.8723435699939728,
      "rewards/Format/mean": 0.44140625,
      "rewards/Format/std": 0.4928258806467056,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.3029807060956955,
      "rewards/ReflectionBalance/mean": 0.3959911912679672,
      "rewards/ReflectionBalance/std": 0.41137532889842987,
      "step": 256,
      "train_speed(iter/s)": 0.007139
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 751.50390625,
      "completions/min_length": 250.0,
      "epoch": 1.3881401617250675,
      "grad_norm": 5.3135355763504375,
      "kl": 3.5390625,
      "learning_rate": 9.588875335084902e-07,
      "loss": 0.418384313583374,
      "memory(GiB)": 75.26,
      "reward": 0.9349185824394226,
      "reward_std": 0.7227137684822083,
      "rewards/Format/mean": 0.4296875,
      "rewards/Format/std": 0.4877929538488388,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.31232215464115143,
      "rewards/ReflectionBalance/mean": 0.3958561271429062,
      "rewards/ReflectionBalance/std": 0.40994128584861755,
      "step": 257,
      "train_speed(iter/s)": 0.007137
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 771.2265625,
      "completions/min_length": 258.0,
      "epoch": 1.3935309973045822,
      "grad_norm": 8.215776166315507,
      "kl": 3.6328125,
      "learning_rate": 9.585461895966074e-07,
      "loss": 0.46215373277664185,
      "memory(GiB)": 75.26,
      "reward": 0.8613427877426147,
      "reward_std": 0.803362250328064,
      "rewards/Format/mean": 0.41015625,
      "rewards/Format/std": 0.49339206516742706,
      "rewards/MultiModalAccuracyORM/mean": 0.09375,
      "rewards/MultiModalAccuracyORM/std": 0.269597589969635,
      "rewards/ReflectionBalance/mean": 0.35743653774261475,
      "rewards/ReflectionBalance/std": 0.40270908176898956,
      "step": 258,
      "train_speed(iter/s)": 0.007135
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 681.484375,
      "completions/min_length": 250.5,
      "epoch": 1.398921832884097,
      "grad_norm": 9.837399207712084,
      "kl": 3.0546875,
      "learning_rate": 9.58203495770918e-07,
      "loss": 0.4319051504135132,
      "memory(GiB)": 75.26,
      "reward": 1.1400488913059235,
      "reward_std": 0.8182388246059418,
      "rewards/Format/mean": 0.51953125,
      "rewards/Format/std": 0.49189116060733795,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3556671589612961,
      "rewards/ReflectionBalance/mean": 0.4681738466024399,
      "rewards/ReflectionBalance/std": 0.4122476130723953,
      "step": 259,
      "train_speed(iter/s)": 0.007135
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1071.0,
      "completions/mean_length": 712.265625,
      "completions/min_length": 264.0,
      "epoch": 1.4043126684636118,
      "grad_norm": 7.48710950795329,
      "kl": 3.828125,
      "learning_rate": 9.578594530402787e-07,
      "loss": 0.48282763361930847,
      "memory(GiB)": 75.26,
      "reward": 1.0973584055900574,
      "reward_std": 0.811619222164154,
      "rewards/Format/mean": 0.4921875,
      "rewards/Format/std": 0.5009216368198395,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3678421527147293,
      "rewards/ReflectionBalance/mean": 0.4450146555900574,
      "rewards/ReflectionBalance/std": 0.41860075294971466,
      "step": 260,
      "train_speed(iter/s)": 0.007134
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 704.0625,
      "completions/min_length": 221.0,
      "epoch": 1.4097035040431267,
      "grad_norm": 4.397160290002626,
      "kl": 4.3125,
      "learning_rate": 9.57514062417516e-07,
      "loss": 0.5332850217819214,
      "memory(GiB)": 75.26,
      "reward": 1.1082226634025574,
      "reward_std": 0.832923412322998,
      "rewards/Format/mean": 0.51171875,
      "rewards/Format/std": 0.49836522340774536,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35675957798957825,
      "rewards/ReflectionBalance/mean": 0.4480663985013962,
      "rewards/ReflectionBalance/std": 0.414963498711586,
      "step": 261,
      "train_speed(iter/s)": 0.007132
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 671.78125,
      "completions/min_length": 227.0,
      "epoch": 1.4150943396226414,
      "grad_norm": 8.474537636516912,
      "kl": 5.296875,
      "learning_rate": 9.571673249194254e-07,
      "loss": 0.5527722239494324,
      "memory(GiB)": 75.26,
      "reward": 1.1067479848861694,
      "reward_std": 0.8296220004558563,
      "rewards/Format/mean": 0.51953125,
      "rewards/Format/std": 0.49339206516742706,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.339918777346611,
      "rewards/ReflectionBalance/mean": 0.4544043093919754,
      "rewards/ReflectionBalance/std": 0.41460007429122925,
      "step": 262,
      "train_speed(iter/s)": 0.007134
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 718.91796875,
      "completions/min_length": 260.0,
      "epoch": 1.4204851752021563,
      "grad_norm": 21.491986909526698,
      "kl": 6.609375,
      "learning_rate": 9.56819241566767e-07,
      "loss": 0.6292961835861206,
      "memory(GiB)": 75.26,
      "reward": 1.0807958841323853,
      "reward_std": 0.8572716414928436,
      "rewards/Format/mean": 0.47265625,
      "rewards/Format/std": 0.4956269711256027,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3820948451757431,
      "rewards/ReflectionBalance/mean": 0.43235838413238525,
      "rewards/ReflectionBalance/std": 0.41106508672237396,
      "step": 263,
      "train_speed(iter/s)": 0.007134
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 731.88671875,
      "completions/min_length": 269.5,
      "epoch": 1.4258760107816713,
      "grad_norm": 20.471499300960108,
      "kl": 6.609375,
      "learning_rate": 9.564698133842636e-07,
      "loss": 0.5947049856185913,
      "memory(GiB)": 75.26,
      "reward": 0.9772900342941284,
      "reward_std": 0.8479308485984802,
      "rewards/Format/mean": 0.44140625,
      "rewards/Format/std": 0.4972374886274338,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.33029159903526306,
      "rewards/ReflectionBalance/mean": 0.3991650342941284,
      "rewards/ReflectionBalance/std": 0.41501617431640625,
      "step": 264,
      "train_speed(iter/s)": 0.007134
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 688.44140625,
      "completions/min_length": 249.5,
      "epoch": 1.431266846361186,
      "grad_norm": 9.72091612982534,
      "kl": 5.28125,
      "learning_rate": 9.561190414005962e-07,
      "loss": 0.5328714847564697,
      "memory(GiB)": 75.26,
      "reward": 1.0573632419109344,
      "reward_std": 0.8209340274333954,
      "rewards/Format/mean": 0.50390625,
      "rewards/Format/std": 0.49638792872428894,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.2831462025642395,
      "rewards/ReflectionBalance/mean": 0.45189452171325684,
      "rewards/ReflectionBalance/std": 0.41223135590553284,
      "step": 265,
      "train_speed(iter/s)": 0.007134
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 652.8125,
      "completions/min_length": 260.5,
      "epoch": 1.4366576819407009,
      "grad_norm": 3.8910550678178693,
      "kl": 3.6953125,
      "learning_rate": 9.557669266484024e-07,
      "loss": 0.430412232875824,
      "memory(GiB)": 75.26,
      "reward": 1.2129296660423279,
      "reward_std": 0.7378998398780823,
      "rewards/Format/mean": 0.57421875,
      "rewards/Format/std": 0.4863941967487335,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35872696340084076,
      "rewards/ReflectionBalance/mean": 0.48636719584465027,
      "rewards/ReflectionBalance/std": 0.40831972658634186,
      "step": 266,
      "train_speed(iter/s)": 0.007132
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 689.265625,
      "completions/min_length": 235.5,
      "epoch": 1.4420485175202156,
      "grad_norm": 4.160247324254659,
      "kl": 4.890625,
      "learning_rate": 9.554134701642722e-07,
      "loss": 0.5865874290466309,
      "memory(GiB)": 75.26,
      "reward": 1.1012206673622131,
      "reward_std": 0.8707475662231445,
      "rewards/Format/mean": 0.5234375,
      "rewards/Format/std": 0.50135138630867,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3405093401670456,
      "rewards/ReflectionBalance/mean": 0.4449706971645355,
      "rewards/ReflectionBalance/std": 0.42119286954402924,
      "step": 267,
      "train_speed(iter/s)": 0.007131
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 714.87109375,
      "completions/min_length": 265.0,
      "epoch": 1.4474393530997305,
      "grad_norm": 20.62918715581854,
      "kl": 5.375,
      "learning_rate": 9.550586729887462e-07,
      "loss": 0.5787465572357178,
      "memory(GiB)": 75.26,
      "reward": 0.9415625333786011,
      "reward_std": 0.8178361654281616,
      "rewards/Format/mean": 0.4765625,
      "rewards/Format/std": 0.49746260046958923,
      "rewards/MultiModalAccuracyORM/mean": 0.05078125,
      "rewards/MultiModalAccuracyORM/std": 0.2029399611055851,
      "rewards/ReflectionBalance/mean": 0.4142187535762787,
      "rewards/ReflectionBalance/std": 0.40167397260665894,
      "step": 268,
      "train_speed(iter/s)": 0.007132
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 727.1484375,
      "completions/min_length": 268.0,
      "epoch": 1.4528301886792452,
      "grad_norm": 13.398427708957453,
      "kl": 4.609375,
      "learning_rate": 9.547025361663116e-07,
      "loss": 0.5369654893875122,
      "memory(GiB)": 75.26,
      "reward": 0.9932714700698853,
      "reward_std": 0.8170150518417358,
      "rewards/Format/mean": 0.453125,
      "rewards/Format/std": 0.49576351046562195,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.29639076441526413,
      "rewards/ReflectionBalance/mean": 0.43467772006988525,
      "rewards/ReflectionBalance/std": 0.4272761195898056,
      "step": 269,
      "train_speed(iter/s)": 0.007133
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 708.46484375,
      "completions/min_length": 241.0,
      "epoch": 1.45822102425876,
      "grad_norm": 5.041812680715814,
      "kl": 4.421875,
      "learning_rate": 9.54345060745399e-07,
      "loss": 0.5146247148513794,
      "memory(GiB)": 75.26,
      "reward": 0.9812842011451721,
      "reward_std": 0.8457160592079163,
      "rewards/Format/mean": 0.4765625,
      "rewards/Format/std": 0.5011668801307678,
      "rewards/MultiModalAccuracyORM/mean": 0.078125,
      "rewards/MultiModalAccuracyORM/std": 0.2690252959728241,
      "rewards/ReflectionBalance/mean": 0.42659667134284973,
      "rewards/ReflectionBalance/std": 0.415029913187027,
      "step": 270,
      "train_speed(iter/s)": 0.007135
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 702.765625,
      "completions/min_length": 251.5,
      "epoch": 1.463611859838275,
      "grad_norm": 8.39596198647626,
      "kl": 4.0703125,
      "learning_rate": 9.539862477783806e-07,
      "loss": 0.4163432717323303,
      "memory(GiB)": 75.26,
      "reward": 1.022802710533142,
      "reward_std": 0.598138302564621,
      "rewards/Format/mean": 0.50390625,
      "rewards/Format/std": 0.4906538128852844,
      "rewards/MultiModalAccuracyORM/mean": 0.07421875,
      "rewards/MultiModalAccuracyORM/std": 0.232606191188097,
      "rewards/ReflectionBalance/mean": 0.4446777254343033,
      "rewards/ReflectionBalance/std": 0.4066241532564163,
      "step": 271,
      "train_speed(iter/s)": 0.007134
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 688.328125,
      "completions/min_length": 233.0,
      "epoch": 1.4690026954177897,
      "grad_norm": 10.508352953743369,
      "kl": 3.796875,
      "learning_rate": 9.536260983215651e-07,
      "loss": 0.506867527961731,
      "memory(GiB)": 75.26,
      "reward": 1.0735986530780792,
      "reward_std": 0.842479944229126,
      "rewards/Format/mean": 0.515625,
      "rewards/Format/std": 0.4877161830663681,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.3021402209997177,
      "rewards/ReflectionBalance/mean": 0.45641113817691803,
      "rewards/ReflectionBalance/std": 0.42003631591796875,
      "step": 272,
      "train_speed(iter/s)": 0.007134
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 610.5546875,
      "completions/min_length": 249.5,
      "epoch": 1.4743935309973046,
      "grad_norm": 11.468095997454078,
      "kl": 3.453125,
      "learning_rate": 9.532646134351967e-07,
      "loss": 0.5084530115127563,
      "memory(GiB)": 75.26,
      "reward": 1.2917675971984863,
      "reward_std": 0.8037393093109131,
      "rewards/Format/mean": 0.6171875,
      "rewards/Format/std": 0.48470155894756317,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.31088338047266006,
      "rewards/ReflectionBalance/mean": 0.5573925971984863,
      "rewards/ReflectionBalance/std": 0.3980850875377655,
      "step": 273,
      "train_speed(iter/s)": 0.007133
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 679.96875,
      "completions/min_length": 239.0,
      "epoch": 1.4797843665768193,
      "grad_norm": 4.590644373909629,
      "kl": 4.390625,
      "learning_rate": 9.5290179418345e-07,
      "loss": 0.5317939519882202,
      "memory(GiB)": 75.26,
      "reward": 1.1041308343410492,
      "reward_std": 0.8019383251667023,
      "rewards/Format/mean": 0.53125,
      "rewards/Format/std": 0.4987593740224838,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.29199452698230743,
      "rewards/ReflectionBalance/mean": 0.4674120992422104,
      "rewards/ReflectionBalance/std": 0.40456752479076385,
      "step": 274,
      "train_speed(iter/s)": 0.007133
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 677.4140625,
      "completions/min_length": 238.0,
      "epoch": 1.4851752021563343,
      "grad_norm": 5.66055521812624,
      "kl": 4.734375,
      "learning_rate": 9.525376416344289e-07,
      "loss": 0.5343739986419678,
      "memory(GiB)": 75.26,
      "reward": 1.2091357707977295,
      "reward_std": 0.8206020891666412,
      "rewards/Format/mean": 0.56640625,
      "rewards/Format/std": 0.4929484575986862,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35062892735004425,
      "rewards/ReflectionBalance/mean": 0.4942919909954071,
      "rewards/ReflectionBalance/std": 0.4168001115322113,
      "step": 275,
      "train_speed(iter/s)": 0.007132
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 644.0546875,
      "completions/min_length": 227.0,
      "epoch": 1.490566037735849,
      "grad_norm": 4.631951469711305,
      "kl": 4.984375,
      "learning_rate": 9.521721568601613e-07,
      "loss": 0.5658146142959595,
      "memory(GiB)": 75.26,
      "reward": 1.2127490639686584,
      "reward_std": 0.9015666544437408,
      "rewards/Format/mean": 0.5859375,
      "rewards/Format/std": 0.49443069100379944,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3131764084100723,
      "rewards/ReflectionBalance/mean": 0.5135302543640137,
      "rewards/ReflectionBalance/std": 0.410501629114151,
      "step": 276,
      "train_speed(iter/s)": 0.007132
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 648.9765625,
      "completions/min_length": 246.0,
      "epoch": 1.4959568733153639,
      "grad_norm": 7.4835187296688455,
      "kl": 5.609375,
      "learning_rate": 9.518053409365979e-07,
      "loss": 0.5834148526191711,
      "memory(GiB)": 75.26,
      "reward": 1.2125830054283142,
      "reward_std": 0.8423444926738739,
      "rewards/Format/mean": 0.55859375,
      "rewards/Format/std": 0.4949738383293152,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.35641176998615265,
      "rewards/ReflectionBalance/mean": 0.4821142554283142,
      "rewards/ReflectionBalance/std": 0.41228486597537994,
      "step": 277,
      "train_speed(iter/s)": 0.007132
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 684.37109375,
      "completions/min_length": 235.0,
      "epoch": 1.5013477088948788,
      "grad_norm": 9.562059587271962,
      "kl": 5.984375,
      "learning_rate": 9.514371949436078e-07,
      "loss": 0.6001251935958862,
      "memory(GiB)": 75.26,
      "reward": 1.0813086032867432,
      "reward_std": 0.8019672632217407,
      "rewards/Format/mean": 0.52734375,
      "rewards/Format/std": 0.4993480294942856,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.29434798657894135,
      "rewards/ReflectionBalance/mean": 0.45630861818790436,
      "rewards/ReflectionBalance/std": 0.41210320591926575,
      "step": 278,
      "train_speed(iter/s)": 0.007132
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 602.7734375,
      "completions/min_length": 267.0,
      "epoch": 1.5067385444743935,
      "grad_norm": 16.025681037464775,
      "kl": 4.734375,
      "learning_rate": 9.510677199649756e-07,
      "loss": 0.5614305734634399,
      "memory(GiB)": 75.26,
      "reward": 1.3397265672683716,
      "reward_std": 0.841354638338089,
      "rewards/Format/mean": 0.64453125,
      "rewards/Format/std": 0.47911877930164337,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3318064510822296,
      "rewards/ReflectionBalance/mean": 0.5701953172683716,
      "rewards/ReflectionBalance/std": 0.3959491550922394,
      "step": 279,
      "train_speed(iter/s)": 0.007133
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 630.21484375,
      "completions/min_length": 232.5,
      "epoch": 1.5121293800539084,
      "grad_norm": 6.285141184566698,
      "kl": 4.5,
      "learning_rate": 9.506969170883985e-07,
      "loss": 0.579586386680603,
      "memory(GiB)": 75.26,
      "reward": 1.230162799358368,
      "reward_std": 0.820617288351059,
      "rewards/Format/mean": 0.6015625,
      "rewards/Format/std": 0.47444386780261993,
      "rewards/MultiModalAccuracyORM/mean": 0.08984375,
      "rewards/MultiModalAccuracyORM/std": 0.27175262570381165,
      "rewards/ReflectionBalance/mean": 0.5387565195560455,
      "rewards/ReflectionBalance/std": 0.39645645022392273,
      "step": 280,
      "train_speed(iter/s)": 0.007132
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 607.85546875,
      "completions/min_length": 266.5,
      "epoch": 1.5175202156334233,
      "grad_norm": 5.193151202116044,
      "kl": 4.78125,
      "learning_rate": 9.50324787405483e-07,
      "loss": 0.561400294303894,
      "memory(GiB)": 75.26,
      "reward": 1.3484374284744263,
      "reward_std": 0.8590945601463318,
      "rewards/Format/mean": 0.625,
      "rewards/Format/std": 0.482703298330307,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38489311933517456,
      "rewards/ReflectionBalance/mean": 0.5437500029802322,
      "rewards/ReflectionBalance/std": 0.39743833243846893,
      "step": 281,
      "train_speed(iter/s)": 0.007131
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 615.078125,
      "completions/min_length": 247.5,
      "epoch": 1.522911051212938,
      "grad_norm": 6.284543623791351,
      "kl": 5.015625,
      "learning_rate": 9.499513320117411e-07,
      "loss": 0.5607603788375854,
      "memory(GiB)": 75.26,
      "reward": 1.2825894951820374,
      "reward_std": 0.8347463011741638,
      "rewards/Format/mean": 0.59765625,
      "rewards/Format/std": 0.49228109419345856,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3734995424747467,
      "rewards/ReflectionBalance/mean": 0.5169645249843597,
      "rewards/ReflectionBalance/std": 0.40255117416381836,
      "step": 282,
      "train_speed(iter/s)": 0.007131
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 720.6875,
      "completions/min_length": 264.5,
      "epoch": 1.5283018867924527,
      "grad_norm": 9.616655256419357,
      "kl": 5.328125,
      "learning_rate": 9.495765520065881e-07,
      "loss": 0.5188866853713989,
      "memory(GiB)": 75.26,
      "reward": 1.0675243437290192,
      "reward_std": 0.8015342652797699,
      "rewards/Format/mean": 0.4921875,
      "rewards/Format/std": 0.49573568999767303,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.34447427093982697,
      "rewards/ReflectionBalance/mean": 0.43861818313598633,
      "rewards/ReflectionBalance/std": 0.4176819324493408,
      "step": 283,
      "train_speed(iter/s)": 0.00713
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 710.3515625,
      "completions/min_length": 267.5,
      "epoch": 1.5336927223719676,
      "grad_norm": 10.729974946713263,
      "kl": 6.3125,
      "learning_rate": 9.492004484933384e-07,
      "loss": 0.6258677244186401,
      "memory(GiB)": 75.26,
      "reward": 0.9719336032867432,
      "reward_std": 0.8494085073471069,
      "rewards/Format/mean": 0.4765625,
      "rewards/Format/std": 0.50135138630867,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.2961650490760803,
      "rewards/ReflectionBalance/mean": 0.39771483838558197,
      "rewards/ReflectionBalance/std": 0.4159630984067917,
      "step": 284,
      "train_speed(iter/s)": 0.007131
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 697.72265625,
      "completions/min_length": 279.5,
      "epoch": 1.5390835579514826,
      "grad_norm": 4.069706725280358,
      "kl": 4.53125,
      "learning_rate": 9.488230225792031e-07,
      "loss": 0.49928754568099976,
      "memory(GiB)": 75.26,
      "reward": 1.0796923339366913,
      "reward_std": 0.7710286378860474,
      "rewards/Format/mean": 0.515625,
      "rewards/Format/std": 0.4967239648103714,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3068038523197174,
      "rewards/ReflectionBalance/mean": 0.45469237864017487,
      "rewards/ReflectionBalance/std": 0.41626761853694916,
      "step": 285,
      "train_speed(iter/s)": 0.007128
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1033.5,
      "completions/mean_length": 694.23828125,
      "completions/min_length": 247.5,
      "epoch": 1.5444743935309972,
      "grad_norm": 47.45433744767722,
      "kl": 4.984375,
      "learning_rate": 9.484442753752861e-07,
      "loss": 0.523672342300415,
      "memory(GiB)": 75.26,
      "reward": 1.1070800721645355,
      "reward_std": 0.8684592843055725,
      "rewards/Format/mean": 0.53125,
      "rewards/Format/std": 0.4979538321495056,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.3021402209997177,
      "rewards/ReflectionBalance/mean": 0.4742676019668579,
      "rewards/ReflectionBalance/std": 0.42774562537670135,
      "step": 286,
      "train_speed(iter/s)": 0.007128
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 658.828125,
      "completions/min_length": 266.5,
      "epoch": 1.5498652291105122,
      "grad_norm": 10.718536356806178,
      "kl": 3.375,
      "learning_rate": 9.480642079965806e-07,
      "loss": 0.43647998571395874,
      "memory(GiB)": 75.26,
      "reward": 1.2634928226470947,
      "reward_std": 0.8041374683380127,
      "rewards/Format/mean": 0.58203125,
      "rewards/Format/std": 0.4947638362646103,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3540267199277878,
      "rewards/ReflectionBalance/mean": 0.5056803524494171,
      "rewards/ReflectionBalance/std": 0.4224071651697159,
      "step": 287,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 609.859375,
      "completions/min_length": 252.0,
      "epoch": 1.555256064690027,
      "grad_norm": 14.502155776921816,
      "kl": 3.171875,
      "learning_rate": 9.476828215619672e-07,
      "loss": 0.47286948561668396,
      "memory(GiB)": 75.26,
      "reward": 1.4248730540275574,
      "reward_std": 0.8287319839000702,
      "rewards/Format/mean": 0.63671875,
      "rewards/Format/std": 0.48267970979213715,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.39705096185207367,
      "rewards/ReflectionBalance/mean": 0.573310524225235,
      "rewards/ReflectionBalance/std": 0.39642302691936493,
      "step": 288,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 618.4140625,
      "completions/min_length": 225.0,
      "epoch": 1.5606469002695418,
      "grad_norm": 8.292619136207163,
      "kl": 3.390625,
      "learning_rate": 9.47300117194209e-07,
      "loss": 0.45564189553260803,
      "memory(GiB)": 75.26,
      "reward": 1.2834423780441284,
      "reward_std": 0.8035134971141815,
      "rewards/Format/mean": 0.62109375,
      "rewards/Format/std": 0.48417605459690094,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3475525379180908,
      "rewards/ReflectionBalance/mean": 0.5178173631429672,
      "rewards/ReflectionBalance/std": 0.3972063958644867,
      "step": 289,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 576.4140625,
      "completions/min_length": 210.5,
      "epoch": 1.5660377358490565,
      "grad_norm": 17.560909218447023,
      "kl": 3.5390625,
      "learning_rate": 9.469160960199489e-07,
      "loss": 0.4553975462913513,
      "memory(GiB)": 75.26,
      "reward": 1.5335204601287842,
      "reward_std": 0.7431342005729675,
      "rewards/Format/mean": 0.703125,
      "rewards/Format/std": 0.45835477113723755,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42526112496852875,
      "rewards/ReflectionBalance/mean": 0.588208019733429,
      "rewards/ReflectionBalance/std": 0.3770087957382202,
      "step": 290,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 663.328125,
      "completions/min_length": 254.0,
      "epoch": 1.5714285714285714,
      "grad_norm": 5.224895778734647,
      "kl": 4.578125,
      "learning_rate": 9.465307591697069e-07,
      "loss": 0.5444307923316956,
      "memory(GiB)": 75.26,
      "reward": 1.1497558951377869,
      "reward_std": 0.8740359544754028,
      "rewards/Format/mean": 0.55859375,
      "rewards/Format/std": 0.4977390468120575,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3166656941175461,
      "rewards/ReflectionBalance/mean": 0.4778808653354645,
      "rewards/ReflectionBalance/std": 0.4172118604183197,
      "step": 291,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 639.7734375,
      "completions/min_length": 266.5,
      "epoch": 1.5768194070080863,
      "grad_norm": 4.8506127510580415,
      "kl": 4.265625,
      "learning_rate": 9.461441077778756e-07,
      "loss": 0.4810435175895691,
      "memory(GiB)": 75.26,
      "reward": 1.3322119116783142,
      "reward_std": 0.8077206611633301,
      "rewards/Format/mean": 0.62109375,
      "rewards/Format/std": 0.48565995693206787,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.40343451499938965,
      "rewards/ReflectionBalance/mean": 0.5079931616783142,
      "rewards/ReflectionBalance/std": 0.428008109331131,
      "step": 292,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 593.421875,
      "completions/min_length": 212.0,
      "epoch": 1.582210242587601,
      "grad_norm": 5.727230516603576,
      "kl": 4.984375,
      "learning_rate": 9.457561429827179e-07,
      "loss": 0.540503978729248,
      "memory(GiB)": 75.26,
      "reward": 1.3166308999061584,
      "reward_std": 0.6910475790500641,
      "rewards/Format/mean": 0.65625,
      "rewards/Format/std": 0.47424188256263733,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3130832314491272,
      "rewards/ReflectionBalance/mean": 0.5510058403015137,
      "rewards/ReflectionBalance/std": 0.38134047389030457,
      "step": 293,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 613.6953125,
      "completions/min_length": 241.5,
      "epoch": 1.587601078167116,
      "grad_norm": 4.279590330008388,
      "kl": 4.640625,
      "learning_rate": 9.453668659263632e-07,
      "loss": 0.5103491544723511,
      "memory(GiB)": 75.26,
      "reward": 1.374233365058899,
      "reward_std": 0.7511624395847321,
      "rewards/Format/mean": 0.67578125,
      "rewards/Format/std": 0.4631071090698242,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.310598224401474,
      "rewards/ReflectionBalance/mean": 0.5851708650588989,
      "rewards/ReflectionBalance/std": 0.3869447112083435,
      "step": 294,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.5,
      "completions/mean_length": 598.61328125,
      "completions/min_length": 267.0,
      "epoch": 1.5929919137466308,
      "grad_norm": 5.025207188728187,
      "kl": 4.765625,
      "learning_rate": 9.449762777548037e-07,
      "loss": 0.5237293839454651,
      "memory(GiB)": 75.26,
      "reward": 1.4026415944099426,
      "reward_std": 0.7944015264511108,
      "rewards/Format/mean": 0.64453125,
      "rewards/Format/std": 0.47967928647994995,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3985297381877899,
      "rewards/ReflectionBalance/mean": 0.5588915944099426,
      "rewards/ReflectionBalance/std": 0.3980036973953247,
      "step": 295,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 631.07421875,
      "completions/min_length": 266.5,
      "epoch": 1.5983827493261455,
      "grad_norm": 5.316384234180509,
      "kl": 5.140625,
      "learning_rate": 9.445843796178918e-07,
      "loss": 0.559828519821167,
      "memory(GiB)": 75.26,
      "reward": 1.2815918326377869,
      "reward_std": 0.8013547956943512,
      "rewards/Format/mean": 0.6328125,
      "rewards/Format/std": 0.48056647181510925,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.30921147763729095,
      "rewards/ReflectionBalance/mean": 0.5394042730331421,
      "rewards/ReflectionBalance/std": 0.4115152657032013,
      "step": 296,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 605.9609375,
      "completions/min_length": 247.5,
      "epoch": 1.6037735849056602,
      "grad_norm": 4.653582659319309,
      "kl": 4.609375,
      "learning_rate": 9.441911726693361e-07,
      "loss": 0.49691247940063477,
      "memory(GiB)": 75.26,
      "reward": 1.358432650566101,
      "reward_std": 0.7750959396362305,
      "rewards/Format/mean": 0.625,
      "rewards/Format/std": 0.48168063163757324,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.38954539597034454,
      "rewards/ReflectionBalance/mean": 0.5420263707637787,
      "rewards/ReflectionBalance/std": 0.39982008934020996,
      "step": 297,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 629.765625,
      "completions/min_length": 263.5,
      "epoch": 1.6091644204851752,
      "grad_norm": 4.480520375028442,
      "kl": 4.484375,
      "learning_rate": 9.437966580666983e-07,
      "loss": 0.517652153968811,
      "memory(GiB)": 75.26,
      "reward": 1.3357080221176147,
      "reward_std": 0.8126619458198547,
      "rewards/Format/mean": 0.6484375,
      "rewards/Format/std": 0.4767917990684509,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3407054841518402,
      "rewards/ReflectionBalance/mean": 0.5544579923152924,
      "rewards/ReflectionBalance/std": 0.39359529316425323,
      "step": 298,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 572.3046875,
      "completions/min_length": 264.5,
      "epoch": 1.61455525606469,
      "grad_norm": 10.89766767746794,
      "kl": 3.2578125,
      "learning_rate": 9.434008369713897e-07,
      "loss": 0.4676779508590698,
      "memory(GiB)": 75.26,
      "reward": 1.4285660982131958,
      "reward_std": 0.7319612503051758,
      "rewards/Format/mean": 0.7265625,
      "rewards/Format/std": 0.4431992620229721,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3177158683538437,
      "rewards/ReflectionBalance/mean": 0.5809098482131958,
      "rewards/ReflectionBalance/std": 0.3795599937438965,
      "step": 299,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 564.10546875,
      "completions/min_length": 222.0,
      "epoch": 1.6199460916442048,
      "grad_norm": 12.01673308421441,
      "kl": 3.53125,
      "learning_rate": 9.430037105486676e-07,
      "loss": 0.5043653249740601,
      "memory(GiB)": 75.26,
      "reward": 1.562900424003601,
      "reward_std": 0.779889702796936,
      "rewards/Format/mean": 0.73828125,
      "rewards/Format/std": 0.4410945028066635,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3973046988248825,
      "rewards/ReflectionBalance/mean": 0.6254003942012787,
      "rewards/ReflectionBalance/std": 0.36803656816482544,
      "step": 300,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 654.99609375,
      "completions/min_length": 223.5,
      "epoch": 1.6253369272237197,
      "grad_norm": 4.668126218469078,
      "kl": 5.21875,
      "learning_rate": 9.426052799676323e-07,
      "loss": 0.5698080062866211,
      "memory(GiB)": 75.26,
      "reward": 1.1102929711341858,
      "reward_std": 0.8030151724815369,
      "rewards/Format/mean": 0.55859375,
      "rewards/Format/std": 0.495855450630188,
      "rewards/MultiModalAccuracyORM/mean": 0.05859375,
      "rewards/MultiModalAccuracyORM/std": 0.2356376200914383,
      "rewards/ReflectionBalance/mean": 0.4931054711341858,
      "rewards/ReflectionBalance/std": 0.41784971952438354,
      "step": 301,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 567.375,
      "completions/min_length": 250.5,
      "epoch": 1.6307277628032346,
      "grad_norm": 6.111955913788258,
      "kl": 3.7421875,
      "learning_rate": 9.422055464012233e-07,
      "loss": 0.4882737398147583,
      "memory(GiB)": 75.26,
      "reward": 1.4240283966064453,
      "reward_std": 0.7547650635242462,
      "rewards/Format/mean": 0.71484375,
      "rewards/Format/std": 0.44853854179382324,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.31812146306037903,
      "rewards/ReflectionBalance/mean": 0.5959033370018005,
      "rewards/ReflectionBalance/std": 0.3925890475511551,
      "step": 302,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 592.8671875,
      "completions/min_length": 257.5,
      "epoch": 1.6361185983827493,
      "grad_norm": 4.168478022098693,
      "kl": 5.4375,
      "learning_rate": 9.418045110262161e-07,
      "loss": 0.5979712009429932,
      "memory(GiB)": 75.26,
      "reward": 1.3260546922683716,
      "reward_std": 0.781084269285202,
      "rewards/Format/mean": 0.66796875,
      "rewards/Format/std": 0.45698806643486023,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.29199452698230743,
      "rewards/ReflectionBalance/mean": 0.5526171773672104,
      "rewards/ReflectionBalance/std": 0.39152073860168457,
      "step": 303,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 620.66796875,
      "completions/min_length": 249.0,
      "epoch": 1.641509433962264,
      "grad_norm": 5.198233778889643,
      "kl": 4.859375,
      "learning_rate": 9.414021750232182e-07,
      "loss": 0.5323922634124756,
      "memory(GiB)": 75.26,
      "reward": 1.4065381288528442,
      "reward_std": 0.8392544090747833,
      "rewards/Format/mean": 0.64453125,
      "rewards/Format/std": 0.4805184155702591,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3940746486186981,
      "rewards/ReflectionBalance/mean": 0.5627880990505219,
      "rewards/ReflectionBalance/std": 0.40340785682201385,
      "step": 304,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 561.36328125,
      "completions/min_length": 260.0,
      "epoch": 1.646900269541779,
      "grad_norm": 4.158991186792216,
      "kl": 4.734375,
      "learning_rate": 9.409985395766665e-07,
      "loss": 0.5767388939857483,
      "memory(GiB)": 75.26,
      "reward": 1.4817577600479126,
      "reward_std": 0.7550703883171082,
      "rewards/Format/mean": 0.7265625,
      "rewards/Format/std": 0.4460870176553726,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3607245087623596,
      "rewards/ReflectionBalance/mean": 0.6028515696525574,
      "rewards/ReflectionBalance/std": 0.3690946400165558,
      "step": 305,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 610.8359375,
      "completions/min_length": 249.0,
      "epoch": 1.6522911051212938,
      "grad_norm": 7.931139131395596,
      "kl": 5.875,
      "learning_rate": 9.405936058748228e-07,
      "loss": 0.6071507930755615,
      "memory(GiB)": 75.26,
      "reward": 1.3060400485992432,
      "reward_std": 0.7759750187397003,
      "rewards/Format/mean": 0.6328125,
      "rewards/Format/std": 0.4833163321018219,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3410066068172455,
      "rewards/ReflectionBalance/mean": 0.5365087985992432,
      "rewards/ReflectionBalance/std": 0.4071555882692337,
      "step": 306,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 635.6171875,
      "completions/min_length": 264.5,
      "epoch": 1.6576819407008085,
      "grad_norm": 7.841273995353563,
      "kl": 5.71875,
      "learning_rate": 9.401873751097715e-07,
      "loss": 0.5931523442268372,
      "memory(GiB)": 75.26,
      "reward": 1.2320165634155273,
      "reward_std": 0.8012723922729492,
      "rewards/Format/mean": 0.62890625,
      "rewards/Format/std": 0.4836175739765167,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.29434798657894135,
      "rewards/ReflectionBalance/mean": 0.5054540932178497,
      "rewards/ReflectionBalance/std": 0.4167952835559845,
      "step": 307,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 566.0,
      "completions/min_length": 257.5,
      "epoch": 1.6630727762803235,
      "grad_norm": 3.9874201984320505,
      "kl": 4.109375,
      "learning_rate": 9.397798484774149e-07,
      "loss": 0.48679158091545105,
      "memory(GiB)": 75.26,
      "reward": 1.5446679592132568,
      "reward_std": 0.7023330628871918,
      "rewards/Format/mean": 0.7421875,
      "rewards/Format/std": 0.43905749917030334,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3749772012233734,
      "rewards/ReflectionBalance/mean": 0.6345117092132568,
      "rewards/ReflectionBalance/std": 0.37851642072200775,
      "step": 308,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 582.58203125,
      "completions/min_length": 259.5,
      "epoch": 1.6684636118598384,
      "grad_norm": 5.095590867447187,
      "kl": 4.6875,
      "learning_rate": 9.393710271774703e-07,
      "loss": 0.5627733469009399,
      "memory(GiB)": 75.26,
      "reward": 1.4486035704612732,
      "reward_std": 0.8039027452468872,
      "rewards/Format/mean": 0.7109375,
      "rewards/Format/std": 0.4543672651052475,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3285723924636841,
      "rewards/ReflectionBalance/mean": 0.6126660108566284,
      "rewards/ReflectionBalance/std": 0.3741928040981293,
      "step": 309,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 613.125,
      "completions/min_length": 239.0,
      "epoch": 1.673854447439353,
      "grad_norm": 5.160338672398014,
      "kl": 5.03125,
      "learning_rate": 9.389609124134665e-07,
      "loss": 0.5450788140296936,
      "memory(GiB)": 75.26,
      "reward": 1.3870947360992432,
      "reward_std": 0.7988343238830566,
      "rewards/Format/mean": 0.65234375,
      "rewards/Format/std": 0.47594553232192993,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3979988098144531,
      "rewards/ReflectionBalance/mean": 0.5394384562969208,
      "rewards/ReflectionBalance/std": 0.39577673375606537,
      "step": 310,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 573.81640625,
      "completions/min_length": 251.5,
      "epoch": 1.6792452830188678,
      "grad_norm": 2.945567618810988,
      "kl": 4.40625,
      "learning_rate": 9.385495053927402e-07,
      "loss": 0.5038343667984009,
      "memory(GiB)": 75.26,
      "reward": 1.4804052114486694,
      "reward_std": 0.759300023317337,
      "rewards/Format/mean": 0.6953125,
      "rewards/Format/std": 0.4620049446821213,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3716365247964859,
      "rewards/ReflectionBalance/mean": 0.6210302710533142,
      "rewards/ReflectionBalance/std": 0.3755379617214203,
      "step": 311,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 556.71875,
      "completions/min_length": 242.5,
      "epoch": 1.684636118598383,
      "grad_norm": 3.9083789097794606,
      "kl": 4.90625,
      "learning_rate": 9.381368073264321e-07,
      "loss": 0.5287462472915649,
      "memory(GiB)": 75.26,
      "reward": 1.4600390791893005,
      "reward_std": 0.7231628894805908,
      "rewards/Format/mean": 0.71484375,
      "rewards/Format/std": 0.45274218916893005,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3556671589612961,
      "rewards/ReflectionBalance/mean": 0.5928515493869781,
      "rewards/ReflectionBalance/std": 0.3884482681751251,
      "step": 312,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 574.44921875,
      "completions/min_length": 273.5,
      "epoch": 1.6900269541778976,
      "grad_norm": 5.768019444969116,
      "kl": 4.984375,
      "learning_rate": 9.377228194294837e-07,
      "loss": 0.5609692931175232,
      "memory(GiB)": 75.26,
      "reward": 1.5291406512260437,
      "reward_std": 0.782200813293457,
      "rewards/Format/mean": 0.71484375,
      "rewards/Format/std": 0.4471786171197891,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.4009389728307724,
      "rewards/ReflectionBalance/mean": 0.6111718714237213,
      "rewards/ReflectionBalance/std": 0.3809555619955063,
      "step": 313,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 550.1171875,
      "completions/min_length": 244.0,
      "epoch": 1.6954177897574123,
      "grad_norm": 15.772168573729877,
      "kl": 5.484375,
      "learning_rate": 9.373075429206338e-07,
      "loss": 0.5421291589736938,
      "memory(GiB)": 75.26,
      "reward": 1.6059765815734863,
      "reward_std": 0.7751793563365936,
      "rewards/Format/mean": 0.74609375,
      "rewards/Format/std": 0.4369315803050995,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.4164551943540573,
      "rewards/ReflectionBalance/mean": 0.6333203017711639,
      "rewards/ReflectionBalance/std": 0.37638038396835327,
      "step": 314,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 557.82421875,
      "completions/min_length": 264.5,
      "epoch": 1.7008086253369272,
      "grad_norm": 3.3446005022968532,
      "kl": 5.796875,
      "learning_rate": 9.368909790224147e-07,
      "loss": 0.6357285976409912,
      "memory(GiB)": 75.26,
      "reward": 1.4957616925239563,
      "reward_std": 0.8474238812923431,
      "rewards/Format/mean": 0.72265625,
      "rewards/Format/std": 0.4489126205444336,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3949224203824997,
      "rewards/ReflectionBalance/mean": 0.5816992223262787,
      "rewards/ReflectionBalance/std": 0.39357730746269226,
      "step": 315,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 580.71484375,
      "completions/min_length": 228.5,
      "epoch": 1.7061994609164421,
      "grad_norm": 6.179852993360216,
      "kl": 4.90625,
      "learning_rate": 9.364731289611483e-07,
      "loss": 0.4820398688316345,
      "memory(GiB)": 75.26,
      "reward": 1.421167016029358,
      "reward_std": 0.723140686750412,
      "rewards/Format/mean": 0.6953125,
      "rewards/Format/std": 0.4620049446821213,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3437131494283676,
      "rewards/ReflectionBalance/mean": 0.5891357660293579,
      "rewards/ReflectionBalance/std": 0.3983532786369324,
      "step": 316,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 522.83984375,
      "completions/min_length": 244.0,
      "epoch": 1.7115902964959568,
      "grad_norm": 4.450537038189734,
      "kl": 3.609375,
      "learning_rate": 9.360539939669433e-07,
      "loss": 0.4143209755420685,
      "memory(GiB)": 75.26,
      "reward": 1.6429736018180847,
      "reward_std": 0.6735435426235199,
      "rewards/Format/mean": 0.79296875,
      "rewards/Format/std": 0.40016549825668335,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3946705758571625,
      "rewards/ReflectionBalance/mean": 0.6585986316204071,
      "rewards/ReflectionBalance/std": 0.3540024757385254,
      "step": 317,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 538.1796875,
      "completions/min_length": 216.0,
      "epoch": 1.7169811320754715,
      "grad_norm": 6.794845576184022,
      "kl": 3.875,
      "learning_rate": 9.356335752736908e-07,
      "loss": 0.4934839904308319,
      "memory(GiB)": 75.26,
      "reward": 1.6541666388511658,
      "reward_std": 0.7643243968486786,
      "rewards/Format/mean": 0.7734375,
      "rewards/Format/std": 0.4193098843097687,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4225567579269409,
      "rewards/ReflectionBalance/mean": 0.6502604186534882,
      "rewards/ReflectionBalance/std": 0.3628522455692291,
      "step": 318,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 524.47265625,
      "completions/min_length": 252.0,
      "epoch": 1.7223719676549867,
      "grad_norm": 8.520250407854823,
      "kl": 3.4296875,
      "learning_rate": 9.352118741190613e-07,
      "loss": 0.45843905210494995,
      "memory(GiB)": 75.26,
      "reward": 1.6121484637260437,
      "reward_std": 0.6816348731517792,
      "rewards/Format/mean": 0.8125,
      "rewards/Format/std": 0.39068229496479034,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3456958532333374,
      "rewards/ReflectionBalance/mean": 0.6394922137260437,
      "rewards/ReflectionBalance/std": 0.34818336367607117,
      "step": 319,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 558.58984375,
      "completions/min_length": 262.0,
      "epoch": 1.7277628032345014,
      "grad_norm": 5.062740383277203,
      "kl": 4.71875,
      "learning_rate": 9.347888917445004e-07,
      "loss": 0.4970533847808838,
      "memory(GiB)": 75.26,
      "reward": 1.5563428401947021,
      "reward_std": 0.6988635063171387,
      "rewards/Format/mean": 0.75,
      "rewards/Format/std": 0.40902179479599,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3749772012233734,
      "rewards/ReflectionBalance/mean": 0.6383740305900574,
      "rewards/ReflectionBalance/std": 0.3591048866510391,
      "step": 320,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.5,
      "completions/mean_length": 577.30078125,
      "completions/min_length": 218.0,
      "epoch": 1.733153638814016,
      "grad_norm": 8.689697463627905,
      "kl": 5.640625,
      "learning_rate": 9.343646293952254e-07,
      "loss": 0.559558629989624,
      "memory(GiB)": 75.26,
      "reward": 1.5328369140625,
      "reward_std": 0.8472154140472412,
      "rewards/Format/mean": 0.703125,
      "rewards/Format/std": 0.45835477113723755,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.4460870176553726,
      "rewards/ReflectionBalance/mean": 0.5562744140625,
      "rewards/ReflectionBalance/std": 0.4064269959926605,
      "step": 321,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1036.0,
      "completions/mean_length": 577.1171875,
      "completions/min_length": 240.5,
      "epoch": 1.738544474393531,
      "grad_norm": 7.652914056540351,
      "kl": 5.390625,
      "learning_rate": 9.339390883202223e-07,
      "loss": 0.5419362783432007,
      "memory(GiB)": 75.26,
      "reward": 1.3818018436431885,
      "reward_std": 0.7146711349487305,
      "rewards/Format/mean": 0.6953125,
      "rewards/Format/std": 0.460112988948822,
      "rewards/MultiModalAccuracyORM/mean": 0.09375,
      "rewards/MultiModalAccuracyORM/std": 0.2923160344362259,
      "rewards/ReflectionBalance/mean": 0.5927392542362213,
      "rewards/ReflectionBalance/std": 0.3846490979194641,
      "step": 322,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 549.6796875,
      "completions/min_length": 277.5,
      "epoch": 1.743935309973046,
      "grad_norm": 7.152525540923269,
      "kl": 4.375,
      "learning_rate": 9.33512269772241e-07,
      "loss": 0.5264996886253357,
      "memory(GiB)": 75.26,
      "reward": 1.4967577457427979,
      "reward_std": 0.7951205372810364,
      "rewards/Format/mean": 0.76953125,
      "rewards/Format/std": 0.42152732610702515,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.29794247448444366,
      "rewards/ReflectionBalance/mean": 0.629570335149765,
      "rewards/ReflectionBalance/std": 0.3724069595336914,
      "step": 323,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.5,
      "completions/mean_length": 502.4765625,
      "completions/min_length": 251.5,
      "epoch": 1.7493261455525606,
      "grad_norm": 4.52630708746868,
      "kl": 3.7578125,
      "learning_rate": 9.330841750077923e-07,
      "loss": 0.4527936577796936,
      "memory(GiB)": 75.26,
      "reward": 1.6944043040275574,
      "reward_std": 0.6950302422046661,
      "rewards/Format/mean": 0.8046875,
      "rewards/Format/std": 0.3823556900024414,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41699807345867157,
      "rewards/ReflectionBalance/mean": 0.667060524225235,
      "rewards/ReflectionBalance/std": 0.3378310054540634,
      "step": 324,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 580.109375,
      "completions/min_length": 150.0,
      "epoch": 1.7547169811320755,
      "grad_norm": 3.9243983592358362,
      "kl": 5.140625,
      "learning_rate": 9.326548052871441e-07,
      "loss": 0.5476266145706177,
      "memory(GiB)": 75.26,
      "reward": 1.436059594154358,
      "reward_std": 0.7465664446353912,
      "rewards/Format/mean": 0.70703125,
      "rewards/Format/std": 0.4568922370672226,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.3341423198580742,
      "rewards/ReflectionBalance/mean": 0.5805908441543579,
      "rewards/ReflectionBalance/std": 0.3841336518526077,
      "step": 325,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.0,
      "completions/mean_length": 568.5390625,
      "completions/min_length": 236.5,
      "epoch": 1.7601078167115904,
      "grad_norm": 4.512859418154903,
      "kl": 5.2109375,
      "learning_rate": 9.322241618743177e-07,
      "loss": 0.5399398803710938,
      "memory(GiB)": 75.26,
      "reward": 1.4018993973731995,
      "reward_std": 0.7276022434234619,
      "rewards/Format/mean": 0.6875,
      "rewards/Format/std": 0.45404255390167236,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3405093401670456,
      "rewards/ReflectionBalance/mean": 0.5815869569778442,
      "rewards/ReflectionBalance/std": 0.39593881368637085,
      "step": 326,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 525.015625,
      "completions/min_length": 265.0,
      "epoch": 1.7654986522911051,
      "grad_norm": 6.806770132490562,
      "kl": 4.203125,
      "learning_rate": 9.317922460370839e-07,
      "loss": 0.5083276629447937,
      "memory(GiB)": 75.26,
      "reward": 1.53275066614151,
      "reward_std": 0.753451943397522,
      "rewards/Format/mean": 0.765625,
      "rewards/Format/std": 0.4243630915880203,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3334042727947235,
      "rewards/ReflectionBalance/mean": 0.6343131363391876,
      "rewards/ReflectionBalance/std": 0.3683331310749054,
      "step": 327,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1033.5,
      "completions/mean_length": 523.55078125,
      "completions/min_length": 216.0,
      "epoch": 1.7708894878706198,
      "grad_norm": 6.837039923063542,
      "kl": 4.046875,
      "learning_rate": 9.313590590469593e-07,
      "loss": 0.5170900821685791,
      "memory(GiB)": 75.26,
      "reward": 1.5387110114097595,
      "reward_std": 0.7017663717269897,
      "rewards/Format/mean": 0.79296875,
      "rewards/Format/std": 0.4032507836818695,
      "rewards/MultiModalAccuracyORM/mean": 0.08203125,
      "rewards/MultiModalAccuracyORM/std": 0.27079907804727554,
      "rewards/ReflectionBalance/mean": 0.6637109518051147,
      "rewards/ReflectionBalance/std": 0.34179484844207764,
      "step": 328,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.5,
      "completions/mean_length": 572.40234375,
      "completions/min_length": 248.5,
      "epoch": 1.7762803234501348,
      "grad_norm": 3.1037305104431256,
      "kl": 5.15625,
      "learning_rate": 9.30924602179203e-07,
      "loss": 0.5578342080116272,
      "memory(GiB)": 75.26,
      "reward": 1.41270512342453,
      "reward_std": 0.7878355979919434,
      "rewards/Format/mean": 0.6953125,
      "rewards/Format/std": 0.4608239531517029,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.314609557390213,
      "rewards/ReflectionBalance/mean": 0.5728613436222076,
      "rewards/ReflectionBalance/std": 0.40082576870918274,
      "step": 329,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1041.5,
      "completions/mean_length": 572.18359375,
      "completions/min_length": 241.5,
      "epoch": 1.7816711590296497,
      "grad_norm": 7.057477063254281,
      "kl": 5.921875,
      "learning_rate": 9.30488876712812e-07,
      "loss": 0.5778623819351196,
      "memory(GiB)": 75.26,
      "reward": 1.514438509941101,
      "reward_std": 0.7924441397190094,
      "rewards/Format/mean": 0.72265625,
      "rewards/Format/std": 0.4492545574903488,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3978758305311203,
      "rewards/ReflectionBalance/mean": 0.5964697301387787,
      "rewards/ReflectionBalance/std": 0.3918752372264862,
      "step": 330,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 588.84765625,
      "completions/min_length": 250.0,
      "epoch": 1.7870619946091644,
      "grad_norm": 6.28049044670769,
      "kl": 6.109375,
      "learning_rate": 9.300518839305186e-07,
      "loss": 0.6130848526954651,
      "memory(GiB)": 75.26,
      "reward": 1.344419002532959,
      "reward_std": 0.8235597908496857,
      "rewards/Format/mean": 0.66796875,
      "rewards/Format/std": 0.4705667197704315,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3405093401670456,
      "rewards/ReflectionBalance/mean": 0.5436377227306366,
      "rewards/ReflectionBalance/std": 0.4075426757335663,
      "step": 331,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1048.5,
      "completions/mean_length": 587.48828125,
      "completions/min_length": 257.5,
      "epoch": 1.7924528301886793,
      "grad_norm": 3.530899163683089,
      "kl": 5.421875,
      "learning_rate": 9.296136251187852e-07,
      "loss": 0.584327220916748,
      "memory(GiB)": 75.26,
      "reward": 1.423507571220398,
      "reward_std": 0.8036566972732544,
      "rewards/Format/mean": 0.69140625,
      "rewards/Format/std": 0.46277445554733276,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.332206666469574,
      "rewards/ReflectionBalance/mean": 0.6031949818134308,
      "rewards/ReflectionBalance/std": 0.39496222138404846,
      "step": 332,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 529.91015625,
      "completions/min_length": 214.5,
      "epoch": 1.7978436657681942,
      "grad_norm": 4.443395435443405,
      "kl": 3.8828125,
      "learning_rate": 9.291741015678017e-07,
      "loss": 0.42702585458755493,
      "memory(GiB)": 75.26,
      "reward": 1.5478954315185547,
      "reward_std": 0.7150473594665527,
      "rewards/Format/mean": 0.765625,
      "rewards/Format/std": 0.421610951423645,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.333705835044384,
      "rewards/ReflectionBalance/mean": 0.6260205209255219,
      "rewards/ReflectionBalance/std": 0.36668381094932556,
      "step": 333,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 526.55078125,
      "completions/min_length": 245.0,
      "epoch": 1.803234501347709,
      "grad_norm": 5.4148874024404,
      "kl": 3.765625,
      "learning_rate": 9.287333145714811e-07,
      "loss": 0.4401835799217224,
      "memory(GiB)": 75.26,
      "reward": 1.6206933856010437,
      "reward_std": 0.7270466089248657,
      "rewards/Format/mean": 0.7734375,
      "rewards/Format/std": 0.4193098843097687,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.40383824706077576,
      "rewards/ReflectionBalance/mean": 0.6285058557987213,
      "rewards/ReflectionBalance/std": 0.3607649803161621,
      "step": 334,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 542.9375,
      "completions/min_length": 212.0,
      "epoch": 1.8086253369272236,
      "grad_norm": 3.906115152753543,
      "kl": 3.984375,
      "learning_rate": 9.282912654274561e-07,
      "loss": 0.4394581913948059,
      "memory(GiB)": 75.26,
      "reward": 1.4848828315734863,
      "reward_std": 0.6883958876132965,
      "rewards/Format/mean": 0.7421875,
      "rewards/Format/std": 0.4387826919555664,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.31812146306037903,
      "rewards/ReflectionBalance/mean": 0.6294140517711639,
      "rewards/ReflectionBalance/std": 0.37422285974025726,
      "step": 335,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.0,
      "completions/mean_length": 568.13671875,
      "completions/min_length": 232.5,
      "epoch": 1.8140161725067385,
      "grad_norm": 4.012215188280665,
      "kl": 4.0625,
      "learning_rate": 9.278479554370746e-07,
      "loss": 0.45112159848213196,
      "memory(GiB)": 75.26,
      "reward": 1.4392284750938416,
      "reward_std": 0.7476045489311218,
      "rewards/Format/mean": 0.7265625,
      "rewards/Format/std": 0.4443409889936447,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.32267695665359497,
      "rewards/ReflectionBalance/mean": 0.5954785346984863,
      "rewards/ReflectionBalance/std": 0.3850851356983185,
      "step": 336,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1036.0,
      "completions/mean_length": 521.84765625,
      "completions/min_length": 265.0,
      "epoch": 1.8194070080862534,
      "grad_norm": 4.156750793214093,
      "kl": 4.484375,
      "learning_rate": 9.274033859053969e-07,
      "loss": 0.5003170371055603,
      "memory(GiB)": 75.26,
      "reward": 1.6309033036231995,
      "reward_std": 0.7304441034793854,
      "rewards/Format/mean": 0.79296875,
      "rewards/Format/std": 0.4067413955926895,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3853980004787445,
      "rewards/ReflectionBalance/mean": 0.6504345834255219,
      "rewards/ReflectionBalance/std": 0.3639112710952759,
      "step": 337,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 573.703125,
      "completions/min_length": 276.0,
      "epoch": 1.8247978436657681,
      "grad_norm": 5.3780950464315085,
      "kl": 5.90625,
      "learning_rate": 9.269575581411908e-07,
      "loss": 0.6214892864227295,
      "memory(GiB)": 75.26,
      "reward": 1.456728458404541,
      "reward_std": 0.8090059459209442,
      "rewards/Format/mean": 0.6953125,
      "rewards/Format/std": 0.4570114016532898,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38864077627658844,
      "rewards/ReflectionBalance/mean": 0.5778222978115082,
      "rewards/ReflectionBalance/std": 0.3970756083726883,
      "step": 338,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 564.59375,
      "completions/min_length": 260.0,
      "epoch": 1.830188679245283,
      "grad_norm": 7.237579446863421,
      "kl": 6.203125,
      "learning_rate": 9.265104734569282e-07,
      "loss": 0.628562331199646,
      "memory(GiB)": 75.26,
      "reward": 1.5169092416763306,
      "reward_std": 0.8044013977050781,
      "rewards/Format/mean": 0.71875,
      "rewards/Format/std": 0.44445569813251495,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3851911425590515,
      "rewards/ReflectionBalance/mean": 0.6028466820716858,
      "rewards/ReflectionBalance/std": 0.3817102313041687,
      "step": 339,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 608.99609375,
      "completions/min_length": 259.0,
      "epoch": 1.835579514824798,
      "grad_norm": 11.531018337361758,
      "kl": 7.09375,
      "learning_rate": 9.260621331687817e-07,
      "loss": 0.6650748252868652,
      "memory(GiB)": 75.26,
      "reward": 1.3695361018180847,
      "reward_std": 0.8549993634223938,
      "rewards/Format/mean": 0.66015625,
      "rewards/Format/std": 0.47549907863140106,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3711828291416168,
      "rewards/ReflectionBalance/mean": 0.5453174114227295,
      "rewards/ReflectionBalance/std": 0.4088527709245682,
      "step": 340,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 568.640625,
      "completions/min_length": 215.0,
      "epoch": 1.8409703504043127,
      "grad_norm": 9.568941909103094,
      "kl": 6.15625,
      "learning_rate": 9.256125385966196e-07,
      "loss": 0.5805461406707764,
      "memory(GiB)": 75.26,
      "reward": 1.426235318183899,
      "reward_std": 0.7742754518985748,
      "rewards/Format/mean": 0.703125,
      "rewards/Format/std": 0.4585958868265152,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3542858511209488,
      "rewards/ReflectionBalance/mean": 0.5551416277885437,
      "rewards/ReflectionBalance/std": 0.39093054831027985,
      "step": 341,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 532.4609375,
      "completions/min_length": 244.5,
      "epoch": 1.8463611859838274,
      "grad_norm": 6.161374804235625,
      "kl": 4.3125,
      "learning_rate": 9.251616910640033e-07,
      "loss": 0.5080592632293701,
      "memory(GiB)": 75.26,
      "reward": 1.6589208841323853,
      "reward_std": 0.7670257687568665,
      "rewards/Format/mean": 0.79296875,
      "rewards/Format/std": 0.4065109044313431,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.40395987033843994,
      "rewards/ReflectionBalance/mean": 0.6550146639347076,
      "rewards/ReflectionBalance/std": 0.367234542965889,
      "step": 342,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 521.078125,
      "completions/min_length": 220.0,
      "epoch": 1.8517520215633423,
      "grad_norm": 4.903421522726265,
      "kl": 3.796875,
      "learning_rate": 9.247095918981824e-07,
      "loss": 0.43792498111724854,
      "memory(GiB)": 75.26,
      "reward": 1.5534667372703552,
      "reward_std": 0.6451294422149658,
      "rewards/Format/mean": 0.7890625,
      "rewards/Format/std": 0.40855903923511505,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.29547789692878723,
      "rewards/ReflectionBalance/mean": 0.6550292670726776,
      "rewards/ReflectionBalance/std": 0.36153191328048706,
      "step": 343,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 500.625,
      "completions/min_length": 191.5,
      "epoch": 1.8571428571428572,
      "grad_norm": 7.6411474955950665,
      "kl": 3.2890625,
      "learning_rate": 9.242562424300915e-07,
      "loss": 0.38391369581222534,
      "memory(GiB)": 75.26,
      "reward": 1.6600195169448853,
      "reward_std": 0.6173563897609711,
      "rewards/Format/mean": 0.80859375,
      "rewards/Format/std": 0.3877098113298416,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3853000998497009,
      "rewards/ReflectionBalance/mean": 0.6717382669448853,
      "rewards/ReflectionBalance/std": 0.3533540815114975,
      "step": 344,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.0,
      "completions/mean_length": 532.6953125,
      "completions/min_length": 262.5,
      "epoch": 1.862533692722372,
      "grad_norm": 4.789530733224015,
      "kl": 3.890625,
      "learning_rate": 9.238016439943455e-07,
      "loss": 0.46254390478134155,
      "memory(GiB)": 75.26,
      "reward": 1.6605908870697021,
      "reward_std": 0.7385529279708862,
      "rewards/Format/mean": 0.8046875,
      "rewards/Format/std": 0.3978758305311203,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.40095221996307373,
      "rewards/ReflectionBalance/mean": 0.6566845774650574,
      "rewards/ReflectionBalance/std": 0.37481844425201416,
      "step": 345,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.0,
      "completions/mean_length": 528.16796875,
      "completions/min_length": 149.5,
      "epoch": 1.8679245283018868,
      "grad_norm": 5.766238129238041,
      "kl": 3.828125,
      "learning_rate": 9.233457979292368e-07,
      "loss": 0.44946596026420593,
      "memory(GiB)": 75.26,
      "reward": 1.51318359375,
      "reward_std": 0.680586040019989,
      "rewards/Format/mean": 0.77734375,
      "rewards/Format/std": 0.4115816503763199,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.28456588089466095,
      "rewards/ReflectionBalance/mean": 0.63818359375,
      "rewards/ReflectionBalance/std": 0.3552539050579071,
      "step": 346,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 573.296875,
      "completions/min_length": 238.5,
      "epoch": 1.8733153638814017,
      "grad_norm": 3.3289805800330474,
      "kl": 5.5625,
      "learning_rate": 9.228887055767298e-07,
      "loss": 0.5903763771057129,
      "memory(GiB)": 75.26,
      "reward": 1.4276806116104126,
      "reward_std": 0.8187748789787292,
      "rewards/Format/mean": 0.7109375,
      "rewards/Format/std": 0.4550260454416275,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.30669571459293365,
      "rewards/ReflectionBalance/mean": 0.6112744212150574,
      "rewards/ReflectionBalance/std": 0.3963021785020828,
      "step": 347,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 554.35546875,
      "completions/min_length": 232.5,
      "epoch": 1.8787061994609164,
      "grad_norm": 13.333490133741718,
      "kl": 6.8125,
      "learning_rate": 9.224303682824587e-07,
      "loss": 0.6191086769104004,
      "memory(GiB)": 75.26,
      "reward": 1.4561767578125,
      "reward_std": 0.6624182462692261,
      "rewards/Format/mean": 0.71484375,
      "rewards/Format/std": 0.4530755430459976,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3425614684820175,
      "rewards/ReflectionBalance/mean": 0.6046142578125,
      "rewards/ReflectionBalance/std": 0.38214559853076935,
      "step": 348,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 565.61328125,
      "completions/min_length": 134.0,
      "epoch": 1.8840970350404311,
      "grad_norm": 10.346269377986928,
      "kl": 6.6875,
      "learning_rate": 9.219707873957223e-07,
      "loss": 0.6537829041481018,
      "memory(GiB)": 75.26,
      "reward": 1.44300776720047,
      "reward_std": 0.8128722906112671,
      "rewards/Format/mean": 0.71484375,
      "rewards/Format/std": 0.45274218916893005,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3407054841518402,
      "rewards/ReflectionBalance/mean": 0.5953515470027924,
      "rewards/ReflectionBalance/std": 0.3870077431201935,
      "step": 349,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.0,
      "completions/mean_length": 539.28125,
      "completions/min_length": 270.0,
      "epoch": 1.889487870619946,
      "grad_norm": 3.368811160392505,
      "kl": 5.8125,
      "learning_rate": 9.215099642694804e-07,
      "loss": 0.6231478452682495,
      "memory(GiB)": 75.26,
      "reward": 1.5306689739227295,
      "reward_std": 0.7890586853027344,
      "rewards/Format/mean": 0.76953125,
      "rewards/Format/std": 0.422762006521225,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.34485287964344025,
      "rewards/ReflectionBalance/mean": 0.6244189441204071,
      "rewards/ReflectionBalance/std": 0.3721328675746918,
      "step": 350,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1038.0,
      "completions/mean_length": 503.0703125,
      "completions/min_length": 224.5,
      "epoch": 1.894878706199461,
      "grad_norm": 5.228585554735936,
      "kl": 4.4140625,
      "learning_rate": 9.210479002603499e-07,
      "loss": 0.4353111982345581,
      "memory(GiB)": 75.26,
      "reward": 1.5937793254852295,
      "reward_std": 0.6526550054550171,
      "rewards/Format/mean": 0.7890625,
      "rewards/Format/std": 0.4094643145799637,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35538628697395325,
      "rewards/ReflectionBalance/mean": 0.6562792956829071,
      "rewards/ReflectionBalance/std": 0.37628065049648285,
      "step": 351,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 519.19140625,
      "completions/min_length": 240.0,
      "epoch": 1.9002695417789757,
      "grad_norm": 3.887080885082733,
      "kl": 3.6796875,
      "learning_rate": 9.205845967286007e-07,
      "loss": 0.4045605957508087,
      "memory(GiB)": 75.26,
      "reward": 1.6265233755111694,
      "reward_std": 0.6798125803470612,
      "rewards/Format/mean": 0.8203125,
      "rewards/Format/std": 0.3842122554779053,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.32746291160583496,
      "rewards/ReflectionBalance/mean": 0.6851172149181366,
      "rewards/ReflectionBalance/std": 0.33769896626472473,
      "step": 352,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 549.75390625,
      "completions/min_length": 249.5,
      "epoch": 1.9056603773584906,
      "grad_norm": 7.487958010954536,
      "kl": 4.046875,
      "learning_rate": 9.20120055038152e-07,
      "loss": 0.3910409212112427,
      "memory(GiB)": 75.26,
      "reward": 1.7121614813804626,
      "reward_std": 0.6435042917728424,
      "rewards/Format/mean": 0.80078125,
      "rewards/Format/std": 0.4007115960121155,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4264890104532242,
      "rewards/ReflectionBalance/mean": 0.6730989515781403,
      "rewards/ReflectionBalance/std": 0.347442165017128,
      "step": 353,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 493.359375,
      "completions/min_length": 240.0,
      "epoch": 1.9110512129380055,
      "grad_norm": 5.923605391048771,
      "kl": 3.390625,
      "learning_rate": 9.196542765565674e-07,
      "loss": 0.40861427783966064,
      "memory(GiB)": 75.26,
      "reward": 1.8167692422866821,
      "reward_std": 0.6493620574474335,
      "rewards/Format/mean": 0.8359375,
      "rewards/Format/std": 0.3716365247964859,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.44712863862514496,
      "rewards/ReflectionBalance/mean": 0.7073942124843597,
      "rewards/ReflectionBalance/std": 0.31542976200580597,
      "step": 354,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 522.21484375,
      "completions/min_length": 236.0,
      "epoch": 1.9164420485175202,
      "grad_norm": 3.978992061148853,
      "kl": 2.75,
      "learning_rate": 9.191872626550519e-07,
      "loss": 0.302203506231308,
      "memory(GiB)": 75.26,
      "reward": 1.6880094408988953,
      "reward_std": 0.5583296418190002,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.35358233749866486,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.33655155450105667,
      "rewards/ReflectionBalance/mean": 0.6880094408988953,
      "rewards/ReflectionBalance/std": 0.342841312289238,
      "step": 355,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 526.03125,
      "completions/min_length": 253.0,
      "epoch": 1.921832884097035,
      "grad_norm": 3.4400467650521787,
      "kl": 4.4609375,
      "learning_rate": 9.187190147084474e-07,
      "loss": 0.5041325688362122,
      "memory(GiB)": 75.26,
      "reward": 1.5482860803604126,
      "reward_std": 0.6687915623188019,
      "rewards/Format/mean": 0.8125,
      "rewards/Format/std": 0.38713616132736206,
      "rewards/MultiModalAccuracyORM/mean": 0.0703125,
      "rewards/MultiModalAccuracyORM/std": 0.2562176361680031,
      "rewards/ReflectionBalance/mean": 0.6654736399650574,
      "rewards/ReflectionBalance/std": 0.3580278158187866,
      "step": 356,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 489.33984375,
      "completions/min_length": 212.0,
      "epoch": 1.9272237196765498,
      "grad_norm": 4.843499521030545,
      "kl": 2.7578125,
      "learning_rate": 9.182495340952288e-07,
      "loss": 0.3467782735824585,
      "memory(GiB)": 75.26,
      "reward": 1.81639164686203,
      "reward_std": 0.6357516944408417,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.33116789162158966,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.37486983835697174,
      "rewards/ReflectionBalance/mean": 0.7031103372573853,
      "rewards/ReflectionBalance/std": 0.3144346922636032,
      "step": 357,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 473.73828125,
      "completions/min_length": 236.5,
      "epoch": 1.9326145552560647,
      "grad_norm": 4.405544918526112,
      "kl": 3.6015625,
      "learning_rate": 9.177788221974997e-07,
      "loss": 0.37925398349761963,
      "memory(GiB)": 75.26,
      "reward": 1.8480273485183716,
      "reward_std": 0.5908013582229614,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.344347283244133,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.4412749409675598,
      "rewards/ReflectionBalance/mean": 0.7269335687160492,
      "rewards/ReflectionBalance/std": 0.30230405926704407,
      "step": 358,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 533.359375,
      "completions/min_length": 243.5,
      "epoch": 1.9380053908355794,
      "grad_norm": 7.958597279206591,
      "kl": 5.453125,
      "learning_rate": 9.173068804009885e-07,
      "loss": 0.5223419666290283,
      "memory(GiB)": 75.26,
      "reward": 1.6404247879981995,
      "reward_std": 0.6920540630817413,
      "rewards/Format/mean": 0.8046875,
      "rewards/Format/std": 0.3948954641819,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3542858511209488,
      "rewards/ReflectionBalance/mean": 0.6677685678005219,
      "rewards/ReflectionBalance/std": 0.3532475233078003,
      "step": 359,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 489.1015625,
      "completions/min_length": 232.0,
      "epoch": 1.9433962264150944,
      "grad_norm": 8.329161223900519,
      "kl": 5.296875,
      "learning_rate": 9.168337100950446e-07,
      "loss": 0.4731750786304474,
      "memory(GiB)": 75.26,
      "reward": 1.7556428909301758,
      "reward_std": 0.678715169429779,
      "rewards/Format/mean": 0.828125,
      "rewards/Format/std": 0.3786114752292633,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.4301380217075348,
      "rewards/ReflectionBalance/mean": 0.6736116707324982,
      "rewards/ReflectionBalance/std": 0.3416251093149185,
      "step": 360,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 501.70703125,
      "completions/min_length": 254.0,
      "epoch": 1.9487870619946093,
      "grad_norm": 5.032658552267099,
      "kl": 4.890625,
      "learning_rate": 9.163593126726332e-07,
      "loss": 0.4887440502643585,
      "memory(GiB)": 75.26,
      "reward": 1.6862255334854126,
      "reward_std": 0.6596717536449432,
      "rewards/Format/mean": 0.828125,
      "rewards/Format/std": 0.3774644732475281,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.38181672990322113,
      "rewards/ReflectionBalance/mean": 0.6823193728923798,
      "rewards/ReflectionBalance/std": 0.34756651520729065,
      "step": 361,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 467.859375,
      "completions/min_length": 234.5,
      "epoch": 1.954177897574124,
      "grad_norm": 3.51572650999052,
      "kl": 3.1484375,
      "learning_rate": 9.158836895303331e-07,
      "loss": 0.34826934337615967,
      "memory(GiB)": 75.26,
      "reward": 1.7408788800239563,
      "reward_std": 0.5721539258956909,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31518664956092834,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.31763872504234314,
      "rewards/ReflectionBalance/mean": 0.7408789396286011,
      "rewards/ReflectionBalance/std": 0.28198830783367157,
      "step": 362,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 449.640625,
      "completions/min_length": 229.0,
      "epoch": 1.9595687331536387,
      "grad_norm": 3.216329066649463,
      "kl": 2.33984375,
      "learning_rate": 9.154068420683305e-07,
      "loss": 0.24754638969898224,
      "memory(GiB)": 75.26,
      "reward": 1.8113476634025574,
      "reward_std": 0.49682946503162384,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27079907804727554,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36045311391353607,
      "rewards/ReflectionBalance/mean": 0.7371289432048798,
      "rewards/ReflectionBalance/std": 0.27246706187725067,
      "step": 363,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 492.8359375,
      "completions/min_length": 245.5,
      "epoch": 1.9649595687331538,
      "grad_norm": 3.2378310823282854,
      "kl": 3.5546875,
      "learning_rate": 9.149287716904163e-07,
      "loss": 0.383732408285141,
      "memory(GiB)": 75.26,
      "reward": 1.7261279821395874,
      "reward_std": 0.6082166731357574,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.3410066068172455,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3597298264503479,
      "rewards/ReflectionBalance/mean": 0.7105029225349426,
      "rewards/ReflectionBalance/std": 0.31618331372737885,
      "step": 364,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 455.7265625,
      "completions/min_length": 233.0,
      "epoch": 1.9703504043126685,
      "grad_norm": 3.5907246667761763,
      "kl": 3.640625,
      "learning_rate": 9.144494798039821e-07,
      "loss": 0.35708558559417725,
      "memory(GiB)": 75.26,
      "reward": 1.7125048637390137,
      "reward_std": 0.5460535287857056,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3208130896091461,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.30072054266929626,
      "rewards/ReflectionBalance/mean": 0.7281298637390137,
      "rewards/ReflectionBalance/std": 0.2982104420661926,
      "step": 365,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 513.17578125,
      "completions/min_length": 227.5,
      "epoch": 1.9757412398921832,
      "grad_norm": 3.7324061971022062,
      "kl": 4.21875,
      "learning_rate": 9.139689678200144e-07,
      "loss": 0.4218708574771881,
      "memory(GiB)": 75.26,
      "reward": 1.6612303853034973,
      "reward_std": 0.6183783113956451,
      "rewards/Format/mean": 0.8203125,
      "rewards/Format/std": 0.38489311933517456,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3752707988023758,
      "rewards/ReflectionBalance/mean": 0.6729492247104645,
      "rewards/ReflectionBalance/std": 0.3286963403224945,
      "step": 366,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 488.04296875,
      "completions/min_length": 252.0,
      "epoch": 1.9811320754716981,
      "grad_norm": 3.1964273148121656,
      "kl": 3.4140625,
      "learning_rate": 9.134872371530923e-07,
      "loss": 0.3282436728477478,
      "memory(GiB)": 75.26,
      "reward": 1.7927213907241821,
      "reward_std": 0.5672342479228973,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3934052139520645,
      "rewards/ReflectionBalance/mean": 0.7145963907241821,
      "rewards/ReflectionBalance/std": 0.3234064429998398,
      "step": 367,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 517.62109375,
      "completions/min_length": 259.0,
      "epoch": 1.986522911051213,
      "grad_norm": 3.4601088199215084,
      "kl": 3.796875,
      "learning_rate": 9.130042892213823e-07,
      "loss": 0.38234958052635193,
      "memory(GiB)": 75.26,
      "reward": 1.6008984446525574,
      "reward_std": 0.5578293204307556,
      "rewards/Format/mean": 0.8359375,
      "rewards/Format/std": 0.3679620176553726,
      "rewards/MultiModalAccuracyORM/mean": 0.06640625,
      "rewards/MultiModalAccuracyORM/std": 0.24678732454776764,
      "rewards/ReflectionBalance/mean": 0.698554664850235,
      "rewards/ReflectionBalance/std": 0.3327695280313492,
      "step": 368,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.0,
      "completions/mean_length": 490.3359375,
      "completions/min_length": 251.5,
      "epoch": 1.9919137466307277,
      "grad_norm": 4.855822563201375,
      "kl": 2.984375,
      "learning_rate": 9.125201254466345e-07,
      "loss": 0.34515318274497986,
      "memory(GiB)": 75.26,
      "reward": 1.6900194883346558,
      "reward_std": 0.6091122031211853,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33636194467544556,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3334042727947235,
      "rewards/ReflectionBalance/mean": 0.6861132979393005,
      "rewards/ReflectionBalance/std": 0.31952889263629913,
      "step": 369,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1033.0,
      "completions/mean_length": 476.7578125,
      "completions/min_length": 226.0,
      "epoch": 1.9973045822102424,
      "grad_norm": 3.7434352248651006,
      "kl": 3.04296875,
      "learning_rate": 9.120347472541783e-07,
      "loss": 0.32202112674713135,
      "memory(GiB)": 75.26,
      "reward": 1.7944335341453552,
      "reward_std": 0.6313987970352173,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3142324984073639,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.423653244972229,
      "rewards/ReflectionBalance/mean": 0.6772460639476776,
      "rewards/ReflectionBalance/std": 0.34226706624031067,
      "step": 370,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 437.74609375,
      "completions/min_length": 231.5,
      "epoch": 2.0053908355795147,
      "grad_norm": 2.8181245335853085,
      "kl": 2.01953125,
      "learning_rate": 9.115481560729184e-07,
      "loss": 0.22363805770874023,
      "memory(GiB)": 75.26,
      "reward": 1.9016748666763306,
      "reward_std": 0.4739510267972946,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4018339514732361,
      "rewards/ReflectionBalance/mean": 0.768862247467041,
      "rewards/ReflectionBalance/std": 0.2677967697381973,
      "step": 371,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 472.51953125,
      "completions/min_length": 237.5,
      "epoch": 2.01078167115903,
      "grad_norm": 5.559124172258777,
      "kl": 1.71875,
      "learning_rate": 9.110603533353301e-07,
      "loss": 0.2217075228691101,
      "memory(GiB)": 75.26,
      "reward": 1.934790015220642,
      "reward_std": 0.5435399413108826,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4225567579269409,
      "rewards/ReflectionBalance/mean": 0.7824462950229645,
      "rewards/ReflectionBalance/std": 0.25938913971185684,
      "step": 372,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 497.5703125,
      "completions/min_length": 257.0,
      "epoch": 2.0161725067385445,
      "grad_norm": 3.4904451799734786,
      "kl": 1.765625,
      "learning_rate": 9.105713404774553e-07,
      "loss": 0.1863100826740265,
      "memory(GiB)": 75.26,
      "reward": 1.936279296875,
      "reward_std": 0.601345419883728,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.4387826919555664,
      "rewards/ReflectionBalance/mean": 0.772216796875,
      "rewards/ReflectionBalance/std": 0.2667642682790756,
      "step": 373,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.0,
      "completions/mean_length": 477.1328125,
      "completions/min_length": 232.0,
      "epoch": 2.0215633423180592,
      "grad_norm": 3.9060038933057224,
      "kl": 2.828125,
      "learning_rate": 9.10081118938899e-07,
      "loss": 0.33439314365386963,
      "memory(GiB)": 75.26,
      "reward": 1.7473242282867432,
      "reward_std": 0.579507514834404,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3208130896091461,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3456958532333374,
      "rewards/ReflectionBalance/mean": 0.7043554484844208,
      "rewards/ReflectionBalance/std": 0.31275521218776703,
      "step": 374,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 470.15234375,
      "completions/min_length": 239.5,
      "epoch": 2.026954177897574,
      "grad_norm": 4.434985224266184,
      "kl": 3.734375,
      "learning_rate": 9.095896901628237e-07,
      "loss": 0.38865774869918823,
      "memory(GiB)": 75.26,
      "reward": 1.7593066096305847,
      "reward_std": 0.5500341355800629,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.35296520590782166,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38489311933517456,
      "rewards/ReflectionBalance/mean": 0.7241503894329071,
      "rewards/ReflectionBalance/std": 0.30757877230644226,
      "step": 375,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 531.078125,
      "completions/min_length": 241.5,
      "epoch": 2.032345013477089,
      "grad_norm": 6.461574651269091,
      "kl": 6.046875,
      "learning_rate": 9.090970555959466e-07,
      "loss": 0.5922461748123169,
      "memory(GiB)": 75.26,
      "reward": 1.5763232111930847,
      "reward_std": 0.7333405017852783,
      "rewards/Format/mean": 0.7734375,
      "rewards/Format/std": 0.41983383893966675,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3820948451757431,
      "rewards/ReflectionBalance/mean": 0.6271044909954071,
      "rewards/ReflectionBalance/std": 0.37627112865448,
      "step": 376,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 514.3359375,
      "completions/min_length": 276.5,
      "epoch": 2.0377358490566038,
      "grad_norm": 5.018651662836702,
      "kl": 4.765625,
      "learning_rate": 9.086032166885339e-07,
      "loss": 0.4788884222507477,
      "memory(GiB)": 75.26,
      "reward": 1.7390901446342468,
      "reward_std": 0.7162266671657562,
      "rewards/Format/mean": 0.828125,
      "rewards/Format/std": 0.3774644732475281,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4243630915880203,
      "rewards/ReflectionBalance/mean": 0.6765901744365692,
      "rewards/ReflectionBalance/std": 0.3746592551469803,
      "step": 377,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 503.07421875,
      "completions/min_length": 247.5,
      "epoch": 2.0431266846361185,
      "grad_norm": 6.173591854646191,
      "kl": 5.671875,
      "learning_rate": 9.081081748943977e-07,
      "loss": 0.554438591003418,
      "memory(GiB)": 75.26,
      "reward": 1.6239355206489563,
      "reward_std": 0.7428691685199738,
      "rewards/Format/mean": 0.80859375,
      "rewards/Format/std": 0.3877098113298416,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.38125884532928467,
      "rewards/ReflectionBalance/mean": 0.6395605206489563,
      "rewards/ReflectionBalance/std": 0.3998623490333557,
      "step": 378,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1036.5,
      "completions/mean_length": 527.609375,
      "completions/min_length": 244.5,
      "epoch": 2.0485175202156336,
      "grad_norm": 6.602137940759217,
      "kl": 5.640625,
      "learning_rate": 9.076119316708915e-07,
      "loss": 0.539423942565918,
      "memory(GiB)": 75.26,
      "reward": 1.5934717655181885,
      "reward_std": 0.776578962802887,
      "rewards/Format/mean": 0.78125,
      "rewards/Format/std": 0.41404466331005096,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3704231530427933,
      "rewards/ReflectionBalance/mean": 0.6481591463088989,
      "rewards/ReflectionBalance/std": 0.3942657709121704,
      "step": 379,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 444.15234375,
      "completions/min_length": 227.5,
      "epoch": 2.0539083557951483,
      "grad_norm": 2.891855662463163,
      "kl": 2.5390625,
      "learning_rate": 9.07114488478905e-07,
      "loss": 0.25266462564468384,
      "memory(GiB)": 75.26,
      "reward": 1.8354443907737732,
      "reward_std": 0.5009027123451233,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3707977831363678,
      "rewards/ReflectionBalance/mean": 0.7495068311691284,
      "rewards/ReflectionBalance/std": 0.2899177819490433,
      "step": 380,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 505.55078125,
      "completions/min_length": 259.0,
      "epoch": 2.059299191374663,
      "grad_norm": 5.491733587112358,
      "kl": 2.9609375,
      "learning_rate": 9.066158467828611e-07,
      "loss": 0.380994588136673,
      "memory(GiB)": 75.26,
      "reward": 1.7756397128105164,
      "reward_std": 0.6049402952194214,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.322907418012619,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38704802095890045,
      "rewards/ReflectionBalance/mean": 0.7092334032058716,
      "rewards/ReflectionBalance/std": 0.32826146483421326,
      "step": 381,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 468.6640625,
      "completions/min_length": 221.5,
      "epoch": 2.0646900269541777,
      "grad_norm": 4.653399962359109,
      "kl": 1.84765625,
      "learning_rate": 9.061160080507104e-07,
      "loss": 0.2180531620979309,
      "memory(GiB)": 75.26,
      "reward": 1.8461865782737732,
      "reward_std": 0.4914207458496094,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3711828291416168,
      "rewards/ReflectionBalance/mean": 0.7641552686691284,
      "rewards/ReflectionBalance/std": 0.27528680860996246,
      "step": 382,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 467.16015625,
      "completions/min_length": 268.5,
      "epoch": 2.070080862533693,
      "grad_norm": 3.9404868881045245,
      "kl": 2.4375,
      "learning_rate": 9.056149737539282e-07,
      "loss": 0.2829543948173523,
      "memory(GiB)": 75.26,
      "reward": 1.8569775819778442,
      "reward_std": 0.539720207452774,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.7554150521755219,
      "rewards/ReflectionBalance/std": 0.28241507709026337,
      "step": 383,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 437.80078125,
      "completions/min_length": 257.0,
      "epoch": 2.0754716981132075,
      "grad_norm": 4.370506726198907,
      "kl": 1.72265625,
      "learning_rate": 9.05112745367509e-07,
      "loss": 0.22078900039196014,
      "memory(GiB)": 75.26,
      "reward": 2.0182374119758606,
      "reward_std": 0.4940738379955292,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.31640625,
      "rewards/MultiModalAccuracyORM/std": 0.4599490314722061,
      "rewards/ReflectionBalance/mean": 0.7682373225688934,
      "rewards/ReflectionBalance/std": 0.2604615315794945,
      "step": 384,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 489.01953125,
      "completions/min_length": 278.0,
      "epoch": 2.0808625336927222,
      "grad_norm": 4.122236560375647,
      "kl": 3.34375,
      "learning_rate": 9.046093243699622e-07,
      "loss": 0.38982969522476196,
      "memory(GiB)": 75.26,
      "reward": 1.7892577648162842,
      "reward_std": 0.5497367680072784,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3208130896091461,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3663744479417801,
      "rewards/ReflectionBalance/mean": 0.722851574420929,
      "rewards/ReflectionBalance/std": 0.32691772282123566,
      "step": 385,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.0,
      "completions/mean_length": 494.86328125,
      "completions/min_length": 256.0,
      "epoch": 2.0862533692722374,
      "grad_norm": 3.4073433020655606,
      "kl": 3.15625,
      "learning_rate": 9.041047122433088e-07,
      "loss": 0.3274180293083191,
      "memory(GiB)": 75.26,
      "reward": 1.7212060689926147,
      "reward_std": 0.5383502244949341,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3247780203819275,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3405093401670456,
      "rewards/ReflectionBalance/mean": 0.7094872891902924,
      "rewards/ReflectionBalance/std": 0.32306456565856934,
      "step": 386,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 557.04296875,
      "completions/min_length": 257.0,
      "epoch": 2.091644204851752,
      "grad_norm": 4.509682088912154,
      "kl": 5.109375,
      "learning_rate": 9.035989104730761e-07,
      "loss": 0.509506106376648,
      "memory(GiB)": 75.26,
      "reward": 1.5545198321342468,
      "reward_std": 0.6547039747238159,
      "rewards/Format/mean": 0.79296875,
      "rewards/Format/std": 0.4065109044313431,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3068038523197174,
      "rewards/ReflectionBalance/mean": 0.6521761417388916,
      "rewards/ReflectionBalance/std": 0.36162763833999634,
      "step": 387,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 486.51953125,
      "completions/min_length": 228.5,
      "epoch": 2.0970350404312668,
      "grad_norm": 4.840583578975698,
      "kl": 3.7578125,
      "learning_rate": 9.030919205482936e-07,
      "loss": 0.39021003246307373,
      "memory(GiB)": 75.26,
      "reward": 1.7184375524520874,
      "reward_std": 0.5623653531074524,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3300945907831192,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.33312736451625824,
      "rewards/ReflectionBalance/mean": 0.702812522649765,
      "rewards/ReflectionBalance/std": 0.32747358083724976,
      "step": 388,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 496.46484375,
      "completions/min_length": 242.5,
      "epoch": 2.1024258760107815,
      "grad_norm": 4.008785175890138,
      "kl": 3.9375,
      "learning_rate": 9.025837439614885e-07,
      "loss": 0.40025341510772705,
      "memory(GiB)": 75.26,
      "reward": 1.6655598878860474,
      "reward_std": 0.5810654759407043,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.34826864302158356,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.31333550810813904,
      "rewards/ReflectionBalance/mean": 0.6968099176883698,
      "rewards/ReflectionBalance/std": 0.3415657728910446,
      "step": 389,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 512.40234375,
      "completions/min_length": 260.0,
      "epoch": 2.1078167115902966,
      "grad_norm": 3.3295685894567195,
      "kl": 4.78125,
      "learning_rate": 9.020743822086814e-07,
      "loss": 0.47291120886802673,
      "memory(GiB)": 75.26,
      "reward": 1.7330371141433716,
      "reward_std": 0.7239001393318176,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.3619322180747986,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.396040678024292,
      "rewards/ReflectionBalance/mean": 0.6822558641433716,
      "rewards/ReflectionBalance/std": 0.3559093624353409,
      "step": 390,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 467.21875,
      "completions/min_length": 245.0,
      "epoch": 2.1132075471698113,
      "grad_norm": 4.448852268849644,
      "kl": 4.0234375,
      "learning_rate": 9.015638367893823e-07,
      "loss": 0.39216184616088867,
      "memory(GiB)": 75.26,
      "reward": 1.7641942501068115,
      "reward_std": 0.557368814945221,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.344347283244133,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38784652948379517,
      "rewards/ReflectionBalance/mean": 0.7212255895137787,
      "rewards/ReflectionBalance/std": 0.3067396432161331,
      "step": 391,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 503.015625,
      "completions/min_length": 159.5,
      "epoch": 2.118598382749326,
      "grad_norm": 5.9719645119790075,
      "kl": 4.2265625,
      "learning_rate": 9.010521092065853e-07,
      "loss": 0.4485158920288086,
      "memory(GiB)": 75.26,
      "reward": 1.6846093535423279,
      "reward_std": 0.6139720678329468,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.365008607506752,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.2948496490716934,
      "rewards/ReflectionBalance/mean": 0.7197656333446503,
      "rewards/ReflectionBalance/std": 0.31921395659446716,
      "step": 392,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 465.3515625,
      "completions/min_length": 240.0,
      "epoch": 2.123989218328841,
      "grad_norm": 4.241090571944151,
      "kl": 3.1015625,
      "learning_rate": 9.005392009667648e-07,
      "loss": 0.34343254566192627,
      "memory(GiB)": 75.26,
      "reward": 1.864990234375,
      "reward_std": 0.5428619980812073,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30283892154693604,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.42273683845996857,
      "rewards/ReflectionBalance/mean": 0.736083984375,
      "rewards/ReflectionBalance/std": 0.29509133845567703,
      "step": 393,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 454.765625,
      "completions/min_length": 248.5,
      "epoch": 2.129380053908356,
      "grad_norm": 3.73635855501685,
      "kl": 3.203125,
      "learning_rate": 9.000251135798711e-07,
      "loss": 0.32516545057296753,
      "memory(GiB)": 75.26,
      "reward": 1.7648340463638306,
      "reward_std": 0.5524336695671082,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.33116789162158966,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.35641810297966003,
      "rewards/ReflectionBalance/mean": 0.7335839867591858,
      "rewards/ReflectionBalance/std": 0.2851985841989517,
      "step": 394,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.5,
      "completions/mean_length": 486.08203125,
      "completions/min_length": 242.5,
      "epoch": 2.1347708894878705,
      "grad_norm": 3.976041101437103,
      "kl": 2.6953125,
      "learning_rate": 8.995098485593257e-07,
      "loss": 0.3075490891933441,
      "memory(GiB)": 75.26,
      "reward": 1.763891577720642,
      "reward_std": 0.5564077496528625,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3265802264213562,
      "rewards/ReflectionBalance/mean": 0.7443603277206421,
      "rewards/ReflectionBalance/std": 0.28967365622520447,
      "step": 395,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 533.50390625,
      "completions/min_length": 263.5,
      "epoch": 2.1401617250673857,
      "grad_norm": 3.606820176369913,
      "kl": 4.1640625,
      "learning_rate": 8.989934074220169e-07,
      "loss": 0.405226469039917,
      "memory(GiB)": 75.26,
      "reward": 1.6436946392059326,
      "reward_std": 0.676225334405899,
      "rewards/Format/mean": 0.8203125,
      "rewards/Format/std": 0.38047872483730316,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3645188808441162,
      "rewards/ReflectionBalance/mean": 0.6671321392059326,
      "rewards/ReflectionBalance/std": 0.3449837416410446,
      "step": 396,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 496.7578125,
      "completions/min_length": 239.0,
      "epoch": 2.1455525606469004,
      "grad_norm": 3.6042627533671765,
      "kl": 3.828125,
      "learning_rate": 8.984757916882956e-07,
      "loss": 0.4187498688697815,
      "memory(GiB)": 75.26,
      "reward": 1.7576025128364563,
      "reward_std": 0.5804620683193207,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.35259076952934265,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.35643893480300903,
      "rewards/ReflectionBalance/mean": 0.7458837926387787,
      "rewards/ReflectionBalance/std": 0.30726616084575653,
      "step": 397,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.5,
      "completions/mean_length": 497.28515625,
      "completions/min_length": 199.0,
      "epoch": 2.150943396226415,
      "grad_norm": 5.487313467791917,
      "kl": 3.7421875,
      "learning_rate": 8.979570028819701e-07,
      "loss": 0.40150460600852966,
      "memory(GiB)": 75.26,
      "reward": 1.7253418564796448,
      "reward_std": 0.6350407600402832,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.339918777346611,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.34826576709747314,
      "rewards/ReflectionBalance/mean": 0.709716796875,
      "rewards/ReflectionBalance/std": 0.3425711393356323,
      "step": 398,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 505.24609375,
      "completions/min_length": 255.5,
      "epoch": 2.1563342318059298,
      "grad_norm": 3.8492715380835603,
      "kl": 4.734375,
      "learning_rate": 8.974370425303025e-07,
      "loss": 0.5122182965278625,
      "memory(GiB)": 75.26,
      "reward": 1.6952636241912842,
      "reward_std": 0.720331609249115,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.3678421527147293,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36935214698314667,
      "rewards/ReflectionBalance/mean": 0.6913574039936066,
      "rewards/ReflectionBalance/std": 0.35113370418548584,
      "step": 399,
      "train_speed(iter/s)": 0.007113
    },
    {
      "epoch": 2.161725067385445,
      "grad_norm": 7.589013248409197,
      "learning_rate": 8.969159121640039e-07,
      "loss": 0.4750770926475525,
      "memory(GiB)": 75.26,
      "step": 400,
      "train_speed(iter/s)": 0.007113
    },
    {
      "epoch": 2.161725067385445,
      "eval_clip_ratio": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 1028.25,
      "eval_completions/mean_length": 519.9921875,
      "eval_completions/min_length": 213.75,
      "eval_kl": 4.375,
      "eval_loss": 0.43333563208580017,
      "eval_reward": 1.6223217248916626,
      "eval_reward_std": 0.6451911479234695,
      "eval_rewards/Format/mean": 0.833984375,
      "eval_rewards/Format/std": 0.3729860335588455,
      "eval_rewards/MultiModalAccuracyORM/mean": 0.107421875,
      "eval_rewards/MultiModalAccuracyORM/std": 0.3046310432255268,
      "eval_rewards/ReflectionBalance/mean": 0.6809155344963074,
      "eval_rewards/ReflectionBalance/std": 0.34763088822364807,
      "eval_runtime": 251.018,
      "eval_samples_per_second": 0.235,
      "eval_steps_per_second": 0.004,
      "step": 400
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.25,
      "completions/mean_length": 484.6171875,
      "completions/min_length": 210.5,
      "epoch": 2.1671159029649596,
      "grad_norm": 2.9875895715858554,
      "kl": 4.17578125,
      "learning_rate": 8.963936133172294e-07,
      "loss": 0.3308410942554474,
      "memory(GiB)": 75.26,
      "reward": 1.710540384054184,
      "reward_std": 0.591062530875206,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.34505700320005417,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35185685753822327,
      "rewards/ReflectionBalance/mean": 0.7027278691530228,
      "rewards/ReflectionBalance/std": 0.31981057673692703,
      "step": 401,
      "train_speed(iter/s)": 0.007077
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 481.81640625,
      "completions/min_length": 229.5,
      "epoch": 2.1725067385444743,
      "grad_norm": 3.111187892148355,
      "kl": 4.09375,
      "learning_rate": 8.958701475275743e-07,
      "loss": 0.42406558990478516,
      "memory(GiB)": 75.26,
      "reward": 1.7405273914337158,
      "reward_std": 0.5830713510513306,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3405093401670456,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3526121824979782,
      "rewards/ReflectionBalance/mean": 0.7288086116313934,
      "rewards/ReflectionBalance/std": 0.3076355457305908,
      "step": 402,
      "train_speed(iter/s)": 0.007077
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 496.90625,
      "completions/min_length": 247.5,
      "epoch": 2.177897574123989,
      "grad_norm": 3.896592916797396,
      "kl": 4.0703125,
      "learning_rate": 8.953455163360691e-07,
      "loss": 0.391851544380188,
      "memory(GiB)": 75.26,
      "reward": 1.7389110326766968,
      "reward_std": 0.5942672193050385,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.34263932704925537,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3597298264503479,
      "rewards/ReflectionBalance/mean": 0.7310986220836639,
      "rewards/ReflectionBalance/std": 0.3055131882429123,
      "step": 403,
      "train_speed(iter/s)": 0.007077
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 477.1875,
      "completions/min_length": 258.0,
      "epoch": 2.183288409703504,
      "grad_norm": 3.26247077011793,
      "kl": 3.3671875,
      "learning_rate": 8.948197212871751e-07,
      "loss": 0.36761924624443054,
      "memory(GiB)": 75.26,
      "reward": 1.8066893815994263,
      "reward_std": 0.5333396196365356,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3764529526233673,
      "rewards/ReflectionBalance/mean": 0.728564441204071,
      "rewards/ReflectionBalance/std": 0.3094162493944168,
      "step": 404,
      "train_speed(iter/s)": 0.007078
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 485.83984375,
      "completions/min_length": 251.0,
      "epoch": 2.188679245283019,
      "grad_norm": 3.253783551341168,
      "kl": 3.5703125,
      "learning_rate": 8.942927639287805e-07,
      "loss": 0.39488816261291504,
      "memory(GiB)": 75.26,
      "reward": 1.7794286608695984,
      "reward_std": 0.5877084136009216,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3764529526233673,
      "rewards/ReflectionBalance/mean": 0.7169286906719208,
      "rewards/ReflectionBalance/std": 0.325573667883873,
      "step": 405,
      "train_speed(iter/s)": 0.007077
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 465.2265625,
      "completions/min_length": 258.5,
      "epoch": 2.1940700808625335,
      "grad_norm": 4.254179742165689,
      "kl": 2.4609375,
      "learning_rate": 8.937646458121942e-07,
      "loss": 0.2804114520549774,
      "memory(GiB)": 75.26,
      "reward": 1.7977213263511658,
      "reward_std": 0.47413673996925354,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.29259639233350754,
      "rewards/ReflectionBalance/mean": 0.7703776359558105,
      "rewards/ReflectionBalance/std": 0.25205107033252716,
      "step": 406,
      "train_speed(iter/s)": 0.007078
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 482.41796875,
      "completions/min_length": 258.0,
      "epoch": 2.1994609164420487,
      "grad_norm": 37.78831036183214,
      "kl": 3.7421875,
      "learning_rate": 8.932353684921426e-07,
      "loss": 0.39879024028778076,
      "memory(GiB)": 75.26,
      "reward": 1.7932844758033752,
      "reward_std": 0.5631749927997589,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3643587827682495,
      "rewards/ReflectionBalance/mean": 0.7425032258033752,
      "rewards/ReflectionBalance/std": 0.3036223500967026,
      "step": 407,
      "train_speed(iter/s)": 0.007078
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.0,
      "completions/mean_length": 533.24609375,
      "completions/min_length": 244.0,
      "epoch": 2.2048517520215634,
      "grad_norm": 7.520125751844371,
      "kl": 6.0,
      "learning_rate": 8.927049335267654e-07,
      "loss": 0.5664602518081665,
      "memory(GiB)": 75.26,
      "reward": 1.6368114948272705,
      "reward_std": 0.7248525619506836,
      "rewards/Format/mean": 0.80078125,
      "rewards/Format/std": 0.40095221996307373,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36045311391353607,
      "rewards/ReflectionBalance/mean": 0.6797802746295929,
      "rewards/ReflectionBalance/std": 0.3646397292613983,
      "step": 408,
      "train_speed(iter/s)": 0.007079
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 519.28125,
      "completions/min_length": 253.5,
      "epoch": 2.210242587601078,
      "grad_norm": 8.175849619930457,
      "kl": 6.1875,
      "learning_rate": 8.921733424776093e-07,
      "loss": 0.5440744161605835,
      "memory(GiB)": 75.26,
      "reward": 1.7822167873382568,
      "reward_std": 0.6867974698543549,
      "rewards/Format/mean": 0.8203125,
      "rewards/Format/std": 0.3853000998497009,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.44853854179382324,
      "rewards/ReflectionBalance/mean": 0.6767480075359344,
      "rewards/ReflectionBalance/std": 0.34083813428878784,
      "step": 409,
      "train_speed(iter/s)": 0.007078
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 517.4296875,
      "completions/min_length": 232.0,
      "epoch": 2.215633423180593,
      "grad_norm": 4.224423615459406,
      "kl": 5.5,
      "learning_rate": 8.916405969096251e-07,
      "loss": 0.5428524613380432,
      "memory(GiB)": 75.26,
      "reward": 1.6377278566360474,
      "reward_std": 0.7091845571994781,
      "rewards/Format/mean": 0.8203125,
      "rewards/Format/std": 0.3854355216026306,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3556671589612961,
      "rewards/ReflectionBalance/mean": 0.6650716364383698,
      "rewards/ReflectionBalance/std": 0.37212295830249786,
      "step": 410,
      "train_speed(iter/s)": 0.007078
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 465.25390625,
      "completions/min_length": 231.0,
      "epoch": 2.221024258760108,
      "grad_norm": 2.9502089534208022,
      "kl": 3.234375,
      "learning_rate": 8.911066983911618e-07,
      "loss": 0.3102502226829529,
      "memory(GiB)": 75.26,
      "reward": 1.9728809595108032,
      "reward_std": 0.47456564009189606,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.3125,
      "rewards/MultiModalAccuracyORM/std": 0.4652567654848099,
      "rewards/ReflectionBalance/mean": 0.7619433701038361,
      "rewards/ReflectionBalance/std": 0.26586537063121796,
      "step": 411,
      "train_speed(iter/s)": 0.007077
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 493.25,
      "completions/min_length": 273.5,
      "epoch": 2.2264150943396226,
      "grad_norm": 3.815370501505482,
      "kl": 3.75,
      "learning_rate": 8.905716484939632e-07,
      "loss": 0.3899938464164734,
      "memory(GiB)": 75.26,
      "reward": 1.8529883027076721,
      "reward_std": 0.6495715975761414,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33513282239437103,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.43674568831920624,
      "rewards/ReflectionBalance/mean": 0.7279883027076721,
      "rewards/ReflectionBalance/std": 0.32816971838474274,
      "step": 412,
      "train_speed(iter/s)": 0.007078
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 445.328125,
      "completions/min_length": 278.5,
      "epoch": 2.2318059299191373,
      "grad_norm": 2.7740116253122045,
      "kl": 1.8203125,
      "learning_rate": 8.900354487931621e-07,
      "loss": 0.16908423602581024,
      "memory(GiB)": 75.26,
      "reward": 1.9694580435752869,
      "reward_std": 0.4345948398113251,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.20884444564580917,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41502299904823303,
      "rewards/ReflectionBalance/mean": 0.7975830137729645,
      "rewards/ReflectionBalance/std": 0.24474408477544785,
      "step": 413,
      "train_speed(iter/s)": 0.00708
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 441.5234375,
      "completions/min_length": 227.0,
      "epoch": 2.2371967654986524,
      "grad_norm": 9.323311976138315,
      "kl": 2.25390625,
      "learning_rate": 8.894981008672763e-07,
      "loss": 0.2477748543024063,
      "memory(GiB)": 75.26,
      "reward": 1.963487982749939,
      "reward_std": 0.4387056529521942,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2488820105791092,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42526112496852875,
      "rewards/ReflectionBalance/mean": 0.7916129231452942,
      "rewards/ReflectionBalance/std": 0.23776955902576447,
      "step": 414,
      "train_speed(iter/s)": 0.007079
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 462.12109375,
      "completions/min_length": 234.5,
      "epoch": 2.242587601078167,
      "grad_norm": 3.2924127309071665,
      "kl": 1.625,
      "learning_rate": 8.889596062982042e-07,
      "loss": 0.1878855675458908,
      "memory(GiB)": 75.26,
      "reward": 1.867656171321869,
      "reward_std": 0.42353059351444244,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3711828291416168,
      "rewards/ReflectionBalance/mean": 0.7778125107288361,
      "rewards/ReflectionBalance/std": 0.25762417167425156,
      "step": 415,
      "train_speed(iter/s)": 0.007079
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 995.0,
      "completions/mean_length": 452.3515625,
      "completions/min_length": 255.0,
      "epoch": 2.247978436657682,
      "grad_norm": 3.0920402603480635,
      "kl": 1.5205078125,
      "learning_rate": 8.884199666712197e-07,
      "loss": 0.16796211898326874,
      "memory(GiB)": 75.26,
      "reward": 1.8605419397354126,
      "reward_std": 0.3720574527978897,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.20856640115380287,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.25923943147063255,
      "rewards/ReflectionBalance/mean": 0.7902294993400574,
      "rewards/ReflectionBalance/std": 0.2277323454618454,
      "step": 416,
      "train_speed(iter/s)": 0.007079
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 436.23046875,
      "completions/min_length": 259.0,
      "epoch": 2.2533692722371965,
      "grad_norm": 3.92816257670341,
      "kl": 1.8828125,
      "learning_rate": 8.878791835749673e-07,
      "loss": 0.22987188398838043,
      "memory(GiB)": 75.26,
      "reward": 1.9446069598197937,
      "reward_std": 0.4328737407922745,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2255912721157074,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.39750634133815765,
      "rewards/ReflectionBalance/mean": 0.8039819300174713,
      "rewards/ReflectionBalance/std": 0.21467851847410202,
      "step": 417,
      "train_speed(iter/s)": 0.00708
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 451.41015625,
      "completions/min_length": 217.5,
      "epoch": 2.2587601078167117,
      "grad_norm": 3.2191826313824716,
      "kl": 2.0,
      "learning_rate": 8.873372586014579e-07,
      "loss": 0.19219297170639038,
      "memory(GiB)": 75.26,
      "reward": 1.8878856897354126,
      "reward_std": 0.41923049092292786,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.24629417806863785,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3753669559955597,
      "rewards/ReflectionBalance/mean": 0.7785107493400574,
      "rewards/ReflectionBalance/std": 0.2632741555571556,
      "step": 418,
      "train_speed(iter/s)": 0.007079
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 473.89453125,
      "completions/min_length": 253.5,
      "epoch": 2.2641509433962264,
      "grad_norm": 2.5605963502163798,
      "kl": 2.28125,
      "learning_rate": 8.867941933460643e-07,
      "loss": 0.23267492651939392,
      "memory(GiB)": 75.26,
      "reward": 1.7735888957977295,
      "reward_std": 0.46991656720638275,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3068038523197174,
      "rewards/ReflectionBalance/mean": 0.7540576457977295,
      "rewards/ReflectionBalance/std": 0.28147414326667786,
      "step": 419,
      "train_speed(iter/s)": 0.007078
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 495.23046875,
      "completions/min_length": 258.0,
      "epoch": 2.269541778975741,
      "grad_norm": 3.206873898673928,
      "kl": 4.1171875,
      "learning_rate": 8.862499894075158e-07,
      "loss": 0.3946881890296936,
      "memory(GiB)": 75.26,
      "reward": 1.770699918270111,
      "reward_std": 0.6520123481750488,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.33569003641605377,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39416566491127014,
      "rewards/ReflectionBalance/mean": 0.7121061384677887,
      "rewards/ReflectionBalance/std": 0.33452147245407104,
      "step": 420,
      "train_speed(iter/s)": 0.007078
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 459.10546875,
      "completions/min_length": 259.5,
      "epoch": 2.274932614555256,
      "grad_norm": 4.401845590730674,
      "kl": 2.6640625,
      "learning_rate": 8.85704648387894e-07,
      "loss": 0.28567811846733093,
      "memory(GiB)": 75.26,
      "reward": 1.8582372665405273,
      "reward_std": 0.4806831628084183,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.36039385199546814,
      "rewards/ReflectionBalance/mean": 0.7762060761451721,
      "rewards/ReflectionBalance/std": 0.25991275906562805,
      "step": 421,
      "train_speed(iter/s)": 0.007079
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 454.52734375,
      "completions/min_length": 257.5,
      "epoch": 2.280323450134771,
      "grad_norm": 3.2088983688219233,
      "kl": 2.9140625,
      "learning_rate": 8.85158171892628e-07,
      "loss": 0.3094125986099243,
      "memory(GiB)": 75.26,
      "reward": 1.8605436086654663,
      "reward_std": 0.498434379696846,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3556671589612961,
      "rewards/ReflectionBalance/mean": 0.7902311086654663,
      "rewards/ReflectionBalance/std": 0.234624944627285,
      "step": 422,
      "train_speed(iter/s)": 0.007079
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 451.91796875,
      "completions/min_length": 201.5,
      "epoch": 2.2857142857142856,
      "grad_norm": 4.486839797083884,
      "kl": 3.078125,
      "learning_rate": 8.846105615304895e-07,
      "loss": 0.2927403748035431,
      "memory(GiB)": 75.26,
      "reward": 1.8848437666893005,
      "reward_std": 0.4592220187187195,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.38954539597034454,
      "rewards/ReflectionBalance/mean": 0.7676562368869781,
      "rewards/ReflectionBalance/std": 0.2735642194747925,
      "step": 423,
      "train_speed(iter/s)": 0.00708
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 451.5859375,
      "completions/min_length": 202.0,
      "epoch": 2.2911051212938007,
      "grad_norm": 3.0050861520643743,
      "kl": 2.34375,
      "learning_rate": 8.840618189135883e-07,
      "loss": 0.2590092122554779,
      "memory(GiB)": 75.26,
      "reward": 1.8476024866104126,
      "reward_std": 0.40689511597156525,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3437131494283676,
      "rewards/ReflectionBalance/mean": 0.7929150462150574,
      "rewards/ReflectionBalance/std": 0.22782111912965775,
      "step": 424,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.5,
      "completions/mean_length": 462.84765625,
      "completions/min_length": 255.5,
      "epoch": 2.2964959568733154,
      "grad_norm": 3.2872868950424974,
      "kl": 2.15234375,
      "learning_rate": 8.835119456573671e-07,
      "loss": 0.22919249534606934,
      "memory(GiB)": 75.26,
      "reward": 1.7808398008346558,
      "reward_std": 0.44379711151123047,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.298691526055336,
      "rewards/ReflectionBalance/mean": 0.7534960806369781,
      "rewards/ReflectionBalance/std": 0.2713572829961777,
      "step": 425,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 498.9140625,
      "completions/min_length": 241.5,
      "epoch": 2.30188679245283,
      "grad_norm": 4.479601493346118,
      "kl": 3.84375,
      "learning_rate": 8.829609433805981e-07,
      "loss": 0.34977349638938904,
      "memory(GiB)": 75.26,
      "reward": 1.8117058277130127,
      "reward_std": 0.5360930263996124,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3679620176553726,
      "rewards/ReflectionBalance/mean": 0.7492057085037231,
      "rewards/ReflectionBalance/std": 0.29356610774993896,
      "step": 426,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 505.5703125,
      "completions/min_length": 269.5,
      "epoch": 2.3072776280323453,
      "grad_norm": 4.34892033639026,
      "kl": 5.09375,
      "learning_rate": 8.824088137053759e-07,
      "loss": 0.4957166314125061,
      "memory(GiB)": 75.26,
      "reward": 1.6841195225715637,
      "reward_std": 0.6342076063156128,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.35538628697395325,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3473474234342575,
      "rewards/ReflectionBalance/mean": 0.691931962966919,
      "rewards/ReflectionBalance/std": 0.3176615983247757,
      "step": 427,
      "train_speed(iter/s)": 0.00708
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 475.9140625,
      "completions/min_length": 260.5,
      "epoch": 2.31266846361186,
      "grad_norm": 3.356561585683665,
      "kl": 2.5625,
      "learning_rate": 8.81855558257115e-07,
      "loss": 0.24331843852996826,
      "memory(GiB)": 75.26,
      "reward": 1.836757779121399,
      "reward_std": 0.4956595450639725,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3689865916967392,
      "rewards/ReflectionBalance/mean": 0.7625390589237213,
      "rewards/ReflectionBalance/std": 0.26525114476680756,
      "step": 428,
      "train_speed(iter/s)": 0.00708
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 442.66796875,
      "completions/min_length": 241.5,
      "epoch": 2.3180592991913747,
      "grad_norm": 3.1728464944669397,
      "kl": 2.171875,
      "learning_rate": 8.813011786645435e-07,
      "loss": 0.2178577035665512,
      "memory(GiB)": 75.26,
      "reward": 1.931064486503601,
      "reward_std": 0.48796820640563965,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23196382820606232,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.4020131081342697,
      "rewards/ReflectionBalance/mean": 0.7865332067012787,
      "rewards/ReflectionBalance/std": 0.24237056076526642,
      "step": 429,
      "train_speed(iter/s)": 0.00708
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 453.5859375,
      "completions/min_length": 282.0,
      "epoch": 2.3234501347708894,
      "grad_norm": 2.903953981017628,
      "kl": 3.40625,
      "learning_rate": 8.807456765596991e-07,
      "loss": 0.35768628120422363,
      "memory(GiB)": 75.26,
      "reward": 1.8352588415145874,
      "reward_std": 0.4926685690879822,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35538628697395325,
      "rewards/ReflectionBalance/mean": 0.7688525319099426,
      "rewards/ReflectionBalance/std": 0.24854932725429535,
      "step": 430,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 460.9453125,
      "completions/min_length": 254.5,
      "epoch": 2.3288409703504045,
      "grad_norm": 4.127613641792446,
      "kl": 3.5390625,
      "learning_rate": 8.801890535779243e-07,
      "loss": 0.3470456302165985,
      "memory(GiB)": 75.26,
      "reward": 1.7870190143585205,
      "reward_std": 0.5827630460262299,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.36721789836883545,
      "rewards/ReflectionBalance/mean": 0.7401440143585205,
      "rewards/ReflectionBalance/std": 0.29146550595760345,
      "step": 431,
      "train_speed(iter/s)": 0.00708
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 455.74609375,
      "completions/min_length": 243.0,
      "epoch": 2.334231805929919,
      "grad_norm": 3.1649988627965904,
      "kl": 3.03125,
      "learning_rate": 8.796313113578608e-07,
      "loss": 0.2958165407180786,
      "memory(GiB)": 75.26,
      "reward": 1.8643065690994263,
      "reward_std": 0.507546678185463,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3711828291416168,
      "rewards/ReflectionBalance/mean": 0.786181628704071,
      "rewards/ReflectionBalance/std": 0.250686876475811,
      "step": 432,
      "train_speed(iter/s)": 0.007079
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 480.12890625,
      "completions/min_length": 249.5,
      "epoch": 2.339622641509434,
      "grad_norm": 3.5276851546348826,
      "kl": 3.890625,
      "learning_rate": 8.790724515414458e-07,
      "loss": 0.350266695022583,
      "memory(GiB)": 75.26,
      "reward": 1.8149544596672058,
      "reward_std": 0.6124798059463501,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33636194467544556,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3842122554779053,
      "rewards/ReflectionBalance/mean": 0.7641731798648834,
      "rewards/ReflectionBalance/std": 0.28062720596790314,
      "step": 433,
      "train_speed(iter/s)": 0.007078
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 493.46484375,
      "completions/min_length": 239.0,
      "epoch": 2.3450134770889486,
      "grad_norm": 4.6715676862621045,
      "kl": 5.078125,
      "learning_rate": 8.785124757739062e-07,
      "loss": 0.4895934462547302,
      "memory(GiB)": 75.26,
      "reward": 1.727807641029358,
      "reward_std": 0.6531721353530884,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.3619322180747986,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3613582104444504,
      "rewards/ReflectionBalance/mean": 0.7160888612270355,
      "rewards/ReflectionBalance/std": 0.31086474657058716,
      "step": 434,
      "train_speed(iter/s)": 0.007078
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 492.7734375,
      "completions/min_length": 255.0,
      "epoch": 2.3504043126684637,
      "grad_norm": 4.283414061700448,
      "kl": 5.40625,
      "learning_rate": 8.779513857037542e-07,
      "loss": 0.5028498768806458,
      "memory(GiB)": 75.26,
      "reward": 1.7065331935882568,
      "reward_std": 0.6594265103340149,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.3607245087623596,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3460448384284973,
      "rewards/ReflectionBalance/mean": 0.7182519435882568,
      "rewards/ReflectionBalance/std": 0.3060024529695511,
      "step": 435,
      "train_speed(iter/s)": 0.007079
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 439.83203125,
      "completions/min_length": 266.5,
      "epoch": 2.3557951482479784,
      "grad_norm": 2.9015466592044663,
      "kl": 2.703125,
      "learning_rate": 8.773891829827826e-07,
      "loss": 0.2686150372028351,
      "memory(GiB)": 75.26,
      "reward": 2.064404308795929,
      "reward_std": 0.5214892625808716,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2488820105791092,
      "rewards/MultiModalAccuracyORM/mean": 0.3671875,
      "rewards/MultiModalAccuracyORM/std": 0.48146240413188934,
      "rewards/ReflectionBalance/mean": 0.767529308795929,
      "rewards/ReflectionBalance/std": 0.2542691305279732,
      "step": 436,
      "train_speed(iter/s)": 0.007079
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 459.67578125,
      "completions/min_length": 240.5,
      "epoch": 2.361185983827493,
      "grad_norm": 3.8627681428563094,
      "kl": 2.8515625,
      "learning_rate": 8.768258692660596e-07,
      "loss": 0.27513158321380615,
      "memory(GiB)": 75.26,
      "reward": 1.8036653995513916,
      "reward_std": 0.49967682361602783,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2897924929857254,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.336664155125618,
      "rewards/ReflectionBalance/mean": 0.7567903399467468,
      "rewards/ReflectionBalance/std": 0.2692219465970993,
      "step": 437,
      "train_speed(iter/s)": 0.007079
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 446.81640625,
      "completions/min_length": 284.0,
      "epoch": 2.3665768194070083,
      "grad_norm": 2.4369536890212964,
      "kl": 1.7890625,
      "learning_rate": 8.762614462119236e-07,
      "loss": 0.16930127143859863,
      "memory(GiB)": 75.26,
      "reward": 1.8709065914154053,
      "reward_std": 0.42282477021217346,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.22757095843553543,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3460448384284973,
      "rewards/ReflectionBalance/mean": 0.7927815616130829,
      "rewards/ReflectionBalance/std": 0.2366221621632576,
      "step": 438,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 438.86328125,
      "completions/min_length": 226.5,
      "epoch": 2.371967654986523,
      "grad_norm": 2.6247731465392805,
      "kl": 1.6015625,
      "learning_rate": 8.756959154819801e-07,
      "loss": 0.16055449843406677,
      "memory(GiB)": 75.26,
      "reward": 1.9312768578529358,
      "reward_std": 0.5031587928533554,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4100748300552368,
      "rewards/ReflectionBalance/mean": 0.7750268578529358,
      "rewards/ReflectionBalance/std": 0.253616601228714,
      "step": 439,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 506.875,
      "completions/min_length": 266.5,
      "epoch": 2.3773584905660377,
      "grad_norm": 2.9483877298962504,
      "kl": 2.06640625,
      "learning_rate": 8.751292787410938e-07,
      "loss": 0.2267971783876419,
      "memory(GiB)": 75.26,
      "reward": 1.8039599657058716,
      "reward_std": 0.46362558007240295,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.298691526055336,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.34447427093982697,
      "rewards/ReflectionBalance/mean": 0.768803745508194,
      "rewards/ReflectionBalance/std": 0.2577533423900604,
      "step": 440,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 468.6953125,
      "completions/min_length": 263.5,
      "epoch": 2.382749326145553,
      "grad_norm": 14.228001742447372,
      "kl": 1.79296875,
      "learning_rate": 8.745615376573867e-07,
      "loss": 0.19594410061836243,
      "memory(GiB)": 75.26,
      "reward": 1.9772526025772095,
      "reward_std": 0.5249453783035278,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.4410945028066635,
      "rewards/ReflectionBalance/mean": 0.8053775727748871,
      "rewards/ReflectionBalance/std": 0.23045889288187027,
      "step": 441,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 448.9296875,
      "completions/min_length": 258.0,
      "epoch": 2.3881401617250675,
      "grad_norm": 3.089472886407138,
      "kl": 1.5234375,
      "learning_rate": 8.739926939022313e-07,
      "loss": 0.1663900911808014,
      "memory(GiB)": 75.26,
      "reward": 1.8865674138069153,
      "reward_std": 0.44204291701316833,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2255912721157074,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3734995424747467,
      "rewards/ReflectionBalance/mean": 0.7732861340045929,
      "rewards/ReflectionBalance/std": 0.24372722953557968,
      "step": 442,
      "train_speed(iter/s)": 0.007082
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 445.56640625,
      "completions/min_length": 267.5,
      "epoch": 2.393530997304582,
      "grad_norm": 3.8268582695429667,
      "kl": 1.828125,
      "learning_rate": 8.734227491502462e-07,
      "loss": 0.22882521152496338,
      "memory(GiB)": 75.26,
      "reward": 1.8895816802978516,
      "reward_std": 0.44692277908325195,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3456958532333374,
      "rewards/ReflectionBalance/mean": 0.784112960100174,
      "rewards/ReflectionBalance/std": 0.23529621213674545,
      "step": 443,
      "train_speed(iter/s)": 0.007082
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 455.4140625,
      "completions/min_length": 239.5,
      "epoch": 2.398921832884097,
      "grad_norm": 4.075007859069508,
      "kl": 3.12109375,
      "learning_rate": 8.728517050792914e-07,
      "loss": 0.3162057399749756,
      "memory(GiB)": 75.26,
      "reward": 1.7607861161231995,
      "reward_std": 0.5945852696895599,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.330063596367836,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3410066068172455,
      "rewards/ReflectionBalance/mean": 0.7529736459255219,
      "rewards/ReflectionBalance/std": 0.283716581761837,
      "step": 444,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 535.25,
      "completions/min_length": 240.5,
      "epoch": 2.404312668463612,
      "grad_norm": 4.547917012301297,
      "kl": 4.90625,
      "learning_rate": 8.722795633704632e-07,
      "loss": 0.46337005496025085,
      "memory(GiB)": 75.26,
      "reward": 1.614804744720459,
      "reward_std": 0.7282222509384155,
      "rewards/Format/mean": 0.80078125,
      "rewards/Format/std": 0.3985297381877899,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3460448384284973,
      "rewards/ReflectionBalance/mean": 0.6733984351158142,
      "rewards/ReflectionBalance/std": 0.36110347509384155,
      "step": 445,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 504.609375,
      "completions/min_length": 188.0,
      "epoch": 2.4097035040431267,
      "grad_norm": 4.072693991216485,
      "kl": 4.625,
      "learning_rate": 8.71706325708089e-07,
      "loss": 0.4429187774658203,
      "memory(GiB)": 75.26,
      "reward": 1.6793017983436584,
      "reward_std": 0.5904094576835632,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.35675957798957825,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3130832314491272,
      "rewards/ReflectionBalance/mean": 0.7183642387390137,
      "rewards/ReflectionBalance/std": 0.2998598664999008,
      "step": 446,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 462.8984375,
      "completions/min_length": 240.0,
      "epoch": 2.4150943396226414,
      "grad_norm": 2.758072277211499,
      "kl": 2.609375,
      "learning_rate": 8.71131993779723e-07,
      "loss": 0.27677658200263977,
      "memory(GiB)": 75.26,
      "reward": 1.839687466621399,
      "reward_std": 0.4807748347520828,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3853980004787445,
      "rewards/ReflectionBalance/mean": 0.7498437166213989,
      "rewards/ReflectionBalance/std": 0.2700895816087723,
      "step": 447,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 539.08984375,
      "completions/min_length": 254.0,
      "epoch": 2.420485175202156,
      "grad_norm": 4.140576325948614,
      "kl": 4.796875,
      "learning_rate": 8.705565692761404e-07,
      "loss": 0.43159157037734985,
      "memory(GiB)": 75.26,
      "reward": 1.6482056379318237,
      "reward_std": 0.6279822587966919,
      "rewards/Format/mean": 0.796875,
      "rewards/Format/std": 0.40284374356269836,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3388633653521538,
      "rewards/ReflectionBalance/mean": 0.695080578327179,
      "rewards/ReflectionBalance/std": 0.3493647873401642,
      "step": 448,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 484.05859375,
      "completions/min_length": 226.0,
      "epoch": 2.4258760107816713,
      "grad_norm": 2.89543152260906,
      "kl": 2.5859375,
      "learning_rate": 8.699800538913331e-07,
      "loss": 0.2737014889717102,
      "memory(GiB)": 75.26,
      "reward": 1.7827115654945374,
      "reward_std": 0.40001145005226135,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.09375,
      "rewards/MultiModalAccuracyORM/std": 0.2516305334866047,
      "rewards/ReflectionBalance/mean": 0.7631803154945374,
      "rewards/ReflectionBalance/std": 0.2640663608908653,
      "step": 449,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 466.78125,
      "completions/min_length": 244.5,
      "epoch": 2.431266846361186,
      "grad_norm": 3.2780661637401973,
      "kl": 2.41796875,
      "learning_rate": 8.694024493225043e-07,
      "loss": 0.2541365325450897,
      "memory(GiB)": 75.26,
      "reward": 1.788706123828888,
      "reward_std": 0.434621199965477,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.27175262570381165,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.29434798657894135,
      "rewards/ReflectionBalance/mean": 0.7808935642242432,
      "rewards/ReflectionBalance/std": 0.2489217221736908,
      "step": 450,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 493.33984375,
      "completions/min_length": 288.0,
      "epoch": 2.4366576819407006,
      "grad_norm": 3.2023866772778615,
      "kl": 2.5234375,
      "learning_rate": 8.688237572700634e-07,
      "loss": 0.27235502004623413,
      "memory(GiB)": 75.26,
      "reward": 1.8798030614852905,
      "reward_std": 0.4570871889591217,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38837648928165436,
      "rewards/ReflectionBalance/mean": 0.7743342816829681,
      "rewards/ReflectionBalance/std": 0.2477685585618019,
      "step": 451,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 457.90234375,
      "completions/min_length": 254.0,
      "epoch": 2.442048517520216,
      "grad_norm": 2.6620239519397257,
      "kl": 1.578125,
      "learning_rate": 8.682439794376221e-07,
      "loss": 0.15692877769470215,
      "memory(GiB)": 75.26,
      "reward": 1.9827864170074463,
      "reward_std": 0.4012216031551361,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22826264798641205,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.42273683845996857,
      "rewards/ReflectionBalance/mean": 0.8030989468097687,
      "rewards/ReflectionBalance/std": 0.2232721596956253,
      "step": 452,
      "train_speed(iter/s)": 0.007082
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 491.19921875,
      "completions/min_length": 233.5,
      "epoch": 2.4474393530997305,
      "grad_norm": 3.3476754507911735,
      "kl": 3.66015625,
      "learning_rate": 8.676631175319875e-07,
      "loss": 0.3340900242328644,
      "memory(GiB)": 75.26,
      "reward": 1.754173219203949,
      "reward_std": 0.5021993815898895,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3210565447807312,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.330063596367836,
      "rewards/ReflectionBalance/mean": 0.7502669095993042,
      "rewards/ReflectionBalance/std": 0.2861425206065178,
      "step": 453,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 474.27734375,
      "completions/min_length": 274.0,
      "epoch": 2.452830188679245,
      "grad_norm": 3.1696746425495173,
      "kl": 3.234375,
      "learning_rate": 8.670811732631587e-07,
      "loss": 0.31941360235214233,
      "memory(GiB)": 75.26,
      "reward": 1.7912126183509827,
      "reward_std": 0.4849284291267395,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.28836025297641754,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.33513282239437103,
      "rewards/ReflectionBalance/mean": 0.7638688683509827,
      "rewards/ReflectionBalance/std": 0.2645503506064415,
      "step": 454,
      "train_speed(iter/s)": 0.007082
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 491.75390625,
      "completions/min_length": 235.0,
      "epoch": 2.4582210242587603,
      "grad_norm": 3.537851620929695,
      "kl": 4.625,
      "learning_rate": 8.664981483443212e-07,
      "loss": 0.4814412295818329,
      "memory(GiB)": 75.26,
      "reward": 1.785463809967041,
      "reward_std": 0.5139557421207428,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32612966001033783,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3046952188014984,
      "rewards/ReflectionBalance/mean": 0.7659326493740082,
      "rewards/ReflectionBalance/std": 0.28616029024124146,
      "step": 455,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 503.1796875,
      "completions/min_length": 261.5,
      "epoch": 2.463611859838275,
      "grad_norm": 3.9983189281869325,
      "kl": 4.390625,
      "learning_rate": 8.659140444918412e-07,
      "loss": 0.44023436307907104,
      "memory(GiB)": 75.26,
      "reward": 1.6933984756469727,
      "reward_std": 0.5938948392868042,
      "rewards/Format/mean": 0.83203125,
      "rewards/Format/std": 0.3752707988023758,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3270204961299896,
      "rewards/ReflectionBalance/mean": 0.7402734160423279,
      "rewards/ReflectionBalance/std": 0.3054921478033066,
      "step": 456,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 493.76953125,
      "completions/min_length": 236.5,
      "epoch": 2.4690026954177897,
      "grad_norm": 4.659950805122108,
      "kl": 4.578125,
      "learning_rate": 8.65328863425262e-07,
      "loss": 0.42287665605545044,
      "memory(GiB)": 75.26,
      "reward": 1.8723208904266357,
      "reward_std": 0.5942528247833252,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33636194467544556,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.41966044902801514,
      "rewards/ReflectionBalance/mean": 0.7707584500312805,
      "rewards/ReflectionBalance/std": 0.26646265387535095,
      "step": 457,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 473.31640625,
      "completions/min_length": 270.5,
      "epoch": 2.4743935309973044,
      "grad_norm": 2.840135750574284,
      "kl": 3.2421875,
      "learning_rate": 8.647426068672977e-07,
      "loss": 0.31512102484703064,
      "memory(GiB)": 75.26,
      "reward": 1.8304800987243652,
      "reward_std": 0.5790559947490692,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.3526121824979782,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4091253876686096,
      "rewards/ReflectionBalance/mean": 0.76407390832901,
      "rewards/ReflectionBalance/std": 0.28280334174633026,
      "step": 458,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 465.390625,
      "completions/min_length": 248.0,
      "epoch": 2.4797843665768196,
      "grad_norm": 3.3352522936654774,
      "kl": 2.484375,
      "learning_rate": 8.641552765438285e-07,
      "loss": 0.28487762808799744,
      "memory(GiB)": 75.26,
      "reward": 1.7961360812187195,
      "reward_std": 0.4901829659938812,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27079907804727554,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.2831462025642395,
      "rewards/ReflectionBalance/mean": 0.7766048312187195,
      "rewards/ReflectionBalance/std": 0.2805042862892151,
      "step": 459,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 492.40625,
      "completions/min_length": 271.5,
      "epoch": 2.4851752021563343,
      "grad_norm": 3.548155629067205,
      "kl": 2.6171875,
      "learning_rate": 8.635668741838957e-07,
      "loss": 0.24321280419826508,
      "memory(GiB)": 75.26,
      "reward": 1.8299121260643005,
      "reward_std": 0.4844600111246109,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31333550810813904,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36175550520420074,
      "rewards/ReflectionBalance/mean": 0.7752245962619781,
      "rewards/ReflectionBalance/std": 0.26992014050483704,
      "step": 460,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 449.2265625,
      "completions/min_length": 247.5,
      "epoch": 2.490566037735849,
      "grad_norm": 3.4988482836309394,
      "kl": 1.494140625,
      "learning_rate": 8.629774015196967e-07,
      "loss": 0.16285735368728638,
      "memory(GiB)": 75.26,
      "reward": 1.9898055791854858,
      "reward_std": 0.43945208191871643,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.19744773209095,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.4098135679960251,
      "rewards/ReflectionBalance/mean": 0.8101179897785187,
      "rewards/ReflectionBalance/std": 0.22614409029483795,
      "step": 461,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 480.66796875,
      "completions/min_length": 258.0,
      "epoch": 2.4959568733153636,
      "grad_norm": 3.1620234887943086,
      "kl": 2.25,
      "learning_rate": 8.623868602865795e-07,
      "loss": 0.2416101098060608,
      "memory(GiB)": 75.26,
      "reward": 1.78956538438797,
      "reward_std": 0.46987172961235046,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.29794247448444366,
      "rewards/ReflectionBalance/mean": 0.7817529141902924,
      "rewards/ReflectionBalance/std": 0.28487686812877655,
      "step": 462,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 498.390625,
      "completions/min_length": 186.5,
      "epoch": 2.501347708894879,
      "grad_norm": 87.59061852423257,
      "kl": 5.90625,
      "learning_rate": 8.61795252223038e-07,
      "loss": 0.45676279067993164,
      "memory(GiB)": 75.26,
      "reward": 1.7076985836029053,
      "reward_std": 0.6549766957759857,
      "rewards/Format/mean": 0.828125,
      "rewards/Format/std": 0.3774644732475281,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3786114752292633,
      "rewards/ReflectionBalance/mean": 0.7076985239982605,
      "rewards/ReflectionBalance/std": 0.3621019273996353,
      "step": 463,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 476.9296875,
      "completions/min_length": 237.5,
      "epoch": 2.5067385444743935,
      "grad_norm": 4.218263923955505,
      "kl": 3.28125,
      "learning_rate": 8.612025790707071e-07,
      "loss": 0.3362045884132385,
      "memory(GiB)": 75.26,
      "reward": 1.8727473616600037,
      "reward_std": 0.5358388721942902,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.4037879854440689,
      "rewards/ReflectionBalance/mean": 0.767278641462326,
      "rewards/ReflectionBalance/std": 0.25921059399843216,
      "step": 464,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 501.3671875,
      "completions/min_length": 218.5,
      "epoch": 2.512129380053908,
      "grad_norm": 4.206052933911793,
      "kl": 4.671875,
      "learning_rate": 8.606088425743565e-07,
      "loss": 0.42225033044815063,
      "memory(GiB)": 75.26,
      "reward": 1.7171826362609863,
      "reward_std": 0.6350760161876678,
      "rewards/Format/mean": 0.81640625,
      "rewards/Format/std": 0.38704802095890045,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38489311933517456,
      "rewards/ReflectionBalance/mean": 0.7210888862609863,
      "rewards/ReflectionBalance/std": 0.31714554131031036,
      "step": 465,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 483.28125,
      "completions/min_length": 230.5,
      "epoch": 2.5175202156334233,
      "grad_norm": 4.237782721875128,
      "kl": 3.7421875,
      "learning_rate": 8.600140444818868e-07,
      "loss": 0.3584863245487213,
      "memory(GiB)": 75.26,
      "reward": 1.8647542595863342,
      "reward_std": 0.5491340458393097,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31518664956092834,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3996165096759796,
      "rewards/ReflectionBalance/mean": 0.7749104797840118,
      "rewards/ReflectionBalance/std": 0.2555556744337082,
      "step": 466,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 472.546875,
      "completions/min_length": 259.0,
      "epoch": 2.522911051212938,
      "grad_norm": 3.051899609755115,
      "kl": 2.7890625,
      "learning_rate": 8.594181865443238e-07,
      "loss": 0.2420138418674469,
      "memory(GiB)": 75.26,
      "reward": 1.9167113900184631,
      "reward_std": 0.5168327838182449,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.28125,
      "rewards/MultiModalAccuracyORM/std": 0.44721250236034393,
      "rewards/ReflectionBalance/mean": 0.7370239198207855,
      "rewards/ReflectionBalance/std": 0.3189496099948883,
      "step": 467,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 477.265625,
      "completions/min_length": 258.5,
      "epoch": 2.5283018867924527,
      "grad_norm": 2.8431993444635113,
      "kl": 2.0390625,
      "learning_rate": 8.588212705158132e-07,
      "loss": 0.18772676587104797,
      "memory(GiB)": 75.26,
      "reward": 1.8240413069725037,
      "reward_std": 0.4200115203857422,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.310598224401474,
      "rewards/ReflectionBalance/mean": 0.788885086774826,
      "rewards/ReflectionBalance/std": 0.22893574833869934,
      "step": 468,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 436.578125,
      "completions/min_length": 251.0,
      "epoch": 2.533692722371968,
      "grad_norm": 4.051152719611667,
      "kl": 1.8359375,
      "learning_rate": 8.582232981536155e-07,
      "loss": 0.18025805056095123,
      "memory(GiB)": 75.26,
      "reward": 1.9866324663162231,
      "reward_std": 0.48408031463623047,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.43386341631412506,
      "rewards/ReflectionBalance/mean": 0.8030387163162231,
      "rewards/ReflectionBalance/std": 0.23744072020053864,
      "step": 469,
      "train_speed(iter/s)": 0.007086
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1019.5,
      "completions/mean_length": 463.64453125,
      "completions/min_length": 262.0,
      "epoch": 2.5390835579514826,
      "grad_norm": 4.645273992981247,
      "kl": 0.939453125,
      "learning_rate": 8.576242712181015e-07,
      "loss": 0.11435724794864655,
      "memory(GiB)": 75.26,
      "reward": 1.8749161958694458,
      "reward_std": 0.31105074286460876,
      "rewards/Format/mean": 0.96484375,
      "rewards/Format/std": 0.1845906376838684,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.2795119285583496,
      "rewards/ReflectionBalance/mean": 0.8124161660671234,
      "rewards/ReflectionBalance/std": 0.19510970264673233,
      "step": 470,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1007.5,
      "completions/mean_length": 446.6953125,
      "completions/min_length": 244.0,
      "epoch": 2.5444743935309972,
      "grad_norm": 2.8571599156985696,
      "kl": 1.296875,
      "learning_rate": 8.570241914727456e-07,
      "loss": 0.13160642981529236,
      "memory(GiB)": 75.26,
      "reward": 1.9065917134284973,
      "reward_std": 0.33876511454582214,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.19343896210193634,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3290177956223488,
      "rewards/ReflectionBalance/mean": 0.8050292730331421,
      "rewards/ReflectionBalance/std": 0.19865841418504715,
      "step": 471,
      "train_speed(iter/s)": 0.007086
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 939.0,
      "completions/mean_length": 443.40234375,
      "completions/min_length": 215.5,
      "epoch": 2.5498652291105124,
      "grad_norm": 4.423856964290928,
      "kl": 1.1533203125,
      "learning_rate": 8.564230606841226e-07,
      "loss": 0.13654199242591858,
      "memory(GiB)": 75.26,
      "reward": 1.9089648127555847,
      "reward_std": 0.4045184552669525,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.19007275998592377,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3390307128429413,
      "rewards/ReflectionBalance/mean": 0.8113085925579071,
      "rewards/ReflectionBalance/std": 0.22047555446624756,
      "step": 472,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1020.5,
      "completions/mean_length": 462.84765625,
      "completions/min_length": 272.0,
      "epoch": 2.555256064690027,
      "grad_norm": 3.347669564933142,
      "kl": 0.966796875,
      "learning_rate": 8.55820880621901e-07,
      "loss": 0.1099107414484024,
      "memory(GiB)": 75.26,
      "reward": 1.9972509741783142,
      "reward_std": 0.36038798093795776,
      "rewards/Format/mean": 0.9765625,
      "rewards/Format/std": 0.1495915986597538,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.37918365001678467,
      "rewards/ReflectionBalance/mean": 0.8253759741783142,
      "rewards/ReflectionBalance/std": 0.1923878937959671,
      "step": 473,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 462.2265625,
      "completions/min_length": 243.5,
      "epoch": 2.560646900269542,
      "grad_norm": 2.4184531498323594,
      "kl": 1.73046875,
      "learning_rate": 8.552176530588379e-07,
      "loss": 0.18393276631832123,
      "memory(GiB)": 75.26,
      "reward": 1.9461816549301147,
      "reward_std": 0.42388543486595154,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.2389676719903946,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.422762006521225,
      "rewards/ReflectionBalance/mean": 0.7821191251277924,
      "rewards/ReflectionBalance/std": 0.22426796704530716,
      "step": 474,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 452.2421875,
      "completions/min_length": 274.5,
      "epoch": 2.5660377358490565,
      "grad_norm": 3.2704751744025318,
      "kl": 1.6328125,
      "learning_rate": 8.546133797707751e-07,
      "loss": 0.2175104171037674,
      "memory(GiB)": 75.26,
      "reward": 1.9149073362350464,
      "reward_std": 0.4311898946762085,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21138381212949753,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35872696340084076,
      "rewards/ReflectionBalance/mean": 0.8094384670257568,
      "rewards/ReflectionBalance/std": 0.2086053341627121,
      "step": 475,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1038.0,
      "completions/mean_length": 494.06640625,
      "completions/min_length": 269.5,
      "epoch": 2.571428571428571,
      "grad_norm": 3.85625957929828,
      "kl": 3.1328125,
      "learning_rate": 8.540080625366317e-07,
      "loss": 0.2810570001602173,
      "memory(GiB)": 75.26,
      "reward": 1.8461474180221558,
      "reward_std": 0.5725243985652924,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.298691526055336,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.7602099478244781,
      "rewards/ReflectionBalance/std": 0.2936290204524994,
      "step": 476,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 517.07421875,
      "completions/min_length": 276.0,
      "epoch": 2.5768194070080863,
      "grad_norm": 3.8832999358793683,
      "kl": 4.8125,
      "learning_rate": 8.534017031384009e-07,
      "loss": 0.4543966054916382,
      "memory(GiB)": 75.26,
      "reward": 1.7046183347702026,
      "reward_std": 0.6118988990783691,
      "rewards/Format/mean": 0.83203125,
      "rewards/Format/std": 0.368965208530426,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3410066068172455,
      "rewards/ReflectionBalance/mean": 0.7358683347702026,
      "rewards/ReflectionBalance/std": 0.3055151700973511,
      "step": 477,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 470.33203125,
      "completions/min_length": 209.5,
      "epoch": 2.582210242587601,
      "grad_norm": 6.8171954661026755,
      "kl": 4.5,
      "learning_rate": 8.52794303361144e-07,
      "loss": 0.37684327363967896,
      "memory(GiB)": 75.26,
      "reward": 1.8625715374946594,
      "reward_std": 0.6285815536975861,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3460448384284973,
      "rewards/MultiModalAccuracyORM/mean": 0.28125,
      "rewards/MultiModalAccuracyORM/std": 0.44721250236034393,
      "rewards/ReflectionBalance/mean": 0.7219466269016266,
      "rewards/ReflectionBalance/std": 0.3380645513534546,
      "step": 478,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 522.08984375,
      "completions/min_length": 278.5,
      "epoch": 2.5876010781671157,
      "grad_norm": 7.397478428826039,
      "kl": 6.0,
      "learning_rate": 8.521858649929843e-07,
      "loss": 0.5243507623672485,
      "memory(GiB)": 75.26,
      "reward": 1.7051334381103516,
      "reward_std": 0.6304300725460052,
      "rewards/Format/mean": 0.81640625,
      "rewards/Format/std": 0.38864077627658844,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3681531548500061,
      "rewards/ReflectionBalance/mean": 0.728570967912674,
      "rewards/ReflectionBalance/std": 0.3425145894289017,
      "step": 479,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 468.44921875,
      "completions/min_length": 259.5,
      "epoch": 2.592991913746631,
      "grad_norm": 2.7493337955035413,
      "kl": 3.1484375,
      "learning_rate": 8.515763898251034e-07,
      "loss": 0.2865508198738098,
      "memory(GiB)": 75.26,
      "reward": 1.8799055814743042,
      "reward_std": 0.5864527076482773,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.298691526055336,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.40395987033843994,
      "rewards/ReflectionBalance/mean": 0.7705306112766266,
      "rewards/ReflectionBalance/std": 0.2750358283519745,
      "step": 480,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 529.7421875,
      "completions/min_length": 299.0,
      "epoch": 2.5983827493261455,
      "grad_norm": 3.221970429194602,
      "kl": 4.125,
      "learning_rate": 8.509658796517348e-07,
      "loss": 0.39467597007751465,
      "memory(GiB)": 75.26,
      "reward": 1.6145394444465637,
      "reward_std": 0.5909997820854187,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.3645188808441162,
      "rewards/MultiModalAccuracyORM/mean": 0.05859375,
      "rewards/MultiModalAccuracyORM/std": 0.23443959653377533,
      "rewards/ReflectionBalance/mean": 0.712195634841919,
      "rewards/ReflectionBalance/std": 0.3462618589401245,
      "step": 481,
      "train_speed(iter/s)": 0.007086
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 480.7421875,
      "completions/min_length": 238.0,
      "epoch": 2.6037735849056602,
      "grad_norm": 2.9572030961840206,
      "kl": 2.58984375,
      "learning_rate": 8.503543362701588e-07,
      "loss": 0.24666531383991241,
      "memory(GiB)": 75.26,
      "reward": 1.8232275247573853,
      "reward_std": 0.5116055607795715,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2897924929857254,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3437131494283676,
      "rewards/ReflectionBalance/mean": 0.7802587747573853,
      "rewards/ReflectionBalance/std": 0.2607910931110382,
      "step": 482,
      "train_speed(iter/s)": 0.007086
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 487.12109375,
      "completions/min_length": 245.0,
      "epoch": 2.6091644204851754,
      "grad_norm": 2.770734195195225,
      "kl": 2.3515625,
      "learning_rate": 8.497417614806976e-07,
      "loss": 0.24844655394554138,
      "memory(GiB)": 75.26,
      "reward": 1.8438916206359863,
      "reward_std": 0.4922962486743927,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3270204961299896,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4065109044313431,
      "rewards/ReflectionBalance/mean": 0.7579541206359863,
      "rewards/ReflectionBalance/std": 0.2983347624540329,
      "step": 483,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 495.4921875,
      "completions/min_length": 256.0,
      "epoch": 2.61455525606469,
      "grad_norm": 3.290312332228826,
      "kl": 2.09765625,
      "learning_rate": 8.491281570867099e-07,
      "loss": 0.2151297777891159,
      "memory(GiB)": 75.26,
      "reward": 1.8910546898841858,
      "reward_std": 0.44820988178253174,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.36721789836883545,
      "rewards/ReflectionBalance/mean": 0.8051171898841858,
      "rewards/ReflectionBalance/std": 0.22618836909532547,
      "step": 484,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 484.3125,
      "completions/min_length": 255.5,
      "epoch": 2.6199460916442048,
      "grad_norm": 3.765994278031358,
      "kl": 2.125,
      "learning_rate": 8.485135248945849e-07,
      "loss": 0.23276159167289734,
      "memory(GiB)": 75.26,
      "reward": 1.810388207435608,
      "reward_std": 0.4792787730693817,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3174293115735054,
      "rewards/ReflectionBalance/mean": 0.7791381776332855,
      "rewards/ReflectionBalance/std": 0.23645007610321045,
      "step": 485,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1039.0,
      "completions/mean_length": 501.40625,
      "completions/min_length": 296.0,
      "epoch": 2.62533692722372,
      "grad_norm": 3.281869631653167,
      "kl": 2.7109375,
      "learning_rate": 8.478978667137381e-07,
      "loss": 0.2629432678222656,
      "memory(GiB)": 75.26,
      "reward": 1.728746771812439,
      "reward_std": 0.500563234090805,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3270204961299896,
      "rewards/MultiModalAccuracyORM/mean": 0.08984375,
      "rewards/MultiModalAccuracyORM/std": 0.2801405265927315,
      "rewards/ReflectionBalance/mean": 0.759996771812439,
      "rewards/ReflectionBalance/std": 0.2802766412496567,
      "step": 486,
      "train_speed(iter/s)": 0.007086
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 471.8046875,
      "completions/min_length": 245.5,
      "epoch": 2.6307277628032346,
      "grad_norm": 3.213281865563068,
      "kl": 3.21875,
      "learning_rate": 8.472811843566053e-07,
      "loss": 0.32983875274658203,
      "memory(GiB)": 75.26,
      "reward": 1.9111585021018982,
      "reward_std": 0.5365035235881805,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41857391595840454,
      "rewards/ReflectionBalance/mean": 0.793971061706543,
      "rewards/ReflectionBalance/std": 0.25035959482192993,
      "step": 487,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 546.515625,
      "completions/min_length": 299.0,
      "epoch": 2.6361185983827493,
      "grad_norm": 4.3205425447345505,
      "kl": 4.6640625,
      "learning_rate": 8.46663479638637e-07,
      "loss": 0.41543278098106384,
      "memory(GiB)": 75.26,
      "reward": 1.673963189125061,
      "reward_std": 0.6140182912349701,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.3634065240621567,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.3219822645187378,
      "rewards/ReflectionBalance/mean": 0.7169319689273834,
      "rewards/ReflectionBalance/std": 0.3529093712568283,
      "step": 488,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 484.3359375,
      "completions/min_length": 257.5,
      "epoch": 2.641509433962264,
      "grad_norm": 4.949686984574938,
      "kl": 3.328125,
      "learning_rate": 8.460447543782936e-07,
      "loss": 0.3682630658149719,
      "memory(GiB)": 75.26,
      "reward": 1.886059582233429,
      "reward_std": 0.5670019388198853,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41328008472919464,
      "rewards/ReflectionBalance/mean": 0.776684582233429,
      "rewards/ReflectionBalance/std": 0.26556794345378876,
      "step": 489,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1039.5,
      "completions/mean_length": 534.26953125,
      "completions/min_length": 283.5,
      "epoch": 2.6469002695417787,
      "grad_norm": 6.446956010379588,
      "kl": 6.0625,
      "learning_rate": 8.454250103970404e-07,
      "loss": 0.5323563814163208,
      "memory(GiB)": 75.26,
      "reward": 1.7279947996139526,
      "reward_std": 0.7126600444316864,
      "rewards/Format/mean": 0.80859375,
      "rewards/Format/std": 0.3946705758571625,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3973046988248825,
      "rewards/ReflectionBalance/mean": 0.7201822996139526,
      "rewards/ReflectionBalance/std": 0.32179005444049835,
      "step": 490,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 510.15234375,
      "completions/min_length": 220.0,
      "epoch": 2.652291105121294,
      "grad_norm": 6.245093274977375,
      "kl": 5.140625,
      "learning_rate": 8.448042495193408e-07,
      "loss": 0.4322138726711273,
      "memory(GiB)": 75.26,
      "reward": 1.7577571272850037,
      "reward_std": 0.601695328950882,
      "rewards/Format/mean": 0.828125,
      "rewards/Format/std": 0.3786114752292633,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3968885838985443,
      "rewards/ReflectionBalance/mean": 0.7343196868896484,
      "rewards/ReflectionBalance/std": 0.30576279759407043,
      "step": 491,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 491.203125,
      "completions/min_length": 286.5,
      "epoch": 2.6576819407008085,
      "grad_norm": 3.305076656411575,
      "kl": 3.3203125,
      "learning_rate": 8.441824735726528e-07,
      "loss": 0.34140142798423767,
      "memory(GiB)": 75.26,
      "reward": 1.8237964510917664,
      "reward_std": 0.5424772500991821,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.29259639233350754,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.33949992060661316,
      "rewards/ReflectionBalance/mean": 0.7808276414871216,
      "rewards/ReflectionBalance/std": 0.26758620142936707,
      "step": 492,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 484.1875,
      "completions/min_length": 224.0,
      "epoch": 2.6630727762803232,
      "grad_norm": 3.3775609705870173,
      "kl": 3.4375,
      "learning_rate": 8.435596843874217e-07,
      "loss": 0.30617499351501465,
      "memory(GiB)": 75.26,
      "reward": 1.788756549358368,
      "reward_std": 0.5529075264930725,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.330063596367836,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3678421527147293,
      "rewards/ReflectionBalance/mean": 0.7575065195560455,
      "rewards/ReflectionBalance/std": 0.2503872960805893,
      "step": 493,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 943.0,
      "completions/mean_length": 452.921875,
      "completions/min_length": 262.0,
      "epoch": 2.6684636118598384,
      "grad_norm": 2.5028765177827133,
      "kl": 1.046875,
      "learning_rate": 8.429358837970764e-07,
      "loss": 0.07508368045091629,
      "memory(GiB)": 75.26,
      "reward": 1.9208227396011353,
      "reward_std": 0.37746360898017883,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36417925357818604,
      "rewards/ReflectionBalance/mean": 0.7997289896011353,
      "rewards/ReflectionBalance/std": 0.18832243233919144,
      "step": 494,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 467.6015625,
      "completions/min_length": 263.0,
      "epoch": 2.673854447439353,
      "grad_norm": 2.944689766593109,
      "kl": 2.01171875,
      "learning_rate": 8.42311073638023e-07,
      "loss": 0.20560044050216675,
      "memory(GiB)": 75.26,
      "reward": 1.8646403551101685,
      "reward_std": 0.4200490564107895,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.36721789836883545,
      "rewards/ReflectionBalance/mean": 0.7865152955055237,
      "rewards/ReflectionBalance/std": 0.25211460143327713,
      "step": 495,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 494.19140625,
      "completions/min_length": 260.5,
      "epoch": 2.6792452830188678,
      "grad_norm": 3.7879357538223464,
      "kl": 1.46875,
      "learning_rate": 8.416852557496396e-07,
      "loss": 0.1865183413028717,
      "memory(GiB)": 75.26,
      "reward": 1.8446110486984253,
      "reward_std": 0.42840123176574707,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.26531852036714554,
      "rewards/ReflectionBalance/mean": 0.8211734890937805,
      "rewards/ReflectionBalance/std": 0.19740112125873566,
      "step": 496,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 468.2578125,
      "completions/min_length": 249.5,
      "epoch": 2.684636118598383,
      "grad_norm": 2.512485459219885,
      "kl": 0.9921875,
      "learning_rate": 8.410584319742713e-07,
      "loss": 0.10589855909347534,
      "memory(GiB)": 75.26,
      "reward": 1.8879736065864563,
      "reward_std": 0.3512592762708664,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20146947354078293,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3035084083676338,
      "rewards/ReflectionBalance/mean": 0.8176611363887787,
      "rewards/ReflectionBalance/std": 0.20933584868907928,
      "step": 497,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 483.96875,
      "completions/min_length": 280.5,
      "epoch": 2.6900269541778976,
      "grad_norm": 3.121381384617055,
      "kl": 2.2578125,
      "learning_rate": 8.404306041572238e-07,
      "loss": 0.2397855669260025,
      "memory(GiB)": 75.26,
      "reward": 1.8554964065551758,
      "reward_std": 0.5549950003623962,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2875154986977577,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3897709399461746,
      "rewards/ReflectionBalance/mean": 0.7617464065551758,
      "rewards/ReflectionBalance/std": 0.26237571239471436,
      "step": 498,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 467.82421875,
      "completions/min_length": 297.0,
      "epoch": 2.6954177897574123,
      "grad_norm": 5.205262844816082,
      "kl": 2.171875,
      "learning_rate": 8.398017741467591e-07,
      "loss": 0.22465476393699646,
      "memory(GiB)": 75.26,
      "reward": 1.9234228730201721,
      "reward_std": 0.5022670179605484,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4060489982366562,
      "rewards/ReflectionBalance/mean": 0.7906103432178497,
      "rewards/ReflectionBalance/std": 0.2534360811114311,
      "step": 499,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 489.62890625,
      "completions/min_length": 241.0,
      "epoch": 2.7008086253369274,
      "grad_norm": 3.0808229549769828,
      "kl": 3.5390625,
      "learning_rate": 8.391719437940896e-07,
      "loss": 0.33432525396347046,
      "memory(GiB)": 75.26,
      "reward": 1.823375642299652,
      "reward_std": 0.5762150287628174,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.3274291008710861,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.39205317199230194,
      "rewards/ReflectionBalance/mean": 0.7530631721019745,
      "rewards/ReflectionBalance/std": 0.2899813801050186,
      "step": 500,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 498.55078125,
      "completions/min_length": 268.0,
      "epoch": 2.706199460916442,
      "grad_norm": 3.9936965019192487,
      "kl": 4.390625,
      "learning_rate": 8.385411149533723e-07,
      "loss": 0.3961998224258423,
      "memory(GiB)": 75.26,
      "reward": 1.795739769935608,
      "reward_std": 0.5830785036087036,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.35675957798957825,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.37864522635936737,
      "rewards/ReflectionBalance/mean": 0.7644897699356079,
      "rewards/ReflectionBalance/std": 0.29122816026210785,
      "step": 501,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 531.62109375,
      "completions/min_length": 272.0,
      "epoch": 2.711590296495957,
      "grad_norm": 7.924214290208419,
      "kl": 5.390625,
      "learning_rate": 8.379092894817039e-07,
      "loss": 0.42847222089767456,
      "memory(GiB)": 75.26,
      "reward": 1.6676050424575806,
      "reward_std": 0.6147034168243408,
      "rewards/Format/mean": 0.8203125,
      "rewards/Format/std": 0.38325364887714386,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3142240568995476,
      "rewards/ReflectionBalance/mean": 0.7261987626552582,
      "rewards/ReflectionBalance/std": 0.32503870129585266,
      "step": 502,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 501.91796875,
      "completions/min_length": 243.5,
      "epoch": 2.7169811320754715,
      "grad_norm": 5.0363056585728,
      "kl": 4.6015625,
      "learning_rate": 8.372764692391147e-07,
      "loss": 0.40052592754364014,
      "memory(GiB)": 75.26,
      "reward": 1.7590771317481995,
      "reward_std": 0.6341991424560547,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.3634065240621567,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3897709399461746,
      "rewards/ReflectionBalance/mean": 0.7317334115505219,
      "rewards/ReflectionBalance/std": 0.32499177753925323,
      "step": 503,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 533.84375,
      "completions/min_length": 289.5,
      "epoch": 2.7223719676549867,
      "grad_norm": 6.859966872383644,
      "kl": 5.890625,
      "learning_rate": 8.36642656088564e-07,
      "loss": 0.4940209984779358,
      "memory(GiB)": 75.26,
      "reward": 1.6758642792701721,
      "reward_std": 0.6708246469497681,
      "rewards/Format/mean": 0.82421875,
      "rewards/Format/std": 0.38125884532928467,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.3191515803337097,
      "rewards/ReflectionBalance/mean": 0.7344579994678497,
      "rewards/ReflectionBalance/std": 0.3103443384170532,
      "step": 504,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 508.3125,
      "completions/min_length": 270.0,
      "epoch": 2.7277628032345014,
      "grad_norm": 3.426572098142761,
      "kl": 3.90625,
      "learning_rate": 8.360078518959339e-07,
      "loss": 0.38355016708374023,
      "memory(GiB)": 75.26,
      "reward": 1.7830175757408142,
      "reward_std": 0.6330873668193817,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.3630705028772354,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.38336627185344696,
      "rewards/ReflectionBalance/mean": 0.7517675757408142,
      "rewards/ReflectionBalance/std": 0.2862914055585861,
      "step": 505,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 477.75390625,
      "completions/min_length": 247.0,
      "epoch": 2.733153638814016,
      "grad_norm": 3.822995141663451,
      "kl": 2.8203125,
      "learning_rate": 8.353720585300238e-07,
      "loss": 0.3108353018760681,
      "memory(GiB)": 75.26,
      "reward": 1.8764469027519226,
      "reward_std": 0.4629902243614197,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.36588746309280396,
      "rewards/ReflectionBalance/mean": 0.802228182554245,
      "rewards/ReflectionBalance/std": 0.23509268462657928,
      "step": 506,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 469.53125,
      "completions/min_length": 217.5,
      "epoch": 2.7385444743935308,
      "grad_norm": 4.360845694908617,
      "kl": 1.953125,
      "learning_rate": 8.347352778625455e-07,
      "loss": 0.2101874202489853,
      "memory(GiB)": 75.26,
      "reward": 1.793613314628601,
      "reward_std": 0.43798407912254333,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.0859375,
      "rewards/MultiModalAccuracyORM/std": 0.2810242921113968,
      "rewards/ReflectionBalance/mean": 0.7858007848262787,
      "rewards/ReflectionBalance/std": 0.2600787431001663,
      "step": 507,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 473.671875,
      "completions/min_length": 226.5,
      "epoch": 2.743935309973046,
      "grad_norm": 3.754052445544739,
      "kl": 2.53515625,
      "learning_rate": 8.340975117681173e-07,
      "loss": 0.26354679465293884,
      "memory(GiB)": 75.26,
      "reward": 1.8783869743347168,
      "reward_std": 0.576628565788269,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3948954641819,
      "rewards/ReflectionBalance/mean": 0.7729182541370392,
      "rewards/ReflectionBalance/std": 0.28091733157634735,
      "step": 508,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 470.1875,
      "completions/min_length": 226.0,
      "epoch": 2.7493261455525606,
      "grad_norm": 3.4220559432176554,
      "kl": 2.04296875,
      "learning_rate": 8.334587621242579e-07,
      "loss": 0.22252042591571808,
      "memory(GiB)": 75.26,
      "reward": 1.9327466487884521,
      "reward_std": 0.49468693137168884,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26276856660842896,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.40794737637043,
      "rewards/ReflectionBalance/mean": 0.7921215891838074,
      "rewards/ReflectionBalance/std": 0.2541298121213913,
      "step": 509,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 484.8203125,
      "completions/min_length": 275.5,
      "epoch": 2.7547169811320753,
      "grad_norm": 3.5442493981488985,
      "kl": 2.8671875,
      "learning_rate": 8.328190308113825e-07,
      "loss": 0.2883029580116272,
      "memory(GiB)": 75.26,
      "reward": 1.8349186182022095,
      "reward_std": 0.5047230571508408,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3050640821456909,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36417925357818604,
      "rewards/ReflectionBalance/mean": 0.7763248682022095,
      "rewards/ReflectionBalance/std": 0.2606455311179161,
      "step": 510,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 504.2109375,
      "completions/min_length": 257.5,
      "epoch": 2.7601078167115904,
      "grad_norm": 3.2813325546049454,
      "kl": 4.34375,
      "learning_rate": 8.321783197127952e-07,
      "loss": 0.38877469301223755,
      "memory(GiB)": 75.26,
      "reward": 1.7097070217132568,
      "reward_std": 0.6779496073722839,
      "rewards/Format/mean": 0.80078125,
      "rewards/Format/std": 0.38974663615226746,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39238542318344116,
      "rewards/ReflectionBalance/mean": 0.7175195217132568,
      "rewards/ReflectionBalance/std": 0.3394276797771454,
      "step": 511,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 484.00390625,
      "completions/min_length": 242.5,
      "epoch": 2.765498652291105,
      "grad_norm": 3.921042580381599,
      "kl": 4.1015625,
      "learning_rate": 8.315366307146853e-07,
      "loss": 0.3713727593421936,
      "memory(GiB)": 75.26,
      "reward": 1.772670865058899,
      "reward_std": 0.5784733593463898,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.34447427093982697,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3787541687488556,
      "rewards/ReflectionBalance/mean": 0.7375146448612213,
      "rewards/ReflectionBalance/std": 0.3020561933517456,
      "step": 512,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 462.6328125,
      "completions/min_length": 251.5,
      "epoch": 2.77088948787062,
      "grad_norm": 6.242922562420631,
      "kl": 3.171875,
      "learning_rate": 8.308939657061204e-07,
      "loss": 0.2591054141521454,
      "memory(GiB)": 75.26,
      "reward": 1.8872394561767578,
      "reward_std": 0.5916838049888611,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3270204961299896,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.41856054961681366,
      "rewards/ReflectionBalance/mean": 0.7700521051883698,
      "rewards/ReflectionBalance/std": 0.2672806531190872,
      "step": 513,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 466.40234375,
      "completions/min_length": 265.0,
      "epoch": 2.776280323450135,
      "grad_norm": 3.452967593973878,
      "kl": 1.30078125,
      "learning_rate": 8.302503265790416e-07,
      "loss": 0.14514058828353882,
      "memory(GiB)": 75.26,
      "reward": 2.032641589641571,
      "reward_std": 0.39473100006580353,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23196382820606232,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.4418751001358032,
      "rewards/ReflectionBalance/mean": 0.817797839641571,
      "rewards/ReflectionBalance/std": 0.20321736484766006,
      "step": 514,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 436.89453125,
      "completions/min_length": 261.5,
      "epoch": 2.7816711590296497,
      "grad_norm": 2.9407164895350753,
      "kl": 1.373046875,
      "learning_rate": 8.296057152282576e-07,
      "loss": 0.13772064447402954,
      "memory(GiB)": 75.26,
      "reward": 2.0305079221725464,
      "reward_std": 0.39611563086509705,
      "rewards/Format/mean": 0.9765625,
      "rewards/Format/std": 0.14144666120409966,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42526112496852875,
      "rewards/ReflectionBalance/mean": 0.8117578029632568,
      "rewards/ReflectionBalance/std": 0.2036316841840744,
      "step": 515,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 497.2421875,
      "completions/min_length": 249.0,
      "epoch": 2.7870619946091644,
      "grad_norm": 3.963122611996376,
      "kl": 2.05078125,
      "learning_rate": 8.289601335514396e-07,
      "loss": 0.2207166850566864,
      "memory(GiB)": 75.26,
      "reward": 1.7868587374687195,
      "reward_std": 0.5561719536781311,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.32050570100545883,
      "rewards/ReflectionBalance/mean": 0.7634212374687195,
      "rewards/ReflectionBalance/std": 0.2704896181821823,
      "step": 516,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 492.1640625,
      "completions/min_length": 272.5,
      "epoch": 2.7924528301886795,
      "grad_norm": 2.4197942116265265,
      "kl": 2.1640625,
      "learning_rate": 8.283135834491147e-07,
      "loss": 0.21382629871368408,
      "memory(GiB)": 75.26,
      "reward": 1.848426103591919,
      "reward_std": 0.49269257485866547,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38597656786441803,
      "rewards/ReflectionBalance/mean": 0.758582353591919,
      "rewards/ReflectionBalance/std": 0.29721061140298843,
      "step": 517,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 438.5390625,
      "completions/min_length": 222.0,
      "epoch": 2.797843665768194,
      "grad_norm": 3.4185780507437116,
      "kl": 2.09765625,
      "learning_rate": 8.276660668246613e-07,
      "loss": 0.17719602584838867,
      "memory(GiB)": 75.26,
      "reward": 1.8710530400276184,
      "reward_std": 0.4146542549133301,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3613582104444504,
      "rewards/ReflectionBalance/mean": 0.7773030400276184,
      "rewards/ReflectionBalance/std": 0.24135740101337433,
      "step": 518,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 490.33203125,
      "completions/min_length": 246.0,
      "epoch": 2.803234501347709,
      "grad_norm": 2.6044433409896857,
      "kl": 3.015625,
      "learning_rate": 8.270175855843035e-07,
      "loss": 0.2932540774345398,
      "memory(GiB)": 75.26,
      "reward": 1.8239192962646484,
      "reward_std": 0.5905983448028564,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38201163709163666,
      "rewards/ReflectionBalance/mean": 0.7497005462646484,
      "rewards/ReflectionBalance/std": 0.28297059237957,
      "step": 519,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1051.5,
      "completions/mean_length": 475.65234375,
      "completions/min_length": 231.0,
      "epoch": 2.8086253369272236,
      "grad_norm": 4.467001599072201,
      "kl": 3.1015625,
      "learning_rate": 8.263681416371044e-07,
      "loss": 0.2670515775680542,
      "memory(GiB)": 75.26,
      "reward": 1.8090096712112427,
      "reward_std": 0.5186476409435272,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.36145633459091187,
      "rewards/ReflectionBalance/mean": 0.7582283616065979,
      "rewards/ReflectionBalance/std": 0.2943470776081085,
      "step": 520,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 464.5859375,
      "completions/min_length": 241.5,
      "epoch": 2.8140161725067383,
      "grad_norm": 3.378380415776508,
      "kl": 2.828125,
      "learning_rate": 8.257177368949618e-07,
      "loss": 0.28688913583755493,
      "memory(GiB)": 75.26,
      "reward": 1.915590763092041,
      "reward_std": 0.6348106265068054,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.42696477472782135,
      "rewards/ReflectionBalance/mean": 0.7710595726966858,
      "rewards/ReflectionBalance/std": 0.292088583111763,
      "step": 521,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 508.578125,
      "completions/min_length": 235.0,
      "epoch": 2.8194070080862534,
      "grad_norm": 5.171628125786674,
      "kl": 4.7265625,
      "learning_rate": 8.250663732726015e-07,
      "loss": 0.39634501934051514,
      "memory(GiB)": 75.26,
      "reward": 1.7142146825790405,
      "reward_std": 0.6281823813915253,
      "rewards/Format/mean": 0.8046875,
      "rewards/Format/std": 0.39185720682144165,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3934052139520645,
      "rewards/ReflectionBalance/mean": 0.7181209623813629,
      "rewards/ReflectionBalance/std": 0.32760611176490784,
      "step": 522,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 510.89453125,
      "completions/min_length": 254.0,
      "epoch": 2.824797843665768,
      "grad_norm": 6.738449965581487,
      "kl": 6.46875,
      "learning_rate": 8.244140526875728e-07,
      "loss": 0.5578604936599731,
      "memory(GiB)": 75.26,
      "reward": 1.6300617456436157,
      "reward_std": 0.7361034154891968,
      "rewards/Format/mean": 0.765625,
      "rewards/Format/std": 0.42527204751968384,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3842122554779053,
      "rewards/ReflectionBalance/mean": 0.6847493350505829,
      "rewards/ReflectionBalance/std": 0.3513321429491043,
      "step": 523,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 516.0703125,
      "completions/min_length": 184.0,
      "epoch": 2.830188679245283,
      "grad_norm": 3.9020544873372423,
      "kl": 4.1875,
      "learning_rate": 8.237607770602415e-07,
      "loss": 0.3722361922264099,
      "memory(GiB)": 75.26,
      "reward": 1.6205435991287231,
      "reward_std": 0.5928677320480347,
      "rewards/Format/mean": 0.828125,
      "rewards/Format/std": 0.3764529526233673,
      "rewards/MultiModalAccuracyORM/mean": 0.08203125,
      "rewards/MultiModalAccuracyORM/std": 0.26326169073581696,
      "rewards/ReflectionBalance/mean": 0.7103873491287231,
      "rewards/ReflectionBalance/std": 0.32548247277736664,
      "step": 524,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 499.58984375,
      "completions/min_length": 263.0,
      "epoch": 2.835579514824798,
      "grad_norm": 2.6851246313244452,
      "kl": 3.3046875,
      "learning_rate": 8.231065483137853e-07,
      "loss": 0.30486106872558594,
      "memory(GiB)": 75.26,
      "reward": 1.7322688698768616,
      "reward_std": 0.5580596923828125,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.3562464565038681,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.30072054266929626,
      "rewards/ReflectionBalance/mean": 0.7791438698768616,
      "rewards/ReflectionBalance/std": 0.27908600866794586,
      "step": 525,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 497.34765625,
      "completions/min_length": 267.5,
      "epoch": 2.8409703504043127,
      "grad_norm": 3.36709393711364,
      "kl": 3.3671875,
      "learning_rate": 8.224513683741881e-07,
      "loss": 0.3096576929092407,
      "memory(GiB)": 75.26,
      "reward": 1.7113679647445679,
      "reward_std": 0.5530379861593246,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.3410066068172455,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.3021402209997177,
      "rewards/ReflectionBalance/mean": 0.7465242743492126,
      "rewards/ReflectionBalance/std": 0.30153754353523254,
      "step": 526,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 458.81640625,
      "completions/min_length": 272.5,
      "epoch": 2.8463611859838274,
      "grad_norm": 4.846766546388782,
      "kl": 2.03125,
      "learning_rate": 8.217952391702332e-07,
      "loss": 0.21700644493103027,
      "memory(GiB)": 75.26,
      "reward": 1.8879061341285706,
      "reward_std": 0.44403699040412903,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3717130124568939,
      "rewards/ReflectionBalance/mean": 0.7863435745239258,
      "rewards/ReflectionBalance/std": 0.22691578418016434,
      "step": 527,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 477.4375,
      "completions/min_length": 261.5,
      "epoch": 2.8517520215633425,
      "grad_norm": 5.304764135758909,
      "kl": 2.23828125,
      "learning_rate": 8.21138162633499e-07,
      "loss": 0.23765483498573303,
      "memory(GiB)": 75.26,
      "reward": 1.918860673904419,
      "reward_std": 0.5181988328695297,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4120787978172302,
      "rewards/ReflectionBalance/mean": 0.786048173904419,
      "rewards/ReflectionBalance/std": 0.25061970204114914,
      "step": 528,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 484.78125,
      "completions/min_length": 257.5,
      "epoch": 2.857142857142857,
      "grad_norm": 3.7253200425837245,
      "kl": 1.59375,
      "learning_rate": 8.204801406983529e-07,
      "loss": 0.1668206751346588,
      "memory(GiB)": 75.26,
      "reward": 1.9240120649337769,
      "reward_std": 0.43622612953186035,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3968885838985443,
      "rewards/ReflectionBalance/mean": 0.7990120351314545,
      "rewards/ReflectionBalance/std": 0.22848013043403625,
      "step": 529,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 451.7265625,
      "completions/min_length": 247.0,
      "epoch": 2.862533692722372,
      "grad_norm": 3.597453689191808,
      "kl": 1.259765625,
      "learning_rate": 8.19821175301945e-07,
      "loss": 0.1600641906261444,
      "memory(GiB)": 75.26,
      "reward": 1.9285937547683716,
      "reward_std": 0.4415707141160965,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21065277606248856,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3734995424747467,
      "rewards/ReflectionBalance/mean": 0.811406284570694,
      "rewards/ReflectionBalance/std": 0.2258257046341896,
      "step": 530,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 453.74609375,
      "completions/min_length": 286.5,
      "epoch": 2.867924528301887,
      "grad_norm": 2.5721228929060116,
      "kl": 1.2890625,
      "learning_rate": 8.191612683842031e-07,
      "loss": 0.14296866953372955,
      "memory(GiB)": 75.26,
      "reward": 1.9895873069763184,
      "reward_std": 0.35591891407966614,
      "rewards/Format/mean": 0.96484375,
      "rewards/Format/std": 0.1845906376838684,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4018339514732361,
      "rewards/ReflectionBalance/mean": 0.8177123963832855,
      "rewards/ReflectionBalance/std": 0.21077822148799896,
      "step": 531,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 499.94921875,
      "completions/min_length": 277.0,
      "epoch": 2.8733153638814017,
      "grad_norm": 4.392402962128204,
      "kl": 2.23046875,
      "learning_rate": 8.185004218878267e-07,
      "loss": 0.1952047049999237,
      "memory(GiB)": 75.26,
      "reward": 1.8631331324577332,
      "reward_std": 0.4738515615463257,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2676052302122116,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35737672448158264,
      "rewards/ReflectionBalance/mean": 0.7967269122600555,
      "rewards/ReflectionBalance/std": 0.2609056234359741,
      "step": 532,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 498.78125,
      "completions/min_length": 243.0,
      "epoch": 2.8787061994609164,
      "grad_norm": 2.5597719476972856,
      "kl": 2.80859375,
      "learning_rate": 8.178386377582816e-07,
      "loss": 0.28146713972091675,
      "memory(GiB)": 75.26,
      "reward": 1.8698087334632874,
      "reward_std": 0.6037485748529434,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3169707804918289,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41635647416114807,
      "rewards/ReflectionBalance/mean": 0.7643399834632874,
      "rewards/ReflectionBalance/std": 0.2858271896839142,
      "step": 533,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 506.3828125,
      "completions/min_length": 282.5,
      "epoch": 2.884097035040431,
      "grad_norm": 5.377697431855948,
      "kl": 4.4375,
      "learning_rate": 8.171759179437931e-07,
      "loss": 0.36704179644584656,
      "memory(GiB)": 75.26,
      "reward": 1.737389326095581,
      "reward_std": 0.622816413640976,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.35190291702747345,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3497565984725952,
      "rewards/ReflectionBalance/mean": 0.7217642962932587,
      "rewards/ReflectionBalance/std": 0.341605469584465,
      "step": 534,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 487.9765625,
      "completions/min_length": 264.5,
      "epoch": 2.889487870619946,
      "grad_norm": 3.0834152956935954,
      "kl": 3.6484375,
      "learning_rate": 8.16512264395342e-07,
      "loss": 0.3352925181388855,
      "memory(GiB)": 75.26,
      "reward": 1.8204426765441895,
      "reward_std": 0.5118741989135742,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3068038523197174,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3110755831003189,
      "rewards/ReflectionBalance/mean": 0.7774739265441895,
      "rewards/ReflectionBalance/std": 0.24946225434541702,
      "step": 535,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 498.703125,
      "completions/min_length": 294.5,
      "epoch": 2.894878706199461,
      "grad_norm": 4.070083162480362,
      "kl": 3.1015625,
      "learning_rate": 8.158476790666575e-07,
      "loss": 0.25091421604156494,
      "memory(GiB)": 75.26,
      "reward": 1.7811197638511658,
      "reward_std": 0.5210830569267273,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.305100217461586,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.34483732283115387,
      "rewards/ReflectionBalance/mean": 0.7498697638511658,
      "rewards/ReflectionBalance/std": 0.2913997396826744,
      "step": 536,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.0,
      "completions/mean_length": 498.671875,
      "completions/min_length": 249.0,
      "epoch": 2.9002695417789757,
      "grad_norm": 3.6101134571555704,
      "kl": 3.6484375,
      "learning_rate": 8.151821639142118e-07,
      "loss": 0.30638331174850464,
      "memory(GiB)": 75.26,
      "reward": 1.77938312292099,
      "reward_std": 0.5400216728448868,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.33116789162158966,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3410066068172455,
      "rewards/ReflectionBalance/mean": 0.7676644325256348,
      "rewards/ReflectionBalance/std": 0.2584993690252304,
      "step": 537,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 464.78515625,
      "completions/min_length": 277.0,
      "epoch": 2.9056603773584904,
      "grad_norm": 2.665842940309033,
      "kl": 2.48828125,
      "learning_rate": 8.145157208972144e-07,
      "loss": 0.235057070851326,
      "memory(GiB)": 75.26,
      "reward": 1.9086686372756958,
      "reward_std": 0.5018502324819565,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4091253876686096,
      "rewards/ReflectionBalance/mean": 0.7953873574733734,
      "rewards/ReflectionBalance/std": 0.25174374878406525,
      "step": 538,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 476.96875,
      "completions/min_length": 274.0,
      "epoch": 2.9110512129380055,
      "grad_norm": 2.4827570225858264,
      "kl": 2.765625,
      "learning_rate": 8.138483519776067e-07,
      "loss": 0.2582785189151764,
      "memory(GiB)": 75.26,
      "reward": 1.7839567065238953,
      "reward_std": 0.5519320964813232,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3219822645187378,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35259076952934265,
      "rewards/ReflectionBalance/mean": 0.7527067065238953,
      "rewards/ReflectionBalance/std": 0.2899843156337738,
      "step": 539,
      "train_speed(iter/s)": 0.007094
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 482.14453125,
      "completions/min_length": 283.0,
      "epoch": 2.91644204851752,
      "grad_norm": 2.572270068953206,
      "kl": 2.0,
      "learning_rate": 8.131800591200553e-07,
      "loss": 0.1921999454498291,
      "memory(GiB)": 75.26,
      "reward": 1.9533674716949463,
      "reward_std": 0.4810611307621002,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41762344539165497,
      "rewards/ReflectionBalance/mean": 0.7932112514972687,
      "rewards/ReflectionBalance/std": 0.22115272283554077,
      "step": 540,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 448.91796875,
      "completions/min_length": 254.5,
      "epoch": 2.921832884097035,
      "grad_norm": 2.5538475807188687,
      "kl": 0.984375,
      "learning_rate": 8.125108442919473e-07,
      "loss": 0.09770803898572922,
      "memory(GiB)": 75.26,
      "reward": 1.9143521785736084,
      "reward_std": 0.3479805886745453,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21875878423452377,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.330063596367836,
      "rewards/ReflectionBalance/mean": 0.8362272083759308,
      "rewards/ReflectionBalance/std": 0.1753830760717392,
      "step": 541,
      "train_speed(iter/s)": 0.007094
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1020.5,
      "completions/mean_length": 453.20703125,
      "completions/min_length": 243.5,
      "epoch": 2.92722371967655,
      "grad_norm": 6.484045735929777,
      "kl": 0.826171875,
      "learning_rate": 8.118407094633835e-07,
      "loss": 0.07458211481571198,
      "memory(GiB)": 75.26,
      "reward": 2.0255484580993652,
      "reward_std": 0.349159374833107,
      "rewards/Format/mean": 0.97265625,
      "rewards/Format/std": 0.1595059372484684,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.3986760377883911,
      "rewards/ReflectionBalance/mean": 0.8341422379016876,
      "rewards/ReflectionBalance/std": 0.16482903808355331,
      "step": 542,
      "train_speed(iter/s)": 0.007094
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 495.34375,
      "completions/min_length": 273.0,
      "epoch": 2.9326145552560647,
      "grad_norm": 3.1676690079914054,
      "kl": 0.998046875,
      "learning_rate": 8.111696566071738e-07,
      "loss": 0.11417896300554276,
      "memory(GiB)": 75.26,
      "reward": 1.8529508113861084,
      "reward_std": 0.36066415905952454,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22826264798641205,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.2961650490760803,
      "rewards/ReflectionBalance/mean": 0.8099820911884308,
      "rewards/ReflectionBalance/std": 0.22337737679481506,
      "step": 543,
      "train_speed(iter/s)": 0.007094
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 460.3359375,
      "completions/min_length": 233.0,
      "epoch": 2.9380053908355794,
      "grad_norm": 3.616373122776003,
      "kl": 1.2734375,
      "learning_rate": 8.104976876988298e-07,
      "loss": 0.15219154953956604,
      "memory(GiB)": 75.26,
      "reward": 1.8732048273086548,
      "reward_std": 0.40574416518211365,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.20884444564580917,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3229459524154663,
      "rewards/ReflectionBalance/mean": 0.79898601770401,
      "rewards/ReflectionBalance/std": 0.24766093492507935,
      "step": 544,
      "train_speed(iter/s)": 0.007094
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 896.5,
      "completions/mean_length": 435.546875,
      "completions/min_length": 231.5,
      "epoch": 2.9433962264150946,
      "grad_norm": 2.4536826176606557,
      "kl": 0.603515625,
      "learning_rate": 8.098248047165604e-07,
      "loss": 0.046754539012908936,
      "memory(GiB)": 75.26,
      "reward": 2.0796384811401367,
      "reward_std": 0.4343741536140442,
      "rewards/Format/mean": 0.96875,
      "rewards/Format/std": 0.17319392412900925,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.45324206352233887,
      "rewards/ReflectionBalance/mean": 0.825732409954071,
      "rewards/ReflectionBalance/std": 0.17527878284454346,
      "step": 545,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 451.8359375,
      "completions/min_length": 273.0,
      "epoch": 2.9487870619946093,
      "grad_norm": 3.1415332832540077,
      "kl": 1.140625,
      "learning_rate": 8.091510096412653e-07,
      "loss": 0.12095971405506134,
      "memory(GiB)": 75.26,
      "reward": 1.9891226887702942,
      "reward_std": 0.40983518958091736,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22826264798641205,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4312489330768585,
      "rewards/ReflectionBalance/mean": 0.7977164685726166,
      "rewards/ReflectionBalance/std": 0.23716706782579422,
      "step": 546,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 483.83984375,
      "completions/min_length": 271.0,
      "epoch": 2.954177897574124,
      "grad_norm": 4.446070773461159,
      "kl": 2.46875,
      "learning_rate": 8.084763044565297e-07,
      "loss": 0.22690477967262268,
      "memory(GiB)": 75.26,
      "reward": 1.836241900920868,
      "reward_std": 0.5125428736209869,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3050640821456909,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3409748077392578,
      "rewards/ReflectionBalance/mean": 0.7893668711185455,
      "rewards/ReflectionBalance/std": 0.2474120780825615,
      "step": 547,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 488.70703125,
      "completions/min_length": 222.5,
      "epoch": 2.9595687331536387,
      "grad_norm": 4.450259885208727,
      "kl": 3.8515625,
      "learning_rate": 8.078006911486173e-07,
      "loss": 0.33413994312286377,
      "memory(GiB)": 75.26,
      "reward": 1.7148136496543884,
      "reward_std": 0.5496651232242584,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3300945907831192,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.29639076441526413,
      "rewards/ReflectionBalance/mean": 0.734344869852066,
      "rewards/ReflectionBalance/std": 0.31993621587753296,
      "step": 548,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 507.81640625,
      "completions/min_length": 226.0,
      "epoch": 2.964959568733154,
      "grad_norm": 4.684681923559452,
      "kl": 4.625,
      "learning_rate": 8.071241717064657e-07,
      "loss": 0.399535208940506,
      "memory(GiB)": 75.26,
      "reward": 1.7370336651802063,
      "reward_std": 0.6220909059047699,
      "rewards/Format/mean": 0.8359375,
      "rewards/Format/std": 0.3704231530427933,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3752707988023758,
      "rewards/ReflectionBalance/mean": 0.7331274151802063,
      "rewards/ReflectionBalance/std": 0.3135607838630676,
      "step": 549,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 498.30859375,
      "completions/min_length": 294.0,
      "epoch": 2.9703504043126685,
      "grad_norm": 2.68719086891706,
      "kl": 2.7265625,
      "learning_rate": 8.064467481216804e-07,
      "loss": 0.26047709584236145,
      "memory(GiB)": 75.26,
      "reward": 1.8877319693565369,
      "reward_std": 0.5001082122325897,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.40095221996307373,
      "rewards/ReflectionBalance/mean": 0.7783569395542145,
      "rewards/ReflectionBalance/std": 0.27488337457180023,
      "step": 550,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 512.875,
      "completions/min_length": 249.0,
      "epoch": 2.975741239892183,
      "grad_norm": 2.738405255423716,
      "kl": 3.546875,
      "learning_rate": 8.057684223885278e-07,
      "loss": 0.32487767934799194,
      "memory(GiB)": 75.26,
      "reward": 1.7463948130607605,
      "reward_std": 0.6019405424594879,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3460448384284973,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3389271944761276,
      "rewards/ReflectionBalance/mean": 0.7542073726654053,
      "rewards/ReflectionBalance/std": 0.29625822603702545,
      "step": 551,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 511.0234375,
      "completions/min_length": 251.5,
      "epoch": 2.981132075471698,
      "grad_norm": 3.075545662859188,
      "kl": 2.99609375,
      "learning_rate": 8.050891965039306e-07,
      "loss": 0.24698352813720703,
      "memory(GiB)": 75.26,
      "reward": 1.8171549439430237,
      "reward_std": 0.5422500967979431,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.31005433946847916,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39238542318344116,
      "rewards/ReflectionBalance/mean": 0.7468424439430237,
      "rewards/ReflectionBalance/std": 0.3004668429493904,
      "step": 552,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 452.21875,
      "completions/min_length": 247.5,
      "epoch": 2.986522911051213,
      "grad_norm": 36.66178982878202,
      "kl": 1.44921875,
      "learning_rate": 8.044090724674615e-07,
      "loss": 0.12269224226474762,
      "memory(GiB)": 75.26,
      "reward": 1.9191503524780273,
      "reward_std": 0.379862442612648,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2524137645959854,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3853000998497009,
      "rewards/ReflectionBalance/mean": 0.8097753822803497,
      "rewards/ReflectionBalance/std": 0.21529348194599152,
      "step": 553,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 490.21875,
      "completions/min_length": 271.0,
      "epoch": 2.9919137466307277,
      "grad_norm": 2.762136878196533,
      "kl": 1.8515625,
      "learning_rate": 8.037280522813373e-07,
      "loss": 0.1800614893436432,
      "memory(GiB)": 75.26,
      "reward": 1.8260921239852905,
      "reward_std": 0.46529729664325714,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.2899702489376068,
      "rewards/ReflectionBalance/mean": 0.7987483441829681,
      "rewards/ReflectionBalance/std": 0.24145478755235672,
      "step": 554,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 469.89453125,
      "completions/min_length": 213.0,
      "epoch": 2.9973045822102424,
      "grad_norm": 3.2570415859273876,
      "kl": 1.44921875,
      "learning_rate": 8.030461379504125e-07,
      "loss": 0.16594873368740082,
      "memory(GiB)": 75.26,
      "reward": 1.9593725800514221,
      "reward_std": 0.39280757308006287,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23443959653377533,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.4009389728307724,
      "rewards/ReflectionBalance/mean": 0.8148413002490997,
      "rewards/ReflectionBalance/std": 0.22719041258096695,
      "step": 555,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 458.703125,
      "completions/min_length": 261.5,
      "epoch": 3.0053908355795147,
      "grad_norm": 2.579533200654342,
      "kl": 1.5,
      "learning_rate": 8.023633314821746e-07,
      "loss": 0.12553781270980835,
      "memory(GiB)": 75.26,
      "reward": 1.9263948202133179,
      "reward_std": 0.4253968894481659,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24081215262413025,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.36930377781391144,
      "rewards/ReflectionBalance/mean": 0.8170198500156403,
      "rewards/ReflectionBalance/std": 0.19537164270877838,
      "step": 556,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 470.15234375,
      "completions/min_length": 269.0,
      "epoch": 3.01078167115903,
      "grad_norm": 2.716881692559698,
      "kl": 2.1484375,
      "learning_rate": 8.01679634886737e-07,
      "loss": 0.20105917751789093,
      "memory(GiB)": 75.26,
      "reward": 1.9270166158676147,
      "reward_std": 0.4428151100873947,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3899272531270981,
      "rewards/ReflectionBalance/mean": 0.8215478658676147,
      "rewards/ReflectionBalance/std": 0.21965373307466507,
      "step": 557,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 485.0625,
      "completions/min_length": 275.0,
      "epoch": 3.0161725067385445,
      "grad_norm": 2.652776394421451,
      "kl": 1.53125,
      "learning_rate": 8.009950501768334e-07,
      "loss": 0.13814882934093475,
      "memory(GiB)": 75.26,
      "reward": 1.9825699925422668,
      "reward_std": 0.41642630100250244,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42526112496852875,
      "rewards/ReflectionBalance/mean": 0.8146012723445892,
      "rewards/ReflectionBalance/std": 0.21487566083669662,
      "step": 558,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 482.69140625,
      "completions/min_length": 244.0,
      "epoch": 3.0215633423180592,
      "grad_norm": 2.5238093316868637,
      "kl": 2.546875,
      "learning_rate": 8.003095793678123e-07,
      "loss": 0.21570923924446106,
      "memory(GiB)": 75.26,
      "reward": 1.8994515538215637,
      "reward_std": 0.5374770015478134,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31518664956092834,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4276812821626663,
      "rewards/ReflectionBalance/mean": 0.774451494216919,
      "rewards/ReflectionBalance/std": 0.2742398679256439,
      "step": 559,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 479.68359375,
      "completions/min_length": 253.5,
      "epoch": 3.026954177897574,
      "grad_norm": 3.4673955771472595,
      "kl": 2.6484375,
      "learning_rate": 7.996232244776311e-07,
      "loss": 0.26574259996414185,
      "memory(GiB)": 75.26,
      "reward": 1.840515911579132,
      "reward_std": 0.46834956109523773,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29262590408325195,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.33968591690063477,
      "rewards/ReflectionBalance/mean": 0.7936409413814545,
      "rewards/ReflectionBalance/std": 0.25968601554632187,
      "step": 560,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 486.26953125,
      "completions/min_length": 263.0,
      "epoch": 3.032345013477089,
      "grad_norm": 3.467360926384367,
      "kl": 3.9609375,
      "learning_rate": 7.98935987526849e-07,
      "loss": 0.33742788434028625,
      "memory(GiB)": 75.26,
      "reward": 1.7990999221801758,
      "reward_std": 0.6480063796043396,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.34222283959388733,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38201163709163666,
      "rewards/ReflectionBalance/mean": 0.7678499221801758,
      "rewards/ReflectionBalance/std": 0.29407064616680145,
      "step": 561,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 506.84375,
      "completions/min_length": 264.0,
      "epoch": 3.0377358490566038,
      "grad_norm": 5.094751848913789,
      "kl": 3.984375,
      "learning_rate": 7.982478705386227e-07,
      "loss": 0.31980761885643005,
      "memory(GiB)": 75.26,
      "reward": 1.7172818779945374,
      "reward_std": 0.5339938402175903,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3460448384284973,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.32612966001033783,
      "rewards/ReflectionBalance/mean": 0.7368131577968597,
      "rewards/ReflectionBalance/std": 0.31145957112312317,
      "step": 562,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 522.55078125,
      "completions/min_length": 273.0,
      "epoch": 3.0431266846361185,
      "grad_norm": 6.4508027831683945,
      "kl": 5.703125,
      "learning_rate": 7.975588755386992e-07,
      "loss": 0.4774683713912964,
      "memory(GiB)": 75.26,
      "reward": 1.7274210453033447,
      "reward_std": 0.6903127431869507,
      "rewards/Format/mean": 0.79296875,
      "rewards/Format/std": 0.4065109044313431,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.4020131081342697,
      "rewards/ReflectionBalance/mean": 0.7313272953033447,
      "rewards/ReflectionBalance/std": 0.32075369358062744,
      "step": 563,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 535.7109375,
      "completions/min_length": 292.0,
      "epoch": 3.0485175202156336,
      "grad_norm": 4.832590139381889,
      "kl": 4.9375,
      "learning_rate": 7.968690045554103e-07,
      "loss": 0.41323113441467285,
      "memory(GiB)": 75.26,
      "reward": 1.7321101427078247,
      "reward_std": 0.6313529312610626,
      "rewards/Format/mean": 0.82421875,
      "rewards/Format/std": 0.3761351853609085,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.36238694190979004,
      "rewards/ReflectionBalance/mean": 0.7321102023124695,
      "rewards/ReflectionBalance/std": 0.3217588812112808,
      "step": 564,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 446.3515625,
      "completions/min_length": 254.0,
      "epoch": 3.0539083557951483,
      "grad_norm": 2.5136103125419784,
      "kl": 1.97265625,
      "learning_rate": 7.961782596196669e-07,
      "loss": 0.17677560448646545,
      "memory(GiB)": 75.26,
      "reward": 1.9753320813179016,
      "reward_std": 0.5238510072231293,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.4284055233001709,
      "rewards/ReflectionBalance/mean": 0.8112695515155792,
      "rewards/ReflectionBalance/std": 0.22110051661729813,
      "step": 565,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 486.28515625,
      "completions/min_length": 262.5,
      "epoch": 3.059299191374663,
      "grad_norm": 2.936493866342292,
      "kl": 2.5703125,
      "learning_rate": 7.954866427649525e-07,
      "loss": 0.2437291294336319,
      "memory(GiB)": 75.26,
      "reward": 1.8491854667663574,
      "reward_std": 0.567573219537735,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3219822645187378,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4065109044313431,
      "rewards/ReflectionBalance/mean": 0.7593416273593903,
      "rewards/ReflectionBalance/std": 0.3323231637477875,
      "step": 566,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 479.3203125,
      "completions/min_length": 273.0,
      "epoch": 3.0646900269541777,
      "grad_norm": 3.2536668368783603,
      "kl": 1.111328125,
      "learning_rate": 7.947941560273172e-07,
      "loss": 0.12468826770782471,
      "memory(GiB)": 75.26,
      "reward": 1.932625412940979,
      "reward_std": 0.4246143400669098,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3580435812473297,
      "rewards/ReflectionBalance/mean": 0.8271565735340118,
      "rewards/ReflectionBalance/std": 0.19905269891023636,
      "step": 567,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 497.84375,
      "completions/min_length": 253.0,
      "epoch": 3.070080862533693,
      "grad_norm": 3.0412068867159694,
      "kl": 2.13671875,
      "learning_rate": 7.941008014453728e-07,
      "loss": 0.21440720558166504,
      "memory(GiB)": 75.26,
      "reward": 1.8834253549575806,
      "reward_std": 0.5108827948570251,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2875154986977577,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3761351853609085,
      "rewards/ReflectionBalance/mean": 0.8013940751552582,
      "rewards/ReflectionBalance/std": 0.24652929604053497,
      "step": 568,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 468.296875,
      "completions/min_length": 236.0,
      "epoch": 3.0754716981132075,
      "grad_norm": 2.7977910458945336,
      "kl": 1.21875,
      "learning_rate": 7.934065810602847e-07,
      "loss": 0.13690048456192017,
      "memory(GiB)": 75.26,
      "reward": 1.811560869216919,
      "reward_std": 0.419961154460907,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.09375,
      "rewards/MultiModalAccuracyORM/std": 0.2875154986977577,
      "rewards/ReflectionBalance/mean": 0.803748369216919,
      "rewards/ReflectionBalance/std": 0.23876722902059555,
      "step": 569,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.5,
      "completions/mean_length": 469.52734375,
      "completions/min_length": 267.0,
      "epoch": 3.0808625336927222,
      "grad_norm": 3.348943344357286,
      "kl": 2.828125,
      "learning_rate": 7.927114969157682e-07,
      "loss": 0.27412593364715576,
      "memory(GiB)": 75.26,
      "reward": 1.9191373586654663,
      "reward_std": 0.5143366008996964,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41328008472919464,
      "rewards/ReflectionBalance/mean": 0.8019498586654663,
      "rewards/ReflectionBalance/std": 0.23587413132190704,
      "step": 570,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 440.51171875,
      "completions/min_length": 220.0,
      "epoch": 3.0862533692722374,
      "grad_norm": 2.9304747998743608,
      "kl": 1.63671875,
      "learning_rate": 7.920155510580809e-07,
      "loss": 0.1688770353794098,
      "memory(GiB)": 75.26,
      "reward": 1.945395588874817,
      "reward_std": 0.4877234101295471,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24356544017791748,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4041530042886734,
      "rewards/ReflectionBalance/mean": 0.7969580292701721,
      "rewards/ReflectionBalance/std": 0.25885556638240814,
      "step": 571,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 490.66796875,
      "completions/min_length": 272.5,
      "epoch": 3.091644204851752,
      "grad_norm": 2.7043542525962754,
      "kl": 3.296875,
      "learning_rate": 7.913187455360175e-07,
      "loss": 0.3048461675643921,
      "memory(GiB)": 75.26,
      "reward": 1.950258731842041,
      "reward_std": 0.6448743343353271,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3219822645187378,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.4431992620229721,
      "rewards/ReflectionBalance/mean": 0.7940087914466858,
      "rewards/ReflectionBalance/std": 0.25549476593732834,
      "step": 572,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 476.70703125,
      "completions/min_length": 249.5,
      "epoch": 3.0970350404312668,
      "grad_norm": 2.6334711451928916,
      "kl": 2.6953125,
      "learning_rate": 7.906210824009029e-07,
      "loss": 0.2588023841381073,
      "memory(GiB)": 75.26,
      "reward": 1.7940722703933716,
      "reward_std": 0.4948050081729889,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3131764084100723,
      "rewards/ReflectionBalance/mean": 0.7862597703933716,
      "rewards/ReflectionBalance/std": 0.2626255676150322,
      "step": 573,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1039.0,
      "completions/mean_length": 501.26171875,
      "completions/min_length": 230.5,
      "epoch": 3.1024258760107815,
      "grad_norm": 3.0839345188197176,
      "kl": 4.03125,
      "learning_rate": 7.899225637065876e-07,
      "loss": 0.376634806394577,
      "memory(GiB)": 75.26,
      "reward": 1.820715844631195,
      "reward_std": 0.6174687147140503,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.35737672448158264,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4065109044313431,
      "rewards/ReflectionBalance/mean": 0.7660283446311951,
      "rewards/ReflectionBalance/std": 0.28501199185848236,
      "step": 574,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 466.27734375,
      "completions/min_length": 241.5,
      "epoch": 3.1078167115902966,
      "grad_norm": 3.111195852203524,
      "kl": 2.5625,
      "learning_rate": 7.892231915094401e-07,
      "loss": 0.22593984007835388,
      "memory(GiB)": 75.26,
      "reward": 1.8773990869522095,
      "reward_std": 0.4668557494878769,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.27558503299951553,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3584487736225128,
      "rewards/ReflectionBalance/mean": 0.7875553369522095,
      "rewards/ReflectionBalance/std": 0.2778269350528717,
      "step": 575,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 482.81640625,
      "completions/min_length": 205.0,
      "epoch": 3.1132075471698113,
      "grad_norm": 5.156458509541603,
      "kl": 4.40625,
      "learning_rate": 7.885229678683416e-07,
      "loss": 0.34887194633483887,
      "memory(GiB)": 75.26,
      "reward": 1.802916705608368,
      "reward_std": 0.6704521477222443,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.3678421527147293,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.39534731209278107,
      "rewards/ReflectionBalance/mean": 0.7521354556083679,
      "rewards/ReflectionBalance/std": 0.30841585993766785,
      "step": 576,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 482.015625,
      "completions/min_length": 230.0,
      "epoch": 3.118598382749326,
      "grad_norm": 3.4021574636682828,
      "kl": 4.546875,
      "learning_rate": 7.8782189484468e-07,
      "loss": 0.4222131073474884,
      "memory(GiB)": 75.26,
      "reward": 1.8200618028640747,
      "reward_std": 0.6338593661785126,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33595384657382965,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.38954539597034454,
      "rewards/ReflectionBalance/mean": 0.7575618624687195,
      "rewards/ReflectionBalance/std": 0.2941470891237259,
      "step": 577,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 487.84375,
      "completions/min_length": 229.5,
      "epoch": 3.123989218328841,
      "grad_norm": 3.564178487541229,
      "kl": 4.078125,
      "learning_rate": 7.871199745023437e-07,
      "loss": 0.35765817761421204,
      "memory(GiB)": 75.26,
      "reward": 1.7785497903823853,
      "reward_std": 0.6433612108230591,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.365008607506752,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3968885838985443,
      "rewards/ReflectionBalance/mean": 0.7433935701847076,
      "rewards/ReflectionBalance/std": 0.3276860862970352,
      "step": 578,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 465.94140625,
      "completions/min_length": 236.5,
      "epoch": 3.129380053908356,
      "grad_norm": 2.5551695849399256,
      "kl": 2.765625,
      "learning_rate": 7.864172089077157e-07,
      "loss": 0.23852531611919403,
      "memory(GiB)": 75.26,
      "reward": 1.839453101158142,
      "reward_std": 0.5073167532682419,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3538244515657425,
      "rewards/ReflectionBalance/mean": 0.7847656309604645,
      "rewards/ReflectionBalance/std": 0.25102248042821884,
      "step": 579,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 473.51171875,
      "completions/min_length": 269.5,
      "epoch": 3.1347708894878705,
      "grad_norm": 4.215070321503692,
      "kl": 1.99609375,
      "learning_rate": 7.857136001296669e-07,
      "loss": 0.20807448029518127,
      "memory(GiB)": 75.26,
      "reward": 1.8789143562316895,
      "reward_std": 0.45791250467300415,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.34826864302158356,
      "rewards/ReflectionBalance/mean": 0.8164143860340118,
      "rewards/ReflectionBalance/std": 0.22370381653308868,
      "step": 580,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 477.38671875,
      "completions/min_length": 254.0,
      "epoch": 3.1401617250673857,
      "grad_norm": 2.697991019757634,
      "kl": 1.46484375,
      "learning_rate": 7.850091502395508e-07,
      "loss": 0.11951965093612671,
      "memory(GiB)": 75.26,
      "reward": 1.9100618362426758,
      "reward_std": 0.4716261476278305,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41502299904823303,
      "rewards/ReflectionBalance/mean": 0.7655306160449982,
      "rewards/ReflectionBalance/std": 0.2811545878648758,
      "step": 581,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 492.26171875,
      "completions/min_length": 279.5,
      "epoch": 3.1455525606469004,
      "grad_norm": 2.732632446966711,
      "kl": 2.640625,
      "learning_rate": 7.843038613111969e-07,
      "loss": 0.2480442225933075,
      "memory(GiB)": 75.26,
      "reward": 1.912644863128662,
      "reward_std": 0.5219105184078217,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.3971414715051651,
      "rewards/ReflectionBalance/mean": 0.7993635535240173,
      "rewards/ReflectionBalance/std": 0.2345631942152977,
      "step": 582,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 457.7265625,
      "completions/min_length": 272.0,
      "epoch": 3.150943396226415,
      "grad_norm": 3.3422304591990426,
      "kl": 2.71875,
      "learning_rate": 7.83597735420905e-07,
      "loss": 0.2789376378059387,
      "memory(GiB)": 75.26,
      "reward": 1.8283502459526062,
      "reward_std": 0.48817019164562225,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29262590408325195,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35675957798957825,
      "rewards/ReflectionBalance/mean": 0.7736627757549286,
      "rewards/ReflectionBalance/std": 0.3052312582731247,
      "step": 583,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 504.578125,
      "completions/min_length": 236.0,
      "epoch": 3.1563342318059298,
      "grad_norm": 3.115536043616949,
      "kl": 3.5078125,
      "learning_rate": 7.828907746474383e-07,
      "loss": 0.31103524565696716,
      "memory(GiB)": 75.26,
      "reward": 1.765171766281128,
      "reward_std": 0.584253340959549,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.36721789836883545,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3584487736225128,
      "rewards/ReflectionBalance/mean": 0.7495467066764832,
      "rewards/ReflectionBalance/std": 0.2768895700573921,
      "step": 584,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 488.91796875,
      "completions/min_length": 247.5,
      "epoch": 3.161725067385445,
      "grad_norm": 2.563533796074352,
      "kl": 3.171875,
      "learning_rate": 7.821829810720182e-07,
      "loss": 0.2929353415966034,
      "memory(GiB)": 75.26,
      "reward": 1.8480159044265747,
      "reward_std": 0.6076913475990295,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3270204961299896,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3968885838985443,
      "rewards/ReflectionBalance/mean": 0.7737971842288971,
      "rewards/ReflectionBalance/std": 0.2667473405599594,
      "step": 585,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 492.11328125,
      "completions/min_length": 264.5,
      "epoch": 3.1671159029649596,
      "grad_norm": 2.920246584148005,
      "kl": 3.1953125,
      "learning_rate": 7.814743567783177e-07,
      "loss": 0.2902529239654541,
      "memory(GiB)": 75.26,
      "reward": 1.8118798732757568,
      "reward_std": 0.5062178075313568,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.31873683631420135,
      "rewards/ReflectionBalance/mean": 0.7806298732757568,
      "rewards/ReflectionBalance/std": 0.27475404739379883,
      "step": 586,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 497.64453125,
      "completions/min_length": 246.0,
      "epoch": 3.1725067385444743,
      "grad_norm": 2.7239211571442663,
      "kl": 2.5,
      "learning_rate": 7.807649038524555e-07,
      "loss": 0.2236396074295044,
      "memory(GiB)": 75.26,
      "reward": 1.8087613582611084,
      "reward_std": 0.48987460136413574,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3337107449769974,
      "rewards/ReflectionBalance/mean": 0.7775113880634308,
      "rewards/ReflectionBalance/std": 0.2677138149738312,
      "step": 587,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 487.44140625,
      "completions/min_length": 298.5,
      "epoch": 3.177897574123989,
      "grad_norm": 3.1725841563115478,
      "kl": 2.5078125,
      "learning_rate": 7.800546243829895e-07,
      "loss": 0.213129460811615,
      "memory(GiB)": 75.26,
      "reward": 1.839931607246399,
      "reward_std": 0.5305795669555664,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3318064510822296,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.39601969718933105,
      "rewards/ReflectionBalance/mean": 0.7696191668510437,
      "rewards/ReflectionBalance/std": 0.30173082649707794,
      "step": 588,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 485.28515625,
      "completions/min_length": 268.0,
      "epoch": 3.183288409703504,
      "grad_norm": 2.5937014471036437,
      "kl": 2.01171875,
      "learning_rate": 7.79343520460911e-07,
      "loss": 0.16796299815177917,
      "memory(GiB)": 75.26,
      "reward": 1.864782691001892,
      "reward_std": 0.498146653175354,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3526121824979782,
      "rewards/ReflectionBalance/mean": 0.7983764708042145,
      "rewards/ReflectionBalance/std": 0.2510836273431778,
      "step": 589,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 463.7421875,
      "completions/min_length": 275.5,
      "epoch": 3.188679245283019,
      "grad_norm": 2.6438678092226233,
      "kl": 1.05078125,
      "learning_rate": 7.786315941796382e-07,
      "loss": 0.10512827336788177,
      "memory(GiB)": 75.26,
      "reward": 1.9023876786231995,
      "reward_std": 0.3497997969388962,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3320184051990509,
      "rewards/ReflectionBalance/mean": 0.8203564584255219,
      "rewards/ReflectionBalance/std": 0.20687296986579895,
      "step": 590,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 996.5,
      "completions/mean_length": 462.71875,
      "completions/min_length": 293.0,
      "epoch": 3.1940700808625335,
      "grad_norm": 2.8015109794392354,
      "kl": 0.765625,
      "learning_rate": 7.779188476350104e-07,
      "loss": 0.08149892091751099,
      "memory(GiB)": 75.26,
      "reward": 2.0801188349723816,
      "reward_std": 0.37334150075912476,
      "rewards/Format/mean": 0.96484375,
      "rewards/Format/std": 0.18204224854707718,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.427789643406868,
      "rewards/ReflectionBalance/mean": 0.8574625253677368,
      "rewards/ReflectionBalance/std": 0.1696467250585556,
      "step": 591,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 996.5,
      "completions/mean_length": 433.1171875,
      "completions/min_length": 225.5,
      "epoch": 3.1994609164420487,
      "grad_norm": 2.94322122447475,
      "kl": 0.806640625,
      "learning_rate": 7.772052829252818e-07,
      "loss": 0.04890470206737518,
      "memory(GiB)": 75.26,
      "reward": 1.9460205435752869,
      "reward_std": 0.3922334760427475,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.3562464565038681,
      "rewards/ReflectionBalance/mean": 0.8405517637729645,
      "rewards/ReflectionBalance/std": 0.17940348386764526,
      "step": 592,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 457.91796875,
      "completions/min_length": 268.0,
      "epoch": 3.2048517520215634,
      "grad_norm": 2.93088147647706,
      "kl": 1.275390625,
      "learning_rate": 7.764909021511148e-07,
      "loss": 0.12313750386238098,
      "memory(GiB)": 75.26,
      "reward": 1.9443125128746033,
      "reward_std": 0.3094108998775482,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.2029399611055851,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3630705028772354,
      "rewards/ReflectionBalance/mean": 0.8388437330722809,
      "rewards/ReflectionBalance/std": 0.1814960166811943,
      "step": 593,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 482.78515625,
      "completions/min_length": 276.0,
      "epoch": 3.210242587601078,
      "grad_norm": 3.228246667821355,
      "kl": 1.5859375,
      "learning_rate": 7.757757074155746e-07,
      "loss": 0.14913815259933472,
      "memory(GiB)": 75.26,
      "reward": 1.9542888402938843,
      "reward_std": 0.45233896374702454,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.42468734085559845,
      "rewards/ReflectionBalance/mean": 0.8019449710845947,
      "rewards/ReflectionBalance/std": 0.2425861433148384,
      "step": 594,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 468.13671875,
      "completions/min_length": 164.0,
      "epoch": 3.215633423180593,
      "grad_norm": 2.46789178057794,
      "kl": 1.7109375,
      "learning_rate": 7.750597008241226e-07,
      "loss": 0.12780754268169403,
      "memory(GiB)": 75.26,
      "reward": 1.8412833213806152,
      "reward_std": 0.4012981504201889,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.31333550810813904,
      "rewards/ReflectionBalance/mean": 0.79440838098526,
      "rewards/ReflectionBalance/std": 0.25254175066947937,
      "step": 595,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 498.7265625,
      "completions/min_length": 254.5,
      "epoch": 3.221024258760108,
      "grad_norm": 12.192481567578383,
      "kl": 3.984375,
      "learning_rate": 7.743428844846101e-07,
      "loss": 0.3384341597557068,
      "memory(GiB)": 75.26,
      "reward": 1.7765445709228516,
      "reward_std": 0.5434440970420837,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.35872696340084076,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3716365247964859,
      "rewards/ReflectionBalance/mean": 0.764825850725174,
      "rewards/ReflectionBalance/std": 0.2792823165655136,
      "step": 596,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 498.71484375,
      "completions/min_length": 250.5,
      "epoch": 3.2264150943396226,
      "grad_norm": 3.767672429017617,
      "kl": 4.15625,
      "learning_rate": 7.736252605072719e-07,
      "loss": 0.35418885946273804,
      "memory(GiB)": 75.26,
      "reward": 1.8155468702316284,
      "reward_std": 0.5785838663578033,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.36387722194194794,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.37836818397045135,
      "rewards/ReflectionBalance/mean": 0.7842968702316284,
      "rewards/ReflectionBalance/std": 0.25921162962913513,
      "step": 597,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 511.50390625,
      "completions/min_length": 277.5,
      "epoch": 3.2318059299191373,
      "grad_norm": 5.0223205195697345,
      "kl": 4.40625,
      "learning_rate": 7.72906831004721e-07,
      "loss": 0.38245689868927,
      "memory(GiB)": 75.26,
      "reward": 1.75566565990448,
      "reward_std": 0.6052336096763611,
      "rewards/Format/mean": 0.8203125,
      "rewards/Format/std": 0.38489311933517456,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3602610230445862,
      "rewards/ReflectionBalance/mean": 0.76347815990448,
      "rewards/ReflectionBalance/std": 0.2912529855966568,
      "step": 598,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 485.25390625,
      "completions/min_length": 271.0,
      "epoch": 3.2371967654986524,
      "grad_norm": 3.032040371616736,
      "kl": 3.1875,
      "learning_rate": 7.721875980919414e-07,
      "loss": 0.28051966428756714,
      "memory(GiB)": 75.26,
      "reward": 1.899998426437378,
      "reward_std": 0.6104416251182556,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.332206666469574,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.4407331943511963,
      "rewards/ReflectionBalance/mean": 0.7671858966350555,
      "rewards/ReflectionBalance/std": 0.29412104189395905,
      "step": 599,
      "train_speed(iter/s)": 0.007102
    },
    {
      "epoch": 3.242587601078167,
      "grad_norm": 2.770003810392053,
      "learning_rate": 7.714675638862825e-07,
      "loss": 0.19626197218894958,
      "memory(GiB)": 75.26,
      "step": 600,
      "train_speed(iter/s)": 0.007103
    },
    {
      "epoch": 3.242587601078167,
      "eval_clip_ratio": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 1026.5,
      "eval_completions/mean_length": 492.9140625,
      "eval_completions/min_length": 268.25,
      "eval_kl": 1.771484375,
      "eval_loss": 0.14268334209918976,
      "eval_reward": 1.8568470776081085,
      "eval_reward_std": 0.44971348345279694,
      "eval_rewards/Format/mean": 0.9140625,
      "eval_rewards/Format/std": 0.2784690894186497,
      "eval_rewards/MultiModalAccuracyORM/mean": 0.140625,
      "eval_rewards/MultiModalAccuracyORM/std": 0.33758318051695824,
      "eval_rewards/ReflectionBalance/mean": 0.8021595925092697,
      "eval_rewards/ReflectionBalance/std": 0.27019447460770607,
      "eval_runtime": 248.2752,
      "eval_samples_per_second": 0.238,
      "eval_steps_per_second": 0.004,
      "step": 600
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.25,
      "completions/mean_length": 470.962890625,
      "completions/min_length": 242.0,
      "epoch": 3.247978436657682,
      "grad_norm": 2.813726814998735,
      "kl": 2.134765625,
      "learning_rate": 7.707467305074523e-07,
      "loss": 0.18733906745910645,
      "memory(GiB)": 75.26,
      "reward": 1.9264404475688934,
      "reward_std": 0.4648054614663124,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2887473963201046,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.40256864577531815,
      "rewards/ReflectionBalance/mean": 0.8131591975688934,
      "rewards/ReflectionBalance/std": 0.2393283024430275,
      "step": 601,
      "train_speed(iter/s)": 0.007078
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 483.26171875,
      "completions/min_length": 221.0,
      "epoch": 3.2533692722371965,
      "grad_norm": 2.4584062042425097,
      "kl": 1.431640625,
      "learning_rate": 7.700251000775119e-07,
      "loss": 0.13141436874866486,
      "memory(GiB)": 75.26,
      "reward": 1.8581151962280273,
      "reward_std": 0.3642187714576721,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24301259219646454,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3130832314491272,
      "rewards/ReflectionBalance/mean": 0.8112402558326721,
      "rewards/ReflectionBalance/std": 0.2386390045285225,
      "step": 602,
      "train_speed(iter/s)": 0.007078
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 906.0,
      "completions/mean_length": 464.0078125,
      "completions/min_length": 240.5,
      "epoch": 3.2587601078167117,
      "grad_norm": 2.466833337205611,
      "kl": 1.591796875,
      "learning_rate": 7.693026747208684e-07,
      "loss": 0.14491748809814453,
      "memory(GiB)": 75.26,
      "reward": 1.8821712136268616,
      "reward_std": 0.43861834704875946,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.32114607095718384,
      "rewards/ReflectionBalance/mean": 0.8157649636268616,
      "rewards/ReflectionBalance/std": 0.22743279486894608,
      "step": 603,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 492.50390625,
      "completions/min_length": 272.5,
      "epoch": 3.2641509433962264,
      "grad_norm": 3.528105517875315,
      "kl": 2.34765625,
      "learning_rate": 7.685794565642694e-07,
      "loss": 0.23398979008197784,
      "memory(GiB)": 75.26,
      "reward": 1.8654825687408447,
      "reward_std": 0.4020248204469681,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.253920242190361,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3247780203819275,
      "rewards/ReflectionBalance/mean": 0.8186075985431671,
      "rewards/ReflectionBalance/std": 0.25171997398138046,
      "step": 604,
      "train_speed(iter/s)": 0.007081
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 477.31640625,
      "completions/min_length": 279.5,
      "epoch": 3.269541778975741,
      "grad_norm": 2.543544017592305,
      "kl": 2.14453125,
      "learning_rate": 7.678554477367964e-07,
      "loss": 0.19518142938613892,
      "memory(GiB)": 75.26,
      "reward": 1.8706185817718506,
      "reward_std": 0.46907083690166473,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.27558503299951553,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36045311391353607,
      "rewards/ReflectionBalance/mean": 0.800305962562561,
      "rewards/ReflectionBalance/std": 0.21690047532320023,
      "step": 605,
      "train_speed(iter/s)": 0.007082
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 457.30859375,
      "completions/min_length": 260.0,
      "epoch": 3.274932614555256,
      "grad_norm": 4.057779634863128,
      "kl": 1.703125,
      "learning_rate": 7.671306503698585e-07,
      "loss": 0.18224576115608215,
      "memory(GiB)": 75.26,
      "reward": 1.898413062095642,
      "reward_std": 0.44580450654029846,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2255912721157074,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3316322788596153,
      "rewards/ReflectionBalance/mean": 0.8085693418979645,
      "rewards/ReflectionBalance/std": 0.2424912080168724,
      "step": 606,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 504.06640625,
      "completions/min_length": 263.0,
      "epoch": 3.280323450134771,
      "grad_norm": 3.0543541107440646,
      "kl": 2.265625,
      "learning_rate": 7.66405066597186e-07,
      "loss": 0.23442307114601135,
      "memory(GiB)": 75.26,
      "reward": 1.97524493932724,
      "reward_std": 0.5479369610548019,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27079907804727554,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.43946409225463867,
      "rewards/ReflectionBalance/mean": 0.7955574989318848,
      "rewards/ReflectionBalance/std": 0.26069727540016174,
      "step": 607,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 480.9140625,
      "completions/min_length": 222.0,
      "epoch": 3.2857142857142856,
      "grad_norm": 3.9640809800562296,
      "kl": 3.78125,
      "learning_rate": 7.656786985548243e-07,
      "loss": 0.318431556224823,
      "memory(GiB)": 75.26,
      "reward": 1.9175423383712769,
      "reward_std": 0.6491641402244568,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3407054841518402,
      "rewards/MultiModalAccuracyORM/mean": 0.28125,
      "rewards/MultiModalAccuracyORM/std": 0.44721250236034393,
      "rewards/ReflectionBalance/mean": 0.7691048085689545,
      "rewards/ReflectionBalance/std": 0.2781345248222351,
      "step": 608,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 460.37890625,
      "completions/min_length": 256.0,
      "epoch": 3.2911051212938007,
      "grad_norm": 5.5077445999843695,
      "kl": 3.71875,
      "learning_rate": 7.649515483811283e-07,
      "loss": 0.30894866585731506,
      "memory(GiB)": 75.26,
      "reward": 1.9400358200073242,
      "reward_std": 0.5363872200250626,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31518664956092834,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.40798476338386536,
      "rewards/ReflectionBalance/mean": 0.7915982902050018,
      "rewards/ReflectionBalance/std": 0.28404054045677185,
      "step": 609,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 514.2265625,
      "completions/min_length": 263.5,
      "epoch": 3.2964959568733154,
      "grad_norm": 8.505639142392743,
      "kl": 5.96875,
      "learning_rate": 7.642236182167544e-07,
      "loss": 0.4633435010910034,
      "memory(GiB)": 75.26,
      "reward": 1.7434749603271484,
      "reward_std": 0.6365396082401276,
      "rewards/Format/mean": 0.8203125,
      "rewards/Format/std": 0.3853000998497009,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.37402482330799103,
      "rewards/ReflectionBalance/mean": 0.7434749603271484,
      "rewards/ReflectionBalance/std": 0.33803434669971466,
      "step": 610,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 504.76953125,
      "completions/min_length": 245.0,
      "epoch": 3.30188679245283,
      "grad_norm": 9.823347694281788,
      "kl": 6.484375,
      "learning_rate": 7.63494910204656e-07,
      "loss": 0.48688429594039917,
      "memory(GiB)": 75.26,
      "reward": 1.806969404220581,
      "reward_std": 0.7185723185539246,
      "rewards/Format/mean": 0.80078125,
      "rewards/Format/std": 0.395822137594223,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.44771209359169006,
      "rewards/ReflectionBalance/mean": 0.728844404220581,
      "rewards/ReflectionBalance/std": 0.3278210461139679,
      "step": 611,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 487.43359375,
      "completions/min_length": 252.0,
      "epoch": 3.3072776280323453,
      "grad_norm": 6.2291621818152265,
      "kl": 5.2890625,
      "learning_rate": 7.62765426490076e-07,
      "loss": 0.42233526706695557,
      "memory(GiB)": 75.26,
      "reward": 1.892174482345581,
      "reward_std": 0.6456160843372345,
      "rewards/Format/mean": 0.81640625,
      "rewards/Format/std": 0.38864077627658844,
      "rewards/MultiModalAccuracyORM/mean": 0.30859375,
      "rewards/MultiModalAccuracyORM/std": 0.4513196349143982,
      "rewards/ReflectionBalance/mean": 0.7671745121479034,
      "rewards/ReflectionBalance/std": 0.29443490505218506,
      "step": 612,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 491.0703125,
      "completions/min_length": 266.0,
      "epoch": 3.31266846361186,
      "grad_norm": 4.189311370541172,
      "kl": 3.9921875,
      "learning_rate": 7.620351692205409e-07,
      "loss": 0.3140447735786438,
      "memory(GiB)": 75.26,
      "reward": 1.802754521369934,
      "reward_std": 0.6165125668048859,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.3643587827682495,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.39185720682144165,
      "rewards/ReflectionBalance/mean": 0.7636920809745789,
      "rewards/ReflectionBalance/std": 0.3011337071657181,
      "step": 613,
      "train_speed(iter/s)": 0.007083
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 457.50390625,
      "completions/min_length": 269.0,
      "epoch": 3.3180592991913747,
      "grad_norm": 3.188730740039086,
      "kl": 2.1875,
      "learning_rate": 7.613041405458547e-07,
      "loss": 0.20475053787231445,
      "memory(GiB)": 75.26,
      "reward": 1.8233023285865784,
      "reward_std": 0.46292844414711,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2801405265927315,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.2948496490716934,
      "rewards/ReflectionBalance/mean": 0.7881461679935455,
      "rewards/ReflectionBalance/std": 0.26068688929080963,
      "step": 614,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 477.10546875,
      "completions/min_length": 278.5,
      "epoch": 3.3234501347708894,
      "grad_norm": 3.160428140846866,
      "kl": 2.484375,
      "learning_rate": 7.605723426180922e-07,
      "loss": 0.24495665729045868,
      "memory(GiB)": 75.26,
      "reward": 1.8811165690422058,
      "reward_std": 0.5400725305080414,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.40855903923511505,
      "rewards/ReflectionBalance/mean": 0.7834602892398834,
      "rewards/ReflectionBalance/std": 0.29755306243896484,
      "step": 615,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 443.0546875,
      "completions/min_length": 267.5,
      "epoch": 3.3288409703504045,
      "grad_norm": 2.9077191817238144,
      "kl": 2.0625,
      "learning_rate": 7.598397775915928e-07,
      "loss": 0.21247360110282898,
      "memory(GiB)": 75.26,
      "reward": 1.9129817485809326,
      "reward_std": 0.460227370262146,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4100748300552368,
      "rewards/ReflectionBalance/mean": 0.772356778383255,
      "rewards/ReflectionBalance/std": 0.2537231966853142,
      "step": 616,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 456.4140625,
      "completions/min_length": 264.0,
      "epoch": 3.334231805929919,
      "grad_norm": 2.4793932238926857,
      "kl": 1.046875,
      "learning_rate": 7.59106447622954e-07,
      "loss": 0.09354211390018463,
      "memory(GiB)": 75.26,
      "reward": 1.971925437450409,
      "reward_std": 0.4019395262002945,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.8273941874504089,
      "rewards/ReflectionBalance/std": 0.17354115843772888,
      "step": 617,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 484.7421875,
      "completions/min_length": 257.5,
      "epoch": 3.339622641509434,
      "grad_norm": 3.615382907009955,
      "kl": 1.62109375,
      "learning_rate": 7.583723548710256e-07,
      "loss": 0.16993404924869537,
      "memory(GiB)": 75.26,
      "reward": 1.8949983716011047,
      "reward_std": 0.4147030860185623,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2524137645959854,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35358233749866486,
      "rewards/ReflectionBalance/mean": 0.8129671216011047,
      "rewards/ReflectionBalance/std": 0.2250276431441307,
      "step": 618,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 946.0,
      "completions/mean_length": 460.046875,
      "completions/min_length": 277.0,
      "epoch": 3.3450134770889486,
      "grad_norm": 3.0029092516870817,
      "kl": 1.72265625,
      "learning_rate": 7.576375014969027e-07,
      "loss": 0.15769526362419128,
      "memory(GiB)": 75.26,
      "reward": 1.8640169501304626,
      "reward_std": 0.48561355471611023,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3538244515657425,
      "rewards/ReflectionBalance/mean": 0.7937044203281403,
      "rewards/ReflectionBalance/std": 0.22746416181325912,
      "step": 619,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 466.16015625,
      "completions/min_length": 262.0,
      "epoch": 3.3504043126684637,
      "grad_norm": 3.3856312263887,
      "kl": 2.08984375,
      "learning_rate": 7.569018896639194e-07,
      "loss": 0.1807887852191925,
      "memory(GiB)": 75.26,
      "reward": 1.9072201251983643,
      "reward_std": 0.5236642062664032,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3068038523197174,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.424868568778038,
      "rewards/ReflectionBalance/mean": 0.7822200655937195,
      "rewards/ReflectionBalance/std": 0.24168728291988373,
      "step": 620,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 461.8125,
      "completions/min_length": 242.0,
      "epoch": 3.3557951482479784,
      "grad_norm": 2.7584181071781093,
      "kl": 2.2265625,
      "learning_rate": 7.561655215376432e-07,
      "loss": 0.20397138595581055,
      "memory(GiB)": 75.26,
      "reward": 1.8342138528823853,
      "reward_std": 0.5183702409267426,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30283892154693604,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.37335334718227386,
      "rewards/ReflectionBalance/mean": 0.7404638528823853,
      "rewards/ReflectionBalance/std": 0.3147635906934738,
      "step": 621,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 477.92578125,
      "completions/min_length": 273.0,
      "epoch": 3.361185983827493,
      "grad_norm": 3.0544494920950815,
      "kl": 1.49609375,
      "learning_rate": 7.554283992858677e-07,
      "loss": 0.12344563007354736,
      "memory(GiB)": 75.26,
      "reward": 1.9204345345497131,
      "reward_std": 0.3647993803024292,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2255912721157074,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36045311391353607,
      "rewards/ReflectionBalance/mean": 0.8188720941543579,
      "rewards/ReflectionBalance/std": 0.1963859498500824,
      "step": 622,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 460.03125,
      "completions/min_length": 275.5,
      "epoch": 3.3665768194070083,
      "grad_norm": 2.6981536299404443,
      "kl": 1.953125,
      "learning_rate": 7.546905250786062e-07,
      "loss": 0.16915610432624817,
      "memory(GiB)": 75.26,
      "reward": 1.8413541316986084,
      "reward_std": 0.5170898139476776,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3334042727947235,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.40395987033843994,
      "rewards/ReflectionBalance/mean": 0.7632291913032532,
      "rewards/ReflectionBalance/std": 0.27645574510097504,
      "step": 623,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 496.2265625,
      "completions/min_length": 285.5,
      "epoch": 3.371967654986523,
      "grad_norm": 2.5665794623674847,
      "kl": 2.3359375,
      "learning_rate": 7.539519010880864e-07,
      "loss": 0.23174402117729187,
      "memory(GiB)": 75.26,
      "reward": 1.8922883868217468,
      "reward_std": 0.5504561960697174,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4018339514732361,
      "rewards/ReflectionBalance/mean": 0.7829134166240692,
      "rewards/ReflectionBalance/std": 0.28053992986679077,
      "step": 624,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 454.80078125,
      "completions/min_length": 269.5,
      "epoch": 3.3773584905660377,
      "grad_norm": 2.6015577476946055,
      "kl": 1.701171875,
      "learning_rate": 7.532125294887428e-07,
      "loss": 0.1581547111272812,
      "memory(GiB)": 75.26,
      "reward": 1.8544026613235474,
      "reward_std": 0.48965156078338623,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.339918777346611,
      "rewards/ReflectionBalance/mean": 0.8036214709281921,
      "rewards/ReflectionBalance/std": 0.2568349242210388,
      "step": 625,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 484.625,
      "completions/min_length": 239.0,
      "epoch": 3.382749326145553,
      "grad_norm": 2.6596717003006187,
      "kl": 3.1953125,
      "learning_rate": 7.52472412457211e-07,
      "loss": 0.3114737272262573,
      "memory(GiB)": 75.26,
      "reward": 1.770094394683838,
      "reward_std": 0.5508893728256226,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.28863146156072617,
      "rewards/ReflectionBalance/mean": 0.7818131446838379,
      "rewards/ReflectionBalance/std": 0.29606112837791443,
      "step": 626,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 470.1484375,
      "completions/min_length": 251.5,
      "epoch": 3.3881401617250675,
      "grad_norm": 2.7533374697955852,
      "kl": 2.7578125,
      "learning_rate": 7.517315521723211e-07,
      "loss": 0.23792225122451782,
      "memory(GiB)": 75.26,
      "reward": 1.8876188397407532,
      "reward_std": 0.5027792155742645,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.27645355463027954,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3934052139520645,
      "rewards/ReflectionBalance/mean": 0.7860563099384308,
      "rewards/ReflectionBalance/std": 0.25549738109111786,
      "step": 627,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 486.1796875,
      "completions/min_length": 272.5,
      "epoch": 3.393530997304582,
      "grad_norm": 3.8927622864612186,
      "kl": 3.08984375,
      "learning_rate": 7.50989950815091e-07,
      "loss": 0.2474876046180725,
      "memory(GiB)": 75.26,
      "reward": 1.8124462962150574,
      "reward_std": 0.5041705071926117,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.332206666469574,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38489311933517456,
      "rewards/ReflectionBalance/mean": 0.7616650462150574,
      "rewards/ReflectionBalance/std": 0.2711568996310234,
      "step": 628,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1039.5,
      "completions/mean_length": 488.296875,
      "completions/min_length": 285.5,
      "epoch": 3.398921832884097,
      "grad_norm": 2.7167801373967224,
      "kl": 3.2265625,
      "learning_rate": 7.502476105687203e-07,
      "loss": 0.29610520601272583,
      "memory(GiB)": 75.26,
      "reward": 1.8395215272903442,
      "reward_std": 0.5465317368507385,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.3131764084100723,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3642655462026596,
      "rewards/ReflectionBalance/mean": 0.7848339676856995,
      "rewards/ReflectionBalance/std": 0.24927786737680435,
      "step": 629,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 502.28125,
      "completions/min_length": 273.0,
      "epoch": 3.404312668463612,
      "grad_norm": 3.5573431307710464,
      "kl": 3.59375,
      "learning_rate": 7.495045336185844e-07,
      "loss": 0.30300241708755493,
      "memory(GiB)": 75.26,
      "reward": 1.766391634941101,
      "reward_std": 0.5554037690162659,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.35296520590782166,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3607245087623596,
      "rewards/ReflectionBalance/mean": 0.7585791051387787,
      "rewards/ReflectionBalance/std": 0.29110056161880493,
      "step": 630,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 503.34765625,
      "completions/min_length": 263.0,
      "epoch": 3.4097035040431267,
      "grad_norm": 2.589985666670408,
      "kl": 2.890625,
      "learning_rate": 7.487607221522265e-07,
      "loss": 0.2685193121433258,
      "memory(GiB)": 75.26,
      "reward": 1.73105788230896,
      "reward_std": 0.5377952754497528,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3318064510822296,
      "rewards/MultiModalAccuracyORM/mean": 0.0859375,
      "rewards/MultiModalAccuracyORM/std": 0.244512889534235,
      "rewards/ReflectionBalance/mean": 0.7701204419136047,
      "rewards/ReflectionBalance/std": 0.2791125625371933,
      "step": 631,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 453.42578125,
      "completions/min_length": 282.5,
      "epoch": 3.4150943396226414,
      "grad_norm": 2.85799016141519,
      "kl": 1.69140625,
      "learning_rate": 7.480161783593531e-07,
      "loss": 0.17798680067062378,
      "memory(GiB)": 75.26,
      "reward": 1.993320345878601,
      "reward_std": 0.43952783942222595,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21875878423452377,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4111319035291672,
      "rewards/ReflectionBalance/mean": 0.8136327862739563,
      "rewards/ReflectionBalance/std": 0.23108621686697006,
      "step": 632,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 471.765625,
      "completions/min_length": 250.0,
      "epoch": 3.420485175202156,
      "grad_norm": 2.915617317483613,
      "kl": 2.59375,
      "learning_rate": 7.472709044318257e-07,
      "loss": 0.24101674556732178,
      "memory(GiB)": 75.26,
      "reward": 1.8356478214263916,
      "reward_std": 0.49456360936164856,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3050640821456909,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3316322788596153,
      "rewards/ReflectionBalance/mean": 0.7965852618217468,
      "rewards/ReflectionBalance/std": 0.2762957215309143,
      "step": 633,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.5,
      "completions/mean_length": 490.01953125,
      "completions/min_length": 289.5,
      "epoch": 3.4258760107816713,
      "grad_norm": 2.9417632577864805,
      "kl": 3.2265625,
      "learning_rate": 7.465249025636559e-07,
      "loss": 0.27896344661712646,
      "memory(GiB)": 75.26,
      "reward": 1.9428296089172363,
      "reward_std": 0.6068441271781921,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3422359675168991,
      "rewards/MultiModalAccuracyORM/mean": 0.3046875,
      "rewards/MultiModalAccuracyORM/std": 0.4541175067424774,
      "rewards/ReflectionBalance/mean": 0.7787671089172363,
      "rewards/ReflectionBalance/std": 0.2663773149251938,
      "step": 634,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 492.30078125,
      "completions/min_length": 299.0,
      "epoch": 3.431266846361186,
      "grad_norm": 3.1437610702162413,
      "kl": 2.875,
      "learning_rate": 7.457781749509983e-07,
      "loss": 0.25547417998313904,
      "memory(GiB)": 75.26,
      "reward": 1.896275520324707,
      "reward_std": 0.4728844463825226,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.42527204751968384,
      "rewards/ReflectionBalance/mean": 0.7712755799293518,
      "rewards/ReflectionBalance/std": 0.2754426747560501,
      "step": 635,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 457.08203125,
      "completions/min_length": 277.0,
      "epoch": 3.4366576819407006,
      "grad_norm": 2.518618295258703,
      "kl": 2.1171875,
      "learning_rate": 7.450307237921436e-07,
      "loss": 0.19422002136707306,
      "memory(GiB)": 75.26,
      "reward": 1.8606559038162231,
      "reward_std": 0.441747784614563,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3410066068172455,
      "rewards/ReflectionBalance/mean": 0.7903434634208679,
      "rewards/ReflectionBalance/std": 0.26229576021432877,
      "step": 636,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 480.8671875,
      "completions/min_length": 259.5,
      "epoch": 3.442048517520216,
      "grad_norm": 3.4525254731781443,
      "kl": 2.15625,
      "learning_rate": 7.442825512875129e-07,
      "loss": 0.20665201544761658,
      "memory(GiB)": 75.26,
      "reward": 1.9140267372131348,
      "reward_std": 0.473262682557106,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29262590408325195,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3700246214866638,
      "rewards/ReflectionBalance/mean": 0.8163704574108124,
      "rewards/ReflectionBalance/std": 0.24054890871047974,
      "step": 637,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 459.77734375,
      "completions/min_length": 242.5,
      "epoch": 3.4474393530997305,
      "grad_norm": 2.6089805099555567,
      "kl": 2.5546875,
      "learning_rate": 7.435336596396502e-07,
      "loss": 0.22922584414482117,
      "memory(GiB)": 75.26,
      "reward": 1.9232828617095947,
      "reward_std": 0.522170901298523,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3320184051990509,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.42517124116420746,
      "rewards/ReflectionBalance/mean": 0.8139078617095947,
      "rewards/ReflectionBalance/std": 0.21062937378883362,
      "step": 638,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 502.96484375,
      "completions/min_length": 255.5,
      "epoch": 3.452830188679245,
      "grad_norm": 3.2186482673311523,
      "kl": 3.109375,
      "learning_rate": 7.427840510532177e-07,
      "loss": 0.2532663941383362,
      "memory(GiB)": 75.26,
      "reward": 1.77055823802948,
      "reward_std": 0.570379912853241,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.3410066068172455,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.34483732283115387,
      "rewards/ReflectionBalance/mean": 0.7549332976341248,
      "rewards/ReflectionBalance/std": 0.28265073895454407,
      "step": 639,
      "train_speed(iter/s)": 0.007084
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 483.8046875,
      "completions/min_length": 195.5,
      "epoch": 3.4582210242587603,
      "grad_norm": 3.6768578802352025,
      "kl": 3.5625,
      "learning_rate": 7.420337277349871e-07,
      "loss": 0.31844431161880493,
      "memory(GiB)": 75.26,
      "reward": 1.747389316558838,
      "reward_std": 0.5862545967102051,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3389271944761276,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.34826864302158356,
      "rewards/ReflectionBalance/mean": 0.7395768165588379,
      "rewards/ReflectionBalance/std": 0.323100283741951,
      "step": 640,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 486.0546875,
      "completions/min_length": 227.0,
      "epoch": 3.463611859838275,
      "grad_norm": 34.83962524893533,
      "kl": 2.40625,
      "learning_rate": 7.412826918938352e-07,
      "loss": 0.17587688565254211,
      "memory(GiB)": 75.26,
      "reward": 1.7539517879486084,
      "reward_std": 0.5035487413406372,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33636194467544556,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.33116789162158966,
      "rewards/ReflectionBalance/mean": 0.7578580975532532,
      "rewards/ReflectionBalance/std": 0.2930729389190674,
      "step": 641,
      "train_speed(iter/s)": 0.007085
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 474.1484375,
      "completions/min_length": 270.0,
      "epoch": 3.4690026954177897,
      "grad_norm": 2.659773215955462,
      "kl": 2.4453125,
      "learning_rate": 7.405309457407354e-07,
      "loss": 0.2236560881137848,
      "memory(GiB)": 75.26,
      "reward": 1.8427188396453857,
      "reward_std": 0.5102245360612869,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38325364887714386,
      "rewards/ReflectionBalance/mean": 0.7606876790523529,
      "rewards/ReflectionBalance/std": 0.29693329334259033,
      "step": 642,
      "train_speed(iter/s)": 0.007086
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 438.30859375,
      "completions/min_length": 249.5,
      "epoch": 3.4743935309973044,
      "grad_norm": 2.7210638558422655,
      "kl": 1.1328125,
      "learning_rate": 7.39778491488753e-07,
      "loss": 0.10480453819036484,
      "memory(GiB)": 75.26,
      "reward": 1.918360948562622,
      "reward_std": 0.3116833120584488,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20335330069065094,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.2899702489376068,
      "rewards/ReflectionBalance/mean": 0.8519547581672668,
      "rewards/ReflectionBalance/std": 0.15501133352518082,
      "step": 643,
      "train_speed(iter/s)": 0.007086
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 468.796875,
      "completions/min_length": 278.5,
      "epoch": 3.4797843665768196,
      "grad_norm": 2.8143836211924245,
      "kl": 1.8828125,
      "learning_rate": 7.390253313530375e-07,
      "loss": 0.19526448845863342,
      "memory(GiB)": 75.26,
      "reward": 1.922376275062561,
      "reward_std": 0.4183785915374756,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.36930377781391144,
      "rewards/ReflectionBalance/mean": 0.8169075548648834,
      "rewards/ReflectionBalance/std": 0.20910444110631943,
      "step": 644,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 467.76953125,
      "completions/min_length": 263.5,
      "epoch": 3.4851752021563343,
      "grad_norm": 3.3816487822068333,
      "kl": 1.3671875,
      "learning_rate": 7.382714675508163e-07,
      "loss": 0.14383406937122345,
      "memory(GiB)": 75.26,
      "reward": 1.8950440287590027,
      "reward_std": 0.44041258096694946,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3242573067545891,
      "rewards/ReflectionBalance/mean": 0.8247314691543579,
      "rewards/ReflectionBalance/std": 0.2021765261888504,
      "step": 645,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 456.125,
      "completions/min_length": 224.0,
      "epoch": 3.490566037735849,
      "grad_norm": 2.9296542312193585,
      "kl": 1.7109375,
      "learning_rate": 7.375169023013888e-07,
      "loss": 0.18384143710136414,
      "memory(GiB)": 75.26,
      "reward": 1.8535937666893005,
      "reward_std": 0.4665782004594803,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36045311391353607,
      "rewards/ReflectionBalance/mean": 0.7832812368869781,
      "rewards/ReflectionBalance/std": 0.27664104849100113,
      "step": 646,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 484.421875,
      "completions/min_length": 271.5,
      "epoch": 3.4959568733153636,
      "grad_norm": 3.044932069210936,
      "kl": 2.4375,
      "learning_rate": 7.36761637826119e-07,
      "loss": 0.2033662497997284,
      "memory(GiB)": 75.26,
      "reward": 1.868040382862091,
      "reward_std": 0.48058614134788513,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29434798657894135,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39171716570854187,
      "rewards/ReflectionBalance/mean": 0.7781966030597687,
      "rewards/ReflectionBalance/std": 0.2595904469490051,
      "step": 647,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 495.6640625,
      "completions/min_length": 230.0,
      "epoch": 3.501347708894879,
      "grad_norm": 3.477319962621818,
      "kl": 4.1015625,
      "learning_rate": 7.360056763484294e-07,
      "loss": 0.3522549867630005,
      "memory(GiB)": 75.26,
      "reward": 1.8875327706336975,
      "reward_std": 0.5538449287414551,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.3493855148553848,
      "rewards/MultiModalAccuracyORM/mean": 0.265625,
      "rewards/MultiModalAccuracyORM/std": 0.439003586769104,
      "rewards/ReflectionBalance/mean": 0.7664391100406647,
      "rewards/ReflectionBalance/std": 0.2866903394460678,
      "step": 648,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 495.84765625,
      "completions/min_length": 267.5,
      "epoch": 3.5067385444743935,
      "grad_norm": 3.992265276980253,
      "kl": 4.421875,
      "learning_rate": 7.352490200937943e-07,
      "loss": 0.3637911081314087,
      "memory(GiB)": 75.26,
      "reward": 1.75261390209198,
      "reward_std": 0.6014615595340729,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.34447427093982697,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.3169707804918289,
      "rewards/ReflectionBalance/mean": 0.7721452116966248,
      "rewards/ReflectionBalance/std": 0.3014613091945648,
      "step": 649,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.5,
      "completions/mean_length": 449.63671875,
      "completions/min_length": 234.5,
      "epoch": 3.512129380053908,
      "grad_norm": 3.1498156517488343,
      "kl": 3.0859375,
      "learning_rate": 7.344916712897337e-07,
      "loss": 0.2736040949821472,
      "memory(GiB)": 75.26,
      "reward": 1.8623388409614563,
      "reward_std": 0.4978564828634262,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3634065240621567,
      "rewards/ReflectionBalance/mean": 0.7920263707637787,
      "rewards/ReflectionBalance/std": 0.28524625301361084,
      "step": 650,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 492.2421875,
      "completions/min_length": 255.5,
      "epoch": 3.5175202156334233,
      "grad_norm": 3.758488342112761,
      "kl": 3.109375,
      "learning_rate": 7.337336321658065e-07,
      "loss": 0.25717687606811523,
      "memory(GiB)": 75.26,
      "reward": 1.8395613431930542,
      "reward_std": 0.5414514541625977,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3229459524154663,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3786114752292633,
      "rewards/ReflectionBalance/mean": 0.788780152797699,
      "rewards/ReflectionBalance/std": 0.2574119344353676,
      "step": 651,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 490.625,
      "completions/min_length": 294.0,
      "epoch": 3.522911051212938,
      "grad_norm": 3.2198734965753837,
      "kl": 2.609375,
      "learning_rate": 7.329749049536032e-07,
      "loss": 0.24032844603061676,
      "memory(GiB)": 75.26,
      "reward": 1.8034505248069763,
      "reward_std": 0.47841185331344604,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32612966001033783,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35538628697395325,
      "rewards/ReflectionBalance/mean": 0.7761067748069763,
      "rewards/ReflectionBalance/std": 0.32575172185897827,
      "step": 652,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 476.234375,
      "completions/min_length": 253.0,
      "epoch": 3.5283018867924527,
      "grad_norm": 2.4976043403649353,
      "kl": 1.859375,
      "learning_rate": 7.322154918867403e-07,
      "loss": 0.16540080308914185,
      "memory(GiB)": 75.26,
      "reward": 2.0108765363693237,
      "reward_std": 0.46223169565200806,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.2379390001296997,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.41646628081798553,
      "rewards/ReflectionBalance/mean": 0.819470226764679,
      "rewards/ReflectionBalance/std": 0.2419031858444214,
      "step": 653,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 450.72265625,
      "completions/min_length": 247.0,
      "epoch": 3.533692722371968,
      "grad_norm": 2.4644353255156104,
      "kl": 1.453125,
      "learning_rate": 7.314553952008535e-07,
      "loss": 0.12544097006320953,
      "memory(GiB)": 75.26,
      "reward": 1.9732848405838013,
      "reward_std": 0.3691280484199524,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3877096325159073,
      "rewards/ReflectionBalance/mean": 0.8404723405838013,
      "rewards/ReflectionBalance/std": 0.18175358325242996,
      "step": 654,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 471.828125,
      "completions/min_length": 236.5,
      "epoch": 3.5390835579514826,
      "grad_norm": 2.5390155507922114,
      "kl": 1.96875,
      "learning_rate": 7.306946171335909e-07,
      "loss": 0.17596399784088135,
      "memory(GiB)": 75.26,
      "reward": 1.9339176416397095,
      "reward_std": 0.4094395935535431,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24081215262413025,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.38315413892269135,
      "rewards/ReflectionBalance/mean": 0.8050113916397095,
      "rewards/ReflectionBalance/std": 0.2445983812212944,
      "step": 655,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 500.73046875,
      "completions/min_length": 302.0,
      "epoch": 3.5444743935309972,
      "grad_norm": 2.9596750964154697,
      "kl": 2.5703125,
      "learning_rate": 7.299331599246063e-07,
      "loss": 0.23401175439357758,
      "memory(GiB)": 75.26,
      "reward": 1.8979662656784058,
      "reward_std": 0.5153161883354187,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.32267695665359497,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4032507836818695,
      "rewards/ReflectionBalance/mean": 0.8081225454807281,
      "rewards/ReflectionBalance/std": 0.23698921501636505,
      "step": 656,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 482.48828125,
      "completions/min_length": 257.5,
      "epoch": 3.5498652291105124,
      "grad_norm": 2.4571634525492563,
      "kl": 2.46875,
      "learning_rate": 7.291710258155535e-07,
      "loss": 0.21245434880256653,
      "memory(GiB)": 75.26,
      "reward": 1.8238964080810547,
      "reward_std": 0.5244689881801605,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35358233749866486,
      "rewards/ReflectionBalance/mean": 0.7770214676856995,
      "rewards/ReflectionBalance/std": 0.2859346568584442,
      "step": 657,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 890.0,
      "completions/mean_length": 459.83203125,
      "completions/min_length": 258.0,
      "epoch": 3.555256064690027,
      "grad_norm": 3.2804419258990447,
      "kl": 1.5703125,
      "learning_rate": 7.284082170500782e-07,
      "loss": 0.1420547515153885,
      "memory(GiB)": 75.26,
      "reward": 1.9375211596488953,
      "reward_std": 0.49553506076335907,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4018339514732361,
      "rewards/ReflectionBalance/mean": 0.8047086596488953,
      "rewards/ReflectionBalance/std": 0.27219606935977936,
      "step": 658,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 483.5859375,
      "completions/min_length": 262.0,
      "epoch": 3.560646900269542,
      "grad_norm": 2.7653688884353684,
      "kl": 2.796875,
      "learning_rate": 7.276447358738127e-07,
      "loss": 0.2574273347854614,
      "memory(GiB)": 75.26,
      "reward": 1.7650642395019531,
      "reward_std": 0.5696921646595001,
      "rewards/Format/mean": 0.8359375,
      "rewards/Format/std": 0.3679620176553726,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39184603095054626,
      "rewards/ReflectionBalance/mean": 0.7416267693042755,
      "rewards/ReflectionBalance/std": 0.3234584182500839,
      "step": 659,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1033.5,
      "completions/mean_length": 491.75,
      "completions/min_length": 232.5,
      "epoch": 3.5660377358490565,
      "grad_norm": 2.783995630905682,
      "kl": 3.359375,
      "learning_rate": 7.268805845343687e-07,
      "loss": 0.3030139207839966,
      "memory(GiB)": 75.26,
      "reward": 1.844448208808899,
      "reward_std": 0.5342830717563629,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3336549401283264,
      "rewards/ReflectionBalance/mean": 0.7897607386112213,
      "rewards/ReflectionBalance/std": 0.286222904920578,
      "step": 660,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1038.0,
      "completions/mean_length": 525.46875,
      "completions/min_length": 264.0,
      "epoch": 3.571428571428571,
      "grad_norm": 5.132499389049705,
      "kl": 3.8125,
      "learning_rate": 7.261157652813308e-07,
      "loss": 0.2823234796524048,
      "memory(GiB)": 75.26,
      "reward": 1.73484206199646,
      "reward_std": 0.6661706864833832,
      "rewards/Format/mean": 0.80078125,
      "rewards/Format/std": 0.4007115960121155,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39416566491127014,
      "rewards/ReflectionBalance/mean": 0.7426546216011047,
      "rewards/ReflectionBalance/std": 0.354586660861969,
      "step": 661,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 492.44140625,
      "completions/min_length": 277.5,
      "epoch": 3.5768194070080863,
      "grad_norm": 4.347189062651536,
      "kl": 3.859375,
      "learning_rate": 7.253502803662498e-07,
      "loss": 0.30977147817611694,
      "memory(GiB)": 75.26,
      "reward": 1.7651789784431458,
      "reward_std": 0.5665301084518433,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.3597298264503479,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36935214698314667,
      "rewards/ReflectionBalance/mean": 0.7534602880477905,
      "rewards/ReflectionBalance/std": 0.2951462119817734,
      "step": 662,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 456.05078125,
      "completions/min_length": 245.0,
      "epoch": 3.582210242587601,
      "grad_norm": 2.9696098577778756,
      "kl": 2.578125,
      "learning_rate": 7.245841320426363e-07,
      "loss": 0.23707443475723267,
      "memory(GiB)": 75.26,
      "reward": 1.8822461366653442,
      "reward_std": 0.5478994250297546,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3996165096759796,
      "rewards/ReflectionBalance/mean": 0.7845898270606995,
      "rewards/ReflectionBalance/std": 0.26351189613342285,
      "step": 663,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 480.4765625,
      "completions/min_length": 272.0,
      "epoch": 3.5876010781671157,
      "grad_norm": 3.118533484819842,
      "kl": 2.578125,
      "learning_rate": 7.238173225659539e-07,
      "loss": 0.25089168548583984,
      "memory(GiB)": 75.26,
      "reward": 1.856711506843567,
      "reward_std": 0.5635623335838318,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3191515803337097,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.38558371365070343,
      "rewards/ReflectionBalance/mean": 0.7824926972389221,
      "rewards/ReflectionBalance/std": 0.2713002562522888,
      "step": 664,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 472.1171875,
      "completions/min_length": 258.0,
      "epoch": 3.592991913746631,
      "grad_norm": 3.399451936930998,
      "kl": 2.078125,
      "learning_rate": 7.230498541936121e-07,
      "loss": 0.20773357152938843,
      "memory(GiB)": 75.26,
      "reward": 1.899000644683838,
      "reward_std": 0.38210149109363556,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3023834079504013,
      "rewards/ReflectionBalance/mean": 0.8286882042884827,
      "rewards/ReflectionBalance/std": 0.21011465042829514,
      "step": 665,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 467.01953125,
      "completions/min_length": 240.5,
      "epoch": 3.5983827493261455,
      "grad_norm": 3.6983045773818235,
      "kl": 2.5703125,
      "learning_rate": 7.22281729184961e-07,
      "loss": 0.23669926822185516,
      "memory(GiB)": 75.26,
      "reward": 1.8388827443122864,
      "reward_std": 0.4752172976732254,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.30283892154693604,
      "rewards/ReflectionBalance/mean": 0.8193514049053192,
      "rewards/ReflectionBalance/std": 0.22154653072357178,
      "step": 666,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 465.75390625,
      "completions/min_length": 254.0,
      "epoch": 3.6037735849056602,
      "grad_norm": 2.9778979939722707,
      "kl": 2.26171875,
      "learning_rate": 7.215129498012829e-07,
      "loss": 0.16356107592582703,
      "memory(GiB)": 75.26,
      "reward": 1.8333560824394226,
      "reward_std": 0.4596138447523117,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.33148839324712753,
      "rewards/ReflectionBalance/mean": 0.7903873324394226,
      "rewards/ReflectionBalance/std": 0.2478262558579445,
      "step": 667,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1033.0,
      "completions/mean_length": 477.5859375,
      "completions/min_length": 259.0,
      "epoch": 3.6091644204851754,
      "grad_norm": 2.8085691177527354,
      "kl": 1.7890625,
      "learning_rate": 7.207435183057871e-07,
      "loss": 0.15293699502944946,
      "memory(GiB)": 75.26,
      "reward": 1.9515023231506348,
      "reward_std": 0.42386212944984436,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2488820105791092,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3756510615348816,
      "rewards/ReflectionBalance/mean": 0.8265022933483124,
      "rewards/ReflectionBalance/std": 0.20089297741651535,
      "step": 668,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 453.38671875,
      "completions/min_length": 261.5,
      "epoch": 3.61455525606469,
      "grad_norm": 3.0016821753798837,
      "kl": 2.14453125,
      "learning_rate": 7.199734369636023e-07,
      "loss": 0.18985725939273834,
      "memory(GiB)": 75.26,
      "reward": 1.9022794365882874,
      "reward_std": 0.5058615803718567,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3985297381877899,
      "rewards/ReflectionBalance/mean": 0.8007169365882874,
      "rewards/ReflectionBalance/std": 0.24153289943933487,
      "step": 669,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 457.5703125,
      "completions/min_length": 275.0,
      "epoch": 3.6199460916442048,
      "grad_norm": 2.773077248851808,
      "kl": 1.93359375,
      "learning_rate": 7.192027080417705e-07,
      "loss": 0.18699946999549866,
      "memory(GiB)": 75.26,
      "reward": 2.048629581928253,
      "reward_std": 0.4333746135234833,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.28125,
      "rewards/MultiModalAccuracyORM/std": 0.44096554815769196,
      "rewards/ReflectionBalance/mean": 0.8376920521259308,
      "rewards/ReflectionBalance/std": 0.20485720038414001,
      "step": 670,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 491.7890625,
      "completions/min_length": 243.0,
      "epoch": 3.62533692722372,
      "grad_norm": 3.4100882702158852,
      "kl": 3.703125,
      "learning_rate": 7.184313338092401e-07,
      "loss": 0.32913368940353394,
      "memory(GiB)": 75.26,
      "reward": 1.7720588445663452,
      "reward_std": 0.4692386984825134,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.322907418012619,
      "rewards/MultiModalAccuracyORM/mean": 0.08984375,
      "rewards/MultiModalAccuracyORM/std": 0.2869994640350342,
      "rewards/ReflectionBalance/mean": 0.7994026839733124,
      "rewards/ReflectionBalance/std": 0.2619583010673523,
      "step": 671,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 480.33984375,
      "completions/min_length": 265.5,
      "epoch": 3.6307277628032346,
      "grad_norm": 2.6801184605202444,
      "kl": 3.34375,
      "learning_rate": 7.17659316536859e-07,
      "loss": 0.3059060573577881,
      "memory(GiB)": 75.26,
      "reward": 1.8364436626434326,
      "reward_std": 0.5032416880130768,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3174293115735054,
      "rewards/ReflectionBalance/mean": 0.7973811626434326,
      "rewards/ReflectionBalance/std": 0.2600906565785408,
      "step": 672,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 477.578125,
      "completions/min_length": 270.5,
      "epoch": 3.6361185983827493,
      "grad_norm": 3.4548826935529138,
      "kl": 3.5,
      "learning_rate": 7.168866584973684e-07,
      "loss": 0.29358863830566406,
      "memory(GiB)": 75.26,
      "reward": 1.7859455943107605,
      "reward_std": 0.5374795794487,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33513282239437103,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3425614684820175,
      "rewards/ReflectionBalance/mean": 0.7781331241130829,
      "rewards/ReflectionBalance/std": 0.25474029779434204,
      "step": 673,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 517.30078125,
      "completions/min_length": 242.5,
      "epoch": 3.641509433962264,
      "grad_norm": 4.567202180108814,
      "kl": 4.203125,
      "learning_rate": 7.161133619653958e-07,
      "loss": 0.3375547528266907,
      "memory(GiB)": 75.26,
      "reward": 1.7997851371765137,
      "reward_std": 0.5783707499504089,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.3508307933807373,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3948954641819,
      "rewards/ReflectionBalance/mean": 0.7490038871765137,
      "rewards/ReflectionBalance/std": 0.33301523327827454,
      "step": 674,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 476.50390625,
      "completions/min_length": 240.5,
      "epoch": 3.6469002695417787,
      "grad_norm": 2.833161654886579,
      "kl": 2.953125,
      "learning_rate": 7.153394292174481e-07,
      "loss": 0.2625114917755127,
      "memory(GiB)": 75.26,
      "reward": 1.898826539516449,
      "reward_std": 0.560785710811615,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3978758305311203,
      "rewards/ReflectionBalance/mean": 0.8011702597141266,
      "rewards/ReflectionBalance/std": 0.2540625333786011,
      "step": 675,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 518.15234375,
      "completions/min_length": 289.0,
      "epoch": 3.652291105121294,
      "grad_norm": 2.82252561680484,
      "kl": 3.3046875,
      "learning_rate": 7.145648625319057e-07,
      "loss": 0.2980339825153351,
      "memory(GiB)": 75.26,
      "reward": 1.8592814207077026,
      "reward_std": 0.5937950909137726,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33636194467544556,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3823556900024414,
      "rewards/ReflectionBalance/mean": 0.7928751409053802,
      "rewards/ReflectionBalance/std": 0.2812541574239731,
      "step": 676,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 476.44140625,
      "completions/min_length": 270.0,
      "epoch": 3.6576819407008085,
      "grad_norm": 2.9112049063743486,
      "kl": 2.59765625,
      "learning_rate": 7.137896641890148e-07,
      "loss": 0.23129203915596008,
      "memory(GiB)": 75.26,
      "reward": 2.0082291960716248,
      "reward_std": 0.4610861837863922,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26276856660842896,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4494253695011139,
      "rewards/ReflectionBalance/mean": 0.8090104162693024,
      "rewards/ReflectionBalance/std": 0.2384086325764656,
      "step": 677,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 466.5703125,
      "completions/min_length": 251.5,
      "epoch": 3.6630727762803232,
      "grad_norm": 3.3541452856499614,
      "kl": 1.98046875,
      "learning_rate": 7.130138364708811e-07,
      "loss": 0.192436084151268,
      "memory(GiB)": 75.26,
      "reward": 1.9477717280387878,
      "reward_std": 0.44169098138809204,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4018339514732361,
      "rewards/ReflectionBalance/mean": 0.807146817445755,
      "rewards/ReflectionBalance/std": 0.2322637140750885,
      "step": 678,
      "train_speed(iter/s)": 0.007094
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 478.06640625,
      "completions/min_length": 295.0,
      "epoch": 3.6684636118598384,
      "grad_norm": 2.80400126633608,
      "kl": 2.02734375,
      "learning_rate": 7.122373816614635e-07,
      "loss": 0.19020932912826538,
      "memory(GiB)": 75.26,
      "reward": 1.946430265903473,
      "reward_std": 0.4072938710451126,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3979988098144531,
      "rewards/ReflectionBalance/mean": 0.8097115755081177,
      "rewards/ReflectionBalance/std": 0.24925993382930756,
      "step": 679,
      "train_speed(iter/s)": 0.007095
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 481.61328125,
      "completions/min_length": 273.5,
      "epoch": 3.673854447439353,
      "grad_norm": 2.5244345115362328,
      "kl": 2.3984375,
      "learning_rate": 7.114603020465668e-07,
      "loss": 0.2040097713470459,
      "memory(GiB)": 75.26,
      "reward": 1.8800276517868042,
      "reward_std": 0.5442572236061096,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32746291160583496,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.40759414434432983,
      "rewards/ReflectionBalance/mean": 0.7862776815891266,
      "rewards/ReflectionBalance/std": 0.28076957166194916,
      "step": 680,
      "train_speed(iter/s)": 0.007095
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 461.06640625,
      "completions/min_length": 262.5,
      "epoch": 3.6792452830188678,
      "grad_norm": 2.4935377664305234,
      "kl": 1.59375,
      "learning_rate": 7.106825999138348e-07,
      "loss": 0.12908747792243958,
      "memory(GiB)": 75.26,
      "reward": 1.9384701251983643,
      "reward_std": 0.3947708010673523,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3387676477432251,
      "rewards/ReflectionBalance/mean": 0.8095638155937195,
      "rewards/ReflectionBalance/std": 0.2534744441509247,
      "step": 681,
      "train_speed(iter/s)": 0.007095
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1033.5,
      "completions/mean_length": 453.19921875,
      "completions/min_length": 241.5,
      "epoch": 3.684636118598383,
      "grad_norm": 2.764592714141248,
      "kl": 2.2578125,
      "learning_rate": 7.099042775527444e-07,
      "loss": 0.21547873318195343,
      "memory(GiB)": 75.26,
      "reward": 1.9494921565055847,
      "reward_std": 0.447318896651268,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3996165096759796,
      "rewards/ReflectionBalance/mean": 0.8244922161102295,
      "rewards/ReflectionBalance/std": 0.22146833688020706,
      "step": 682,
      "train_speed(iter/s)": 0.007095
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 479.34375,
      "completions/min_length": 238.0,
      "epoch": 3.6900269541778976,
      "grad_norm": 2.387641674085519,
      "kl": 2.15234375,
      "learning_rate": 7.091253372545985e-07,
      "loss": 0.17272163927555084,
      "memory(GiB)": 75.26,
      "reward": 1.9575512409210205,
      "reward_std": 0.517164945602417,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42921192944049835,
      "rewards/ReflectionBalance/mean": 0.7973949909210205,
      "rewards/ReflectionBalance/std": 0.2840646505355835,
      "step": 683,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 467.42578125,
      "completions/min_length": 267.0,
      "epoch": 3.6954177897574123,
      "grad_norm": 2.5131396115574987,
      "kl": 2.04296875,
      "learning_rate": 7.083457813125187e-07,
      "loss": 0.17859598994255066,
      "memory(GiB)": 75.26,
      "reward": 1.9030777215957642,
      "reward_std": 0.541512131690979,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.40284374356269836,
      "rewards/ReflectionBalance/mean": 0.8054215312004089,
      "rewards/ReflectionBalance/std": 0.234404556453228,
      "step": 684,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 506.1953125,
      "completions/min_length": 244.0,
      "epoch": 3.7008086253369274,
      "grad_norm": 2.8504723085046786,
      "kl": 3.03125,
      "learning_rate": 7.075656120214392e-07,
      "loss": 0.2624048888683319,
      "memory(GiB)": 75.26,
      "reward": 1.8003596663475037,
      "reward_std": 0.5477414727210999,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.34826864302158356,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3556671589612961,
      "rewards/ReflectionBalance/mean": 0.788640946149826,
      "rewards/ReflectionBalance/std": 0.258970245718956,
      "step": 685,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 478.19921875,
      "completions/min_length": 208.5,
      "epoch": 3.706199460916442,
      "grad_norm": 3.1830897824320834,
      "kl": 2.7890625,
      "learning_rate": 7.067848316781e-07,
      "loss": 0.22558417916297913,
      "memory(GiB)": 75.26,
      "reward": 1.8561605215072632,
      "reward_std": 0.549098014831543,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.4007115960121155,
      "rewards/ReflectionBalance/mean": 0.7702229917049408,
      "rewards/ReflectionBalance/std": 0.31016509234905243,
      "step": 686,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 486.1171875,
      "completions/min_length": 239.5,
      "epoch": 3.711590296495957,
      "grad_norm": 2.6941503039424073,
      "kl": 3.3515625,
      "learning_rate": 7.060034425810394e-07,
      "loss": 0.306358277797699,
      "memory(GiB)": 75.26,
      "reward": 1.7995409965515137,
      "reward_std": 0.5801549255847931,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3473474234342575,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.37408973276615143,
      "rewards/ReflectionBalance/mean": 0.7643847763538361,
      "rewards/ReflectionBalance/std": 0.28509873151779175,
      "step": 687,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 455.296875,
      "completions/min_length": 244.5,
      "epoch": 3.7169811320754715,
      "grad_norm": 3.351368903198689,
      "kl": 1.8359375,
      "learning_rate": 7.052214470305885e-07,
      "loss": 0.15130925178527832,
      "memory(GiB)": 75.26,
      "reward": 1.9628238677978516,
      "reward_std": 0.505824863910675,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.43684960901737213,
      "rewards/ReflectionBalance/mean": 0.7870426177978516,
      "rewards/ReflectionBalance/std": 0.28332750499248505,
      "step": 688,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 486.83984375,
      "completions/min_length": 254.5,
      "epoch": 3.7223719676549867,
      "grad_norm": 12.939553101788249,
      "kl": 2.6953125,
      "learning_rate": 7.044388473288636e-07,
      "loss": 0.2059965431690216,
      "memory(GiB)": 75.26,
      "reward": 1.750718593597412,
      "reward_std": 0.4463058114051819,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.0703125,
      "rewards/MultiModalAccuracyORM/std": 0.2548178359866142,
      "rewards/ReflectionBalance/mean": 0.7936873137950897,
      "rewards/ReflectionBalance/std": 0.2596680819988251,
      "step": 689,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 533.32421875,
      "completions/min_length": 271.0,
      "epoch": 3.7277628032345014,
      "grad_norm": 3.3484959245184394,
      "kl": 3.3359375,
      "learning_rate": 7.036556457797591e-07,
      "loss": 0.270172655582428,
      "memory(GiB)": 75.26,
      "reward": 1.6763826608657837,
      "reward_std": 0.5924808382987976,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.3678421527147293,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.31812146306037903,
      "rewards/ReflectionBalance/mean": 0.7232576608657837,
      "rewards/ReflectionBalance/std": 0.36147329211235046,
      "step": 690,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 986.0,
      "completions/mean_length": 455.17578125,
      "completions/min_length": 269.0,
      "epoch": 3.733153638814016,
      "grad_norm": 2.838839679574774,
      "kl": 1.96484375,
      "learning_rate": 7.028718446889416e-07,
      "loss": 0.171390563249588,
      "memory(GiB)": 75.26,
      "reward": 1.8672216534614563,
      "reward_std": 0.4177858680486679,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3166656941175461,
      "rewards/ReflectionBalance/mean": 0.8281591534614563,
      "rewards/ReflectionBalance/std": 0.21053898334503174,
      "step": 691,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 474.29296875,
      "completions/min_length": 188.5,
      "epoch": 3.7385444743935308,
      "grad_norm": 3.03894307104875,
      "kl": 2.3984375,
      "learning_rate": 7.020874463638427e-07,
      "loss": 0.20329563319683075,
      "memory(GiB)": 75.26,
      "reward": 1.9122875928878784,
      "reward_std": 0.5148373991250992,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3389271944761276,
      "rewards/MultiModalAccuracyORM/mean": 0.265625,
      "rewards/MultiModalAccuracyORM/std": 0.4430450052022934,
      "rewards/ReflectionBalance/mean": 0.7794750928878784,
      "rewards/ReflectionBalance/std": 0.3023349344730377,
      "step": 692,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 507.875,
      "completions/min_length": 276.5,
      "epoch": 3.743935309973046,
      "grad_norm": 3.2170785653025757,
      "kl": 3.27734375,
      "learning_rate": 7.013024531136519e-07,
      "loss": 0.27263548970222473,
      "memory(GiB)": 75.26,
      "reward": 1.8241374492645264,
      "reward_std": 0.5730181485414505,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.3586205691099167,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.39205317199230194,
      "rewards/ReflectionBalance/mean": 0.7811686396598816,
      "rewards/ReflectionBalance/std": 0.26881104707717896,
      "step": 693,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 496.9453125,
      "completions/min_length": 269.0,
      "epoch": 3.7493261455525606,
      "grad_norm": 2.79155839329389,
      "kl": 2.890625,
      "learning_rate": 7.005168672493104e-07,
      "loss": 0.24150824546813965,
      "memory(GiB)": 75.26,
      "reward": 1.8669645190238953,
      "reward_std": 0.530105471611023,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.3166656941175461,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3979988098144531,
      "rewards/ReflectionBalance/mean": 0.7849332690238953,
      "rewards/ReflectionBalance/std": 0.29840999841690063,
      "step": 694,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 481.4609375,
      "completions/min_length": 215.0,
      "epoch": 3.7547169811320753,
      "grad_norm": 2.7194562029179123,
      "kl": 2.3515625,
      "learning_rate": 6.997306910835038e-07,
      "loss": 0.1803709864616394,
      "memory(GiB)": 75.26,
      "reward": 1.8611838817596436,
      "reward_std": 0.577847808599472,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3473474234342575,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4312489330768585,
      "rewards/ReflectionBalance/mean": 0.7557151317596436,
      "rewards/ReflectionBalance/std": 0.3266574442386627,
      "step": 695,
      "train_speed(iter/s)": 0.007095
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 494.94140625,
      "completions/min_length": 279.5,
      "epoch": 3.7601078167115904,
      "grad_norm": 2.7811212220342885,
      "kl": 2.234375,
      "learning_rate": 6.989439269306557e-07,
      "loss": 0.19113193452358246,
      "memory(GiB)": 75.26,
      "reward": 1.914965808391571,
      "reward_std": 0.4340221583843231,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39171716570854187,
      "rewards/ReflectionBalance/mean": 0.8094970881938934,
      "rewards/ReflectionBalance/std": 0.24748043715953827,
      "step": 696,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 456.3203125,
      "completions/min_length": 144.0,
      "epoch": 3.765498652291105,
      "grad_norm": 4.597930740726031,
      "kl": 2.06640625,
      "learning_rate": 6.981565771069204e-07,
      "loss": 0.17648807168006897,
      "memory(GiB)": 75.26,
      "reward": 1.82134610414505,
      "reward_std": 0.45942550897598267,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2721100151538849,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.30669571459293365,
      "rewards/ReflectionBalance/mean": 0.8018147945404053,
      "rewards/ReflectionBalance/std": 0.2758297771215439,
      "step": 697,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 463.32421875,
      "completions/min_length": 257.5,
      "epoch": 3.77088948787062,
      "grad_norm": 3.0783815128453718,
      "kl": 2.11328125,
      "learning_rate": 6.973686439301765e-07,
      "loss": 0.1619856059551239,
      "memory(GiB)": 75.26,
      "reward": 1.9809244871139526,
      "reward_std": 0.46752412617206573,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4202718883752823,
      "rewards/ReflectionBalance/mean": 0.8168619871139526,
      "rewards/ReflectionBalance/std": 0.21308910846710205,
      "step": 698,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 516.52734375,
      "completions/min_length": 307.0,
      "epoch": 3.776280323450135,
      "grad_norm": 2.7521251039217067,
      "kl": 2.3125,
      "learning_rate": 6.9658012972002e-07,
      "loss": 0.22899602353572845,
      "memory(GiB)": 75.26,
      "reward": 1.8330233097076416,
      "reward_std": 0.5036189556121826,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3678421527147293,
      "rewards/ReflectionBalance/mean": 0.7705232501029968,
      "rewards/ReflectionBalance/std": 0.2838452011346817,
      "step": 699,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1018.5,
      "completions/mean_length": 458.75,
      "completions/min_length": 268.0,
      "epoch": 3.7816711590296497,
      "grad_norm": 2.558675937587989,
      "kl": 1.14453125,
      "learning_rate": 6.957910367977574e-07,
      "loss": 0.10895291715860367,
      "memory(GiB)": 75.26,
      "reward": 2.0378092527389526,
      "reward_std": 0.45916688442230225,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.2890625,
      "rewards/MultiModalAccuracyORM/std": 0.45378951728343964,
      "rewards/ReflectionBalance/mean": 0.8151529729366302,
      "rewards/ReflectionBalance/std": 0.24780593067407608,
      "step": 700,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 908.0,
      "completions/mean_length": 472.58984375,
      "completions/min_length": 264.0,
      "epoch": 3.7870619946091644,
      "grad_norm": 3.2776663331795217,
      "kl": 1.46875,
      "learning_rate": 6.950013674863986e-07,
      "loss": 0.15014638006687164,
      "memory(GiB)": 75.26,
      "reward": 1.8710424900054932,
      "reward_std": 0.3896297961473465,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23196382820606232,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.300025038421154,
      "rewards/ReflectionBalance/mean": 0.8202612400054932,
      "rewards/ReflectionBalance/std": 0.20767123997211456,
      "step": 701,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 468.421875,
      "completions/min_length": 253.5,
      "epoch": 3.7924528301886795,
      "grad_norm": 2.484939248950284,
      "kl": 1.8671875,
      "learning_rate": 6.942111241106506e-07,
      "loss": 0.14571966230869293,
      "memory(GiB)": 75.26,
      "reward": 2.0114063024520874,
      "reward_std": 0.47754697501659393,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.4231107383966446,
      "rewards/ReflectionBalance/mean": 0.8239062428474426,
      "rewards/ReflectionBalance/std": 0.21242468804121017,
      "step": 702,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 504.28125,
      "completions/min_length": 276.5,
      "epoch": 3.797843665768194,
      "grad_norm": 3.224117661397666,
      "kl": 3.265625,
      "learning_rate": 6.934203089969104e-07,
      "loss": 0.2885509431362152,
      "memory(GiB)": 75.26,
      "reward": 1.8146859407424927,
      "reward_std": 0.6130194813013077,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33595384657382965,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36175550520420074,
      "rewards/ReflectionBalance/mean": 0.7795296311378479,
      "rewards/ReflectionBalance/std": 0.3032692074775696,
      "step": 703,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.0,
      "completions/mean_length": 517.8515625,
      "completions/min_length": 292.0,
      "epoch": 3.803234501347709,
      "grad_norm": 2.878366252837768,
      "kl": 3.2109375,
      "learning_rate": 6.926289244732579e-07,
      "loss": 0.26909464597702026,
      "memory(GiB)": 75.26,
      "reward": 1.7459814548492432,
      "reward_std": 0.6186171472072601,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.3562464565038681,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.332206666469574,
      "rewards/ReflectionBalance/mean": 0.7655127048492432,
      "rewards/ReflectionBalance/std": 0.3252001404762268,
      "step": 704,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1042.0,
      "completions/mean_length": 535.87890625,
      "completions/min_length": 205.0,
      "epoch": 3.8086253369272236,
      "grad_norm": 4.434061972828392,
      "kl": 3.9921875,
      "learning_rate": 6.918369728694497e-07,
      "loss": 0.3295914828777313,
      "memory(GiB)": 75.26,
      "reward": 1.850480079650879,
      "reward_std": 0.569673016667366,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.330063596367836,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.39169713854789734,
      "rewards/ReflectionBalance/mean": 0.7762613892555237,
      "rewards/ReflectionBalance/std": 0.2686515599489212,
      "step": 705,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 498.875,
      "completions/min_length": 247.0,
      "epoch": 3.8140161725067383,
      "grad_norm": 5.953373777547602,
      "kl": 4.4296875,
      "learning_rate": 6.910444565169114e-07,
      "loss": 0.3293772041797638,
      "memory(GiB)": 75.26,
      "reward": 1.7835953831672668,
      "reward_std": 0.6234959661960602,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.35538628697395325,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3764989674091339,
      "rewards/ReflectionBalance/mean": 0.7523453831672668,
      "rewards/ReflectionBalance/std": 0.32609619200229645,
      "step": 706,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 485.46875,
      "completions/min_length": 247.5,
      "epoch": 3.8194070080862534,
      "grad_norm": 3.1521766776697078,
      "kl": 2.46875,
      "learning_rate": 6.902513777487313e-07,
      "loss": 0.21629191935062408,
      "memory(GiB)": 75.26,
      "reward": 1.8085969686508179,
      "reward_std": 0.46502985060214996,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3068038523197174,
      "rewards/ReflectionBalance/mean": 0.7968782484531403,
      "rewards/ReflectionBalance/std": 0.265775203704834,
      "step": 707,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.0,
      "completions/mean_length": 475.58203125,
      "completions/min_length": 247.5,
      "epoch": 3.824797843665768,
      "grad_norm": 3.1020411177445637,
      "kl": 2.19140625,
      "learning_rate": 6.894577388996536e-07,
      "loss": 0.21061944961547852,
      "memory(GiB)": 75.26,
      "reward": 1.9044840335845947,
      "reward_std": 0.45046503841876984,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2524137645959854,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3511015325784683,
      "rewards/ReflectionBalance/mean": 0.8224528133869171,
      "rewards/ReflectionBalance/std": 0.20170807093381882,
      "step": 708,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 507.76171875,
      "completions/min_length": 267.5,
      "epoch": 3.830188679245283,
      "grad_norm": 2.8217867224064483,
      "kl": 3.0703125,
      "learning_rate": 6.886635423060711e-07,
      "loss": 0.26974380016326904,
      "memory(GiB)": 75.26,
      "reward": 1.8789795637130737,
      "reward_std": 0.5636350214481354,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.34485287964344025,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.4013945013284683,
      "rewards/ReflectionBalance/mean": 0.796948254108429,
      "rewards/ReflectionBalance/std": 0.2640247195959091,
      "step": 709,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 489.7890625,
      "completions/min_length": 270.5,
      "epoch": 3.835579514824798,
      "grad_norm": 2.9449826285843965,
      "kl": 2.9765625,
      "learning_rate": 6.878687903060184e-07,
      "loss": 0.23560994863510132,
      "memory(GiB)": 75.26,
      "reward": 1.7615103721618652,
      "reward_std": 0.5684338510036469,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.3425614684820175,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3206050544977188,
      "rewards/ReflectionBalance/mean": 0.7771354019641876,
      "rewards/ReflectionBalance/std": 0.2911268174648285,
      "step": 710,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 541.3515625,
      "completions/min_length": 292.5,
      "epoch": 3.8409703504043127,
      "grad_norm": 2.8676624509680217,
      "kl": 3.375,
      "learning_rate": 6.870734852391655e-07,
      "loss": 0.2825932502746582,
      "memory(GiB)": 75.26,
      "reward": 1.7653206586837769,
      "reward_std": 0.5196380317211151,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3460448384284973,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.30737847089767456,
      "rewards/ReflectionBalance/mean": 0.7731331586837769,
      "rewards/ReflectionBalance/std": 0.28753462433815,
      "step": 711,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 488.984375,
      "completions/min_length": 278.5,
      "epoch": 3.8463611859838274,
      "grad_norm": 3.3419107894518176,
      "kl": 3.125,
      "learning_rate": 6.862776294468105e-07,
      "loss": 0.2693656086921692,
      "memory(GiB)": 75.26,
      "reward": 1.8789664506912231,
      "reward_std": 0.5335004329681396,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3208130896091461,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41404466331005096,
      "rewards/ReflectionBalance/mean": 0.7774039506912231,
      "rewards/ReflectionBalance/std": 0.3070443868637085,
      "step": 712,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 495.3359375,
      "completions/min_length": 260.5,
      "epoch": 3.8517520215633425,
      "grad_norm": 7.620473633214982,
      "kl": 3.5,
      "learning_rate": 6.854812252718725e-07,
      "loss": 0.29350849986076355,
      "memory(GiB)": 75.26,
      "reward": 1.8824689984321594,
      "reward_std": 0.5152511596679688,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.30921147763729095,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4276812821626663,
      "rewards/ReflectionBalance/mean": 0.7535628378391266,
      "rewards/ReflectionBalance/std": 0.3233391046524048,
      "step": 713,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 529.6484375,
      "completions/min_length": 279.5,
      "epoch": 3.857142857142857,
      "grad_norm": 3.3168854550564624,
      "kl": 3.15625,
      "learning_rate": 6.846842750588854e-07,
      "loss": 0.30300378799438477,
      "memory(GiB)": 75.26,
      "reward": 1.793857455253601,
      "reward_std": 0.5817517638206482,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33595384657382965,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.35641810297966003,
      "rewards/ReflectionBalance/mean": 0.7665137052536011,
      "rewards/ReflectionBalance/std": 0.2814445048570633,
      "step": 714,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 470.94921875,
      "completions/min_length": 251.5,
      "epoch": 3.862533692722372,
      "grad_norm": 2.4849072628061792,
      "kl": 2.16015625,
      "learning_rate": 6.838867811539903e-07,
      "loss": 0.1908164918422699,
      "memory(GiB)": 75.26,
      "reward": 1.8756005764007568,
      "reward_std": 0.4776914715766907,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3716365247964859,
      "rewards/ReflectionBalance/mean": 0.7974755764007568,
      "rewards/ReflectionBalance/std": 0.22980159521102905,
      "step": 715,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.0,
      "completions/mean_length": 485.859375,
      "completions/min_length": 250.5,
      "epoch": 3.867924528301887,
      "grad_norm": 3.5051003122157405,
      "kl": 2.171875,
      "learning_rate": 6.830887459049291e-07,
      "loss": 0.21297571063041687,
      "memory(GiB)": 75.26,
      "reward": 1.8982674479484558,
      "reward_std": 0.5062870532274246,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.30921147763729095,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3973046988248825,
      "rewards/ReflectionBalance/mean": 0.8084236681461334,
      "rewards/ReflectionBalance/std": 0.24465181678533554,
      "step": 716,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 508.3203125,
      "completions/min_length": 232.5,
      "epoch": 3.8733153638814017,
      "grad_norm": 2.519302394593909,
      "kl": 2.515625,
      "learning_rate": 6.822901716610369e-07,
      "loss": 0.23389622569084167,
      "memory(GiB)": 75.26,
      "reward": 1.7979289293289185,
      "reward_std": 0.5644501447677612,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.3166656941175461,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3645188808441162,
      "rewards/ReflectionBalance/mean": 0.7549601197242737,
      "rewards/ReflectionBalance/std": 0.28942718356847763,
      "step": 717,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 490.76171875,
      "completions/min_length": 286.5,
      "epoch": 3.8787061994609164,
      "grad_norm": 2.799666509924083,
      "kl": 2.6640625,
      "learning_rate": 6.814910607732362e-07,
      "loss": 0.2512388229370117,
      "memory(GiB)": 75.26,
      "reward": 1.762330710887909,
      "reward_std": 0.5442482233047485,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.32267695665359497,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.32612966001033783,
      "rewards/ReflectionBalance/mean": 0.7584244906902313,
      "rewards/ReflectionBalance/std": 0.2974558025598526,
      "step": 718,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 464.515625,
      "completions/min_length": 243.0,
      "epoch": 3.884097035040431,
      "grad_norm": 2.7373879355812365,
      "kl": 2.265625,
      "learning_rate": 6.806914155940285e-07,
      "loss": 0.2058030366897583,
      "memory(GiB)": 75.26,
      "reward": 1.9225536584854126,
      "reward_std": 0.5951056778430939,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2829600051045418,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.40301957726478577,
      "rewards/ReflectionBalance/mean": 0.7897412180900574,
      "rewards/ReflectionBalance/std": 0.2752426490187645,
      "step": 719,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.5,
      "completions/mean_length": 479.453125,
      "completions/min_length": 281.0,
      "epoch": 3.889487870619946,
      "grad_norm": 3.4316329130393695,
      "kl": 3.4609375,
      "learning_rate": 6.798912384774891e-07,
      "loss": 0.2947503924369812,
      "memory(GiB)": 75.26,
      "reward": 1.86724454164505,
      "reward_std": 0.4998105466365814,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.40095221996307373,
      "rewards/ReflectionBalance/mean": 0.7734944522380829,
      "rewards/ReflectionBalance/std": 0.2826758027076721,
      "step": 720,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.5,
      "completions/mean_length": 471.3984375,
      "completions/min_length": 257.0,
      "epoch": 3.894878706199461,
      "grad_norm": 2.900868479028771,
      "kl": 2.21875,
      "learning_rate": 6.790905317792589e-07,
      "loss": 0.2011415958404541,
      "memory(GiB)": 75.26,
      "reward": 1.9441723227500916,
      "reward_std": 0.4218185693025589,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3949224203824997,
      "rewards/ReflectionBalance/mean": 0.8230786323547363,
      "rewards/ReflectionBalance/std": 0.2467796355485916,
      "step": 721,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 476.48828125,
      "completions/min_length": 268.0,
      "epoch": 3.9002695417789757,
      "grad_norm": 2.7026703288508345,
      "kl": 2.484375,
      "learning_rate": 6.782892978565376e-07,
      "loss": 0.21323835849761963,
      "memory(GiB)": 75.26,
      "reward": 1.8725391030311584,
      "reward_std": 0.48794025182724,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3985297381877899,
      "rewards/ReflectionBalance/mean": 0.7787891030311584,
      "rewards/ReflectionBalance/std": 0.29182446002960205,
      "step": 722,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 464.12890625,
      "completions/min_length": 244.0,
      "epoch": 3.9056603773584904,
      "grad_norm": 2.585129129756491,
      "kl": 1.76171875,
      "learning_rate": 6.77487539068077e-07,
      "loss": 0.16122612357139587,
      "memory(GiB)": 75.26,
      "reward": 1.9430109858512878,
      "reward_std": 0.3887673020362854,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3733299523591995,
      "rewards/ReflectionBalance/mean": 0.8219172954559326,
      "rewards/ReflectionBalance/std": 0.20088499784469604,
      "step": 723,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 479.11328125,
      "completions/min_length": 226.0,
      "epoch": 3.9110512129380055,
      "grad_norm": 2.743068054137875,
      "kl": 2.4453125,
      "learning_rate": 6.766852577741745e-07,
      "loss": 0.1831265538930893,
      "memory(GiB)": 75.26,
      "reward": 1.8940007090568542,
      "reward_std": 0.5155321657657623,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.40095221996307373,
      "rewards/ReflectionBalance/mean": 0.7885318994522095,
      "rewards/ReflectionBalance/std": 0.2612603232264519,
      "step": 724,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 471.7578125,
      "completions/min_length": 254.5,
      "epoch": 3.91644204851752,
      "grad_norm": 2.668028398648714,
      "kl": 2.203125,
      "learning_rate": 6.758824563366655e-07,
      "loss": 0.19940556585788727,
      "memory(GiB)": 75.26,
      "reward": 1.893530249595642,
      "reward_std": 0.44776035845279694,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3774644732475281,
      "rewards/ReflectionBalance/mean": 0.8114989995956421,
      "rewards/ReflectionBalance/std": 0.22119639068841934,
      "step": 725,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 452.53125,
      "completions/min_length": 241.0,
      "epoch": 3.921832884097035,
      "grad_norm": 3.8158988264524933,
      "kl": 2.6328125,
      "learning_rate": 6.750791371189165e-07,
      "loss": 0.22766900062561035,
      "memory(GiB)": 75.26,
      "reward": 1.86003577709198,
      "reward_std": 0.48222462832927704,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31333550810813904,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.37864522635936737,
      "rewards/ReflectionBalance/mean": 0.7897233366966248,
      "rewards/ReflectionBalance/std": 0.2828822284936905,
      "step": 726,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.5,
      "completions/mean_length": 507.515625,
      "completions/min_length": 195.5,
      "epoch": 3.92722371967655,
      "grad_norm": 2.6945563539327457,
      "kl": 2.90625,
      "learning_rate": 6.742753024858183e-07,
      "loss": 0.24397584795951843,
      "memory(GiB)": 75.26,
      "reward": 1.7653401494026184,
      "reward_std": 0.5735636800527573,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3208130896091461,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3229459524154663,
      "rewards/ReflectionBalance/mean": 0.7614338994026184,
      "rewards/ReflectionBalance/std": 0.3160513639450073,
      "step": 727,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1048.5,
      "completions/mean_length": 501.3359375,
      "completions/min_length": 248.0,
      "epoch": 3.9326145552560647,
      "grad_norm": 5.067124533354932,
      "kl": 3.6953125,
      "learning_rate": 6.734709548037792e-07,
      "loss": 0.2874712646007538,
      "memory(GiB)": 75.26,
      "reward": 1.8441373705863953,
      "reward_std": 0.5703986883163452,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.35538628697395325,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42010872066020966,
      "rewards/ReflectionBalance/mean": 0.7503874003887177,
      "rewards/ReflectionBalance/std": 0.3362247943878174,
      "step": 728,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 495.4140625,
      "completions/min_length": 168.5,
      "epoch": 3.9380053908355794,
      "grad_norm": 2.571526348753081,
      "kl": 2.7734375,
      "learning_rate": 6.726660964407176e-07,
      "loss": 0.230971097946167,
      "memory(GiB)": 75.26,
      "reward": 1.8554182648658752,
      "reward_std": 0.49609386920928955,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.29639076441526413,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3597298264503479,
      "rewards/ReflectionBalance/mean": 0.8085432648658752,
      "rewards/ReflectionBalance/std": 0.21795228123664856,
      "step": 729,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 476.625,
      "completions/min_length": 250.5,
      "epoch": 3.9433962264150946,
      "grad_norm": 3.1867485683331505,
      "kl": 1.9453125,
      "learning_rate": 6.718607297660557e-07,
      "loss": 0.17830801010131836,
      "memory(GiB)": 75.26,
      "reward": 1.8620458245277405,
      "reward_std": 0.5071020871400833,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3666747808456421,
      "rewards/ReflectionBalance/mean": 0.7917333841323853,
      "rewards/ReflectionBalance/std": 0.26576952636241913,
      "step": 730,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 508.0390625,
      "completions/min_length": 270.0,
      "epoch": 3.9487870619946093,
      "grad_norm": 2.837939264925716,
      "kl": 2.6796875,
      "learning_rate": 6.71054857150712e-07,
      "loss": 0.2353876531124115,
      "memory(GiB)": 75.26,
      "reward": 1.8267744779586792,
      "reward_std": 0.48281072080135345,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3437131494283676,
      "rewards/ReflectionBalance/mean": 0.7955244481563568,
      "rewards/ReflectionBalance/std": 0.2491610124707222,
      "step": 731,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 463.03515625,
      "completions/min_length": 158.0,
      "epoch": 3.954177897574124,
      "grad_norm": 3.960312765650359,
      "kl": 2.2890625,
      "learning_rate": 6.70248480967094e-07,
      "loss": 0.1526850163936615,
      "memory(GiB)": 75.26,
      "reward": 1.954197645187378,
      "reward_std": 0.5093957036733627,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.4407331943511963,
      "rewards/ReflectionBalance/mean": 0.7940413355827332,
      "rewards/ReflectionBalance/std": 0.2494587078690529,
      "step": 732,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 482.92578125,
      "completions/min_length": 243.0,
      "epoch": 3.9595687331536387,
      "grad_norm": 3.1626033255328707,
      "kl": 2.57421875,
      "learning_rate": 6.69441603589092e-07,
      "loss": 0.2163044810295105,
      "memory(GiB)": 75.26,
      "reward": 1.7645695209503174,
      "reward_std": 0.5523200631141663,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3142324984073639,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.30072054266929626,
      "rewards/ReflectionBalance/mean": 0.780194491147995,
      "rewards/ReflectionBalance/std": 0.2953507527709007,
      "step": 733,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 471.42578125,
      "completions/min_length": 237.5,
      "epoch": 3.964959568733154,
      "grad_norm": 2.5945153244918875,
      "kl": 2.1015625,
      "learning_rate": 6.68634227392072e-07,
      "loss": 0.1980929672718048,
      "memory(GiB)": 75.26,
      "reward": 1.9627197980880737,
      "reward_std": 0.4440998435020447,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42010872066020966,
      "rewards/ReflectionBalance/mean": 0.8025634586811066,
      "rewards/ReflectionBalance/std": 0.25327757000923157,
      "step": 734,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 484.1796875,
      "completions/min_length": 255.5,
      "epoch": 3.9703504043126685,
      "grad_norm": 3.341532082797806,
      "kl": 1.58984375,
      "learning_rate": 6.67826354752868e-07,
      "loss": 0.14319255948066711,
      "memory(GiB)": 75.26,
      "reward": 1.8433365225791931,
      "reward_std": 0.4016661196947098,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.28863146156072617,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3174293115735054,
      "rewards/ReflectionBalance/mean": 0.8159928321838379,
      "rewards/ReflectionBalance/std": 0.20299120992422104,
      "step": 735,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 505.01953125,
      "completions/min_length": 266.5,
      "epoch": 3.975741239892183,
      "grad_norm": 3.2490105655370733,
      "kl": 2.5703125,
      "learning_rate": 6.67017988049776e-07,
      "loss": 0.2292959839105606,
      "memory(GiB)": 75.26,
      "reward": 1.97002112865448,
      "reward_std": 0.5178980827331543,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.43914902210235596,
      "rewards/ReflectionBalance/mean": 0.8098649084568024,
      "rewards/ReflectionBalance/std": 0.24108454585075378,
      "step": 736,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 491.359375,
      "completions/min_length": 306.5,
      "epoch": 3.981132075471698,
      "grad_norm": 1171.6254627401172,
      "kl": 4.1328125,
      "learning_rate": 6.662091296625456e-07,
      "loss": 0.2976890504360199,
      "memory(GiB)": 75.26,
      "reward": 1.917190432548523,
      "reward_std": 0.4430815130472183,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3854355216026306,
      "rewards/ReflectionBalance/mean": 0.8117217123508453,
      "rewards/ReflectionBalance/std": 0.24561936408281326,
      "step": 737,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 471.69921875,
      "completions/min_length": 277.5,
      "epoch": 3.986522911051213,
      "grad_norm": 2.6316012810264082,
      "kl": 1.322265625,
      "learning_rate": 6.653997819723752e-07,
      "loss": 0.11990901827812195,
      "memory(GiB)": 75.26,
      "reward": 1.9548968076705933,
      "reward_std": 0.3764268755912781,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21138381212949753,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.37352070212364197,
      "rewards/ReflectionBalance/mean": 0.8298968374729156,
      "rewards/ReflectionBalance/std": 0.22249584645032883,
      "step": 738,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 488.4609375,
      "completions/min_length": 263.5,
      "epoch": 3.9919137466307277,
      "grad_norm": 3.1416366372021414,
      "kl": 2.046875,
      "learning_rate": 6.645899473619021e-07,
      "loss": 0.20058751106262207,
      "memory(GiB)": 75.26,
      "reward": 1.957679033279419,
      "reward_std": 0.47459109127521515,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.4154975265264511,
      "rewards/ReflectionBalance/mean": 0.801429033279419,
      "rewards/ReflectionBalance/std": 0.25484175235033035,
      "step": 739,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 495.18359375,
      "completions/min_length": 260.5,
      "epoch": 3.9973045822102424,
      "grad_norm": 6.059211701130869,
      "kl": 4.140625,
      "learning_rate": 6.637796282151984e-07,
      "loss": 0.3081033229827881,
      "memory(GiB)": 75.26,
      "reward": 1.7844759225845337,
      "reward_std": 0.5389288365840912,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.3475525379180908,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3679620176553726,
      "rewards/ReflectionBalance/mean": 0.7649447023868561,
      "rewards/ReflectionBalance/std": 0.2836607098579407,
      "step": 740,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 998.0,
      "completions/mean_length": 480.03125,
      "completions/min_length": 260.5,
      "epoch": 4.005390835579515,
      "grad_norm": 3.02176526931636,
      "kl": 2.1171875,
      "learning_rate": 6.62968826917762e-07,
      "loss": 0.17513349652290344,
      "memory(GiB)": 75.26,
      "reward": 1.8606776595115662,
      "reward_std": 0.47646771371364594,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3533225357532501,
      "rewards/ReflectionBalance/mean": 0.8020839393138885,
      "rewards/ReflectionBalance/std": 0.26118823140859604,
      "step": 741,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 498.12109375,
      "completions/min_length": 273.0,
      "epoch": 4.010781671159029,
      "grad_norm": 3.0318954436084877,
      "kl": 3.96875,
      "learning_rate": 6.621575458565102e-07,
      "loss": 0.3516633212566376,
      "memory(GiB)": 75.26,
      "reward": 1.8238776922225952,
      "reward_std": 0.6407043635845184,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.3437131494283676,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4089515954256058,
      "rewards/ReflectionBalance/mean": 0.7457527816295624,
      "rewards/ReflectionBalance/std": 0.3191092014312744,
      "step": 742,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 522.765625,
      "completions/min_length": 291.0,
      "epoch": 4.0161725067385445,
      "grad_norm": 3.340288556489725,
      "kl": 3.90625,
      "learning_rate": 6.613457874197726e-07,
      "loss": 0.3251652717590332,
      "memory(GiB)": 75.26,
      "reward": 1.7207010388374329,
      "reward_std": 0.6333113312721252,
      "rewards/Format/mean": 0.8203125,
      "rewards/Format/std": 0.3764989674091339,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.34531235694885254,
      "rewards/ReflectionBalance/mean": 0.7558572590351105,
      "rewards/ReflectionBalance/std": 0.305716335773468,
      "step": 743,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 491.79296875,
      "completions/min_length": 240.5,
      "epoch": 4.02156334231806,
      "grad_norm": 3.2149206880833643,
      "kl": 3.3359375,
      "learning_rate": 6.605335539972845e-07,
      "loss": 0.32929766178131104,
      "memory(GiB)": 75.26,
      "reward": 1.8278483152389526,
      "reward_std": 0.5592562258243561,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3270204961299896,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36417925357818604,
      "rewards/ReflectionBalance/mean": 0.7848795354366302,
      "rewards/ReflectionBalance/std": 0.24565134942531586,
      "step": 744,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 521.3671875,
      "completions/min_length": 277.5,
      "epoch": 4.026954177897574,
      "grad_norm": 3.0585221234264797,
      "kl": 3.640625,
      "learning_rate": 6.59720847980179e-07,
      "loss": 0.3249925673007965,
      "memory(GiB)": 75.26,
      "reward": 1.7081689834594727,
      "reward_std": 0.6520623564720154,
      "rewards/Format/mean": 0.8359375,
      "rewards/Format/std": 0.3716365247964859,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.3338889628648758,
      "rewards/ReflectionBalance/mean": 0.7433252334594727,
      "rewards/ReflectionBalance/std": 0.33392947912216187,
      "step": 745,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 508.53515625,
      "completions/min_length": 296.5,
      "epoch": 4.032345013477089,
      "grad_norm": 3.706076642997158,
      "kl": 3.3671875,
      "learning_rate": 6.589076717609812e-07,
      "loss": 0.2682369351387024,
      "memory(GiB)": 75.26,
      "reward": 1.8809212446212769,
      "reward_std": 0.569036215543747,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.33116789162158966,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.4164551943540573,
      "rewards/ReflectionBalance/mean": 0.7793587446212769,
      "rewards/ReflectionBalance/std": 0.2942711114883423,
      "step": 746,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 514.4140625,
      "completions/min_length": 255.0,
      "epoch": 4.037735849056604,
      "grad_norm": 3.2145337410072323,
      "kl": 2.796875,
      "learning_rate": 6.580940277335998e-07,
      "loss": 0.22613322734832764,
      "memory(GiB)": 75.26,
      "reward": 1.7878776788711548,
      "reward_std": 0.5575689077377319,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.339918777346611,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35259076952934265,
      "rewards/ReflectionBalance/mean": 0.7722525894641876,
      "rewards/ReflectionBalance/std": 0.2855135649442673,
      "step": 747,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 487.34765625,
      "completions/min_length": 171.5,
      "epoch": 4.0431266846361185,
      "grad_norm": 2.6226048849363806,
      "kl": 1.8125,
      "learning_rate": 6.572799182933209e-07,
      "loss": 0.13718181848526,
      "memory(GiB)": 75.26,
      "reward": 1.9296403527259827,
      "reward_std": 0.4829599857330322,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.4002292603254318,
      "rewards/ReflectionBalance/mean": 0.8046402931213379,
      "rewards/ReflectionBalance/std": 0.2882899045944214,
      "step": 748,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 466.703125,
      "completions/min_length": 211.0,
      "epoch": 4.048517520215634,
      "grad_norm": 2.7124574484459503,
      "kl": 1.36328125,
      "learning_rate": 6.564653458368007e-07,
      "loss": 0.10139761120080948,
      "memory(GiB)": 75.26,
      "reward": 1.9349812269210815,
      "reward_std": 0.45174089074134827,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.37864522635936737,
      "rewards/ReflectionBalance/mean": 0.8295125365257263,
      "rewards/ReflectionBalance/std": 0.22031350433826447,
      "step": 749,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 479.2109375,
      "completions/min_length": 275.5,
      "epoch": 4.053908355795148,
      "grad_norm": 2.8289016615837608,
      "kl": 1.0546875,
      "learning_rate": 6.556503127620584e-07,
      "loss": 0.08663621544837952,
      "memory(GiB)": 75.26,
      "reward": 1.92119961977005,
      "reward_std": 0.40458373725414276,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.365008607506752,
      "rewards/ReflectionBalance/mean": 0.8235432803630829,
      "rewards/ReflectionBalance/std": 0.18723148852586746,
      "step": 750,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 503.1796875,
      "completions/min_length": 259.0,
      "epoch": 4.059299191374663,
      "grad_norm": 2.5921910415958918,
      "kl": 1.65234375,
      "learning_rate": 6.548348214684693e-07,
      "loss": 0.15414077043533325,
      "memory(GiB)": 75.26,
      "reward": 1.86616051197052,
      "reward_std": 0.4182058125734329,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.2389676719903946,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.31088338047266006,
      "rewards/ReflectionBalance/mean": 0.8153792321681976,
      "rewards/ReflectionBalance/std": 0.19363103806972504,
      "step": 751,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 506.6953125,
      "completions/min_length": 281.5,
      "epoch": 4.064690026954178,
      "grad_norm": 3.3670075149593686,
      "kl": 2.6875,
      "learning_rate": 6.540188743567579e-07,
      "loss": 0.2269287407398224,
      "memory(GiB)": 75.26,
      "reward": 1.9653645753860474,
      "reward_std": 0.5517763793468475,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.4466951787471771,
      "rewards/ReflectionBalance/mean": 0.7934896051883698,
      "rewards/ReflectionBalance/std": 0.24492205679416656,
      "step": 752,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 515.6015625,
      "completions/min_length": 278.5,
      "epoch": 4.070080862533692,
      "grad_norm": 2.9800406129779975,
      "kl": 3.015625,
      "learning_rate": 6.5320247382899e-07,
      "loss": 0.25865787267684937,
      "memory(GiB)": 75.26,
      "reward": 1.7253702878952026,
      "reward_std": 0.5762501955032349,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3247780203819275,
      "rewards/MultiModalAccuracyORM/mean": 0.0859375,
      "rewards/MultiModalAccuracyORM/std": 0.2810242921113968,
      "rewards/ReflectionBalance/mean": 0.7605265378952026,
      "rewards/ReflectionBalance/std": 0.3158799558877945,
      "step": 753,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 507.9921875,
      "completions/min_length": 280.0,
      "epoch": 4.0754716981132075,
      "grad_norm": 2.9655644307157063,
      "kl": 4.0078125,
      "learning_rate": 6.523856222885669e-07,
      "loss": 0.361136794090271,
      "memory(GiB)": 75.26,
      "reward": 1.78899085521698,
      "reward_std": 0.6226524412631989,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3318064510822296,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3475525379180908,
      "rewards/ReflectionBalance/mean": 0.76945960521698,
      "rewards/ReflectionBalance/std": 0.32005856931209564,
      "step": 754,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 486.88671875,
      "completions/min_length": 266.5,
      "epoch": 4.080862533692723,
      "grad_norm": 5.553161469175707,
      "kl": 2.9609375,
      "learning_rate": 6.515683221402169e-07,
      "loss": 0.26827192306518555,
      "memory(GiB)": 75.26,
      "reward": 1.8526546359062195,
      "reward_std": 0.5424023866653442,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3734995424747467,
      "rewards/ReflectionBalance/mean": 0.7667171359062195,
      "rewards/ReflectionBalance/std": 0.2981405258178711,
      "step": 755,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 452.6171875,
      "completions/min_length": 261.5,
      "epoch": 4.086253369272237,
      "grad_norm": 2.832687623450174,
      "kl": 1.54296875,
      "learning_rate": 6.507505757899893e-07,
      "loss": 0.13699428737163544,
      "memory(GiB)": 75.26,
      "reward": 2.0670214891433716,
      "reward_std": 0.39402760565280914,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.3203125,
      "rewards/MultiModalAccuracyORM/std": 0.4672212451696396,
      "rewards/ReflectionBalance/mean": 0.8131152093410492,
      "rewards/ReflectionBalance/std": 0.23225896805524826,
      "step": 756,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 467.9296875,
      "completions/min_length": 279.5,
      "epoch": 4.091644204851752,
      "grad_norm": 3.132435760600758,
      "kl": 2.21875,
      "learning_rate": 6.499323856452475e-07,
      "loss": 0.21858081221580505,
      "memory(GiB)": 75.26,
      "reward": 1.9043277502059937,
      "reward_std": 0.4309155344963074,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3666747808456421,
      "rewards/ReflectionBalance/mean": 0.810577780008316,
      "rewards/ReflectionBalance/std": 0.22458013892173767,
      "step": 757,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 464.90234375,
      "completions/min_length": 209.0,
      "epoch": 4.097035040431267,
      "grad_norm": 3.6686182860270673,
      "kl": 3.1484375,
      "learning_rate": 6.491137541146607e-07,
      "loss": 0.25257450342178345,
      "memory(GiB)": 75.26,
      "reward": 1.8894091844558716,
      "reward_std": 0.5063847601413727,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3208130896091461,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.38954539597034454,
      "rewards/ReflectionBalance/mean": 0.8151904344558716,
      "rewards/ReflectionBalance/std": 0.22175227850675583,
      "step": 758,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 477.109375,
      "completions/min_length": 267.5,
      "epoch": 4.1024258760107815,
      "grad_norm": 3.297683581909078,
      "kl": 2.8515625,
      "learning_rate": 6.482946836081974e-07,
      "loss": 0.24055977165699005,
      "memory(GiB)": 75.26,
      "reward": 1.8794400691986084,
      "reward_std": 0.5101277232170105,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3804178833961487,
      "rewards/ReflectionBalance/mean": 0.8130338191986084,
      "rewards/ReflectionBalance/std": 0.2150231897830963,
      "step": 759,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 485.15625,
      "completions/min_length": 258.5,
      "epoch": 4.107816711590297,
      "grad_norm": 2.481770865469386,
      "kl": 3.0859375,
      "learning_rate": 6.47475176537119e-07,
      "loss": 0.2836550772190094,
      "memory(GiB)": 75.26,
      "reward": 1.979062557220459,
      "reward_std": 0.4785902351140976,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.4412749409675598,
      "rewards/ReflectionBalance/mean": 0.8150000274181366,
      "rewards/ReflectionBalance/std": 0.22702883183956146,
      "step": 760,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 482.0,
      "completions/min_length": 283.0,
      "epoch": 4.113207547169812,
      "grad_norm": 4.66171877054184,
      "kl": 3.6796875,
      "learning_rate": 6.466552353139716e-07,
      "loss": 0.2771633267402649,
      "memory(GiB)": 75.26,
      "reward": 1.8311409950256348,
      "reward_std": 0.5354161560535431,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3375232368707657,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3853000998497009,
      "rewards/ReflectionBalance/mean": 0.78426593542099,
      "rewards/ReflectionBalance/std": 0.2644483745098114,
      "step": 761,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 496.5625,
      "completions/min_length": 297.0,
      "epoch": 4.118598382749326,
      "grad_norm": 3.3775200928438744,
      "kl": 3.4609375,
      "learning_rate": 6.458348623525798e-07,
      "loss": 0.2822043299674988,
      "memory(GiB)": 75.26,
      "reward": 1.8045881986618042,
      "reward_std": 0.5186426341533661,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3586205691099167,
      "rewards/ReflectionBalance/mean": 0.7538069486618042,
      "rewards/ReflectionBalance/std": 0.3117240369319916,
      "step": 762,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 475.34765625,
      "completions/min_length": 268.5,
      "epoch": 4.123989218328841,
      "grad_norm": 4.225733579808764,
      "kl": 3.6640625,
      "learning_rate": 6.450140600680387e-07,
      "loss": 0.3034987449645996,
      "memory(GiB)": 75.26,
      "reward": 1.870168387889862,
      "reward_std": 0.4394298195838928,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38105446100234985,
      "rewards/ReflectionBalance/mean": 0.7881371974945068,
      "rewards/ReflectionBalance/std": 0.2342882752418518,
      "step": 763,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 467.83984375,
      "completions/min_length": 251.0,
      "epoch": 4.129380053908355,
      "grad_norm": 2.6914143917611786,
      "kl": 2.78125,
      "learning_rate": 6.441928308767078e-07,
      "loss": 0.2641245722770691,
      "memory(GiB)": 75.26,
      "reward": 1.9173794984817505,
      "reward_std": 0.46802982687950134,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.40016549825668335,
      "rewards/ReflectionBalance/mean": 0.7923795580863953,
      "rewards/ReflectionBalance/std": 0.26164552569389343,
      "step": 764,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 442.57421875,
      "completions/min_length": 231.5,
      "epoch": 4.1347708894878705,
      "grad_norm": 2.922383654933877,
      "kl": 2.6953125,
      "learning_rate": 6.433711771962031e-07,
      "loss": 0.23191964626312256,
      "memory(GiB)": 75.26,
      "reward": 1.8884227871894836,
      "reward_std": 0.5090112835168839,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.40759414434432983,
      "rewards/ReflectionBalance/mean": 0.7829541265964508,
      "rewards/ReflectionBalance/std": 0.25696250796318054,
      "step": 765,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 480.40625,
      "completions/min_length": 274.0,
      "epoch": 4.140161725067386,
      "grad_norm": 3.3636242148361033,
      "kl": 2.73828125,
      "learning_rate": 6.425491014453901e-07,
      "loss": 0.24339117109775543,
      "memory(GiB)": 75.26,
      "reward": 1.9536539316177368,
      "reward_std": 0.5723270773887634,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30283892154693604,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.43903933465480804,
      "rewards/ReflectionBalance/mean": 0.7895914912223816,
      "rewards/ReflectionBalance/std": 0.27841976284980774,
      "step": 766,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 498.08984375,
      "completions/min_length": 278.5,
      "epoch": 4.1455525606469,
      "grad_norm": 2.776170758596027,
      "kl": 3.4453125,
      "learning_rate": 6.417266060443774e-07,
      "loss": 0.3044986426830292,
      "memory(GiB)": 75.26,
      "reward": 1.8711459040641785,
      "reward_std": 0.5226618647575378,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4077630192041397,
      "rewards/ReflectionBalance/mean": 0.7695833146572113,
      "rewards/ReflectionBalance/std": 0.2769088074564934,
      "step": 767,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 450.2265625,
      "completions/min_length": 212.0,
      "epoch": 4.150943396226415,
      "grad_norm": 2.5205562108034893,
      "kl": 2.3828125,
      "learning_rate": 6.409036934145084e-07,
      "loss": 0.1989305168390274,
      "memory(GiB)": 75.26,
      "reward": 1.9753369688987732,
      "reward_std": 0.5227498412132263,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2801405265927315,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.45274218916893005,
      "rewards/ReflectionBalance/mean": 0.7800244390964508,
      "rewards/ReflectionBalance/std": 0.2802596315741539,
      "step": 768,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 460.6015625,
      "completions/min_length": 272.5,
      "epoch": 4.15633423180593,
      "grad_norm": 2.81824418996499,
      "kl": 1.8984375,
      "learning_rate": 6.40080365978355e-07,
      "loss": 0.18530189990997314,
      "memory(GiB)": 75.26,
      "reward": 1.9426497220993042,
      "reward_std": 0.4808390587568283,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41635647416114807,
      "rewards/ReflectionBalance/mean": 0.7981184720993042,
      "rewards/ReflectionBalance/std": 0.2446039319038391,
      "step": 769,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 458.5390625,
      "completions/min_length": 197.5,
      "epoch": 4.1617250673854445,
      "grad_norm": 2.6554928728957545,
      "kl": 1.9453125,
      "learning_rate": 6.392566261597103e-07,
      "loss": 0.17687204480171204,
      "memory(GiB)": 75.26,
      "reward": 1.8970133066177368,
      "reward_std": 0.46515342593193054,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2897924929857254,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3804178833961487,
      "rewards/ReflectionBalance/mean": 0.8149821162223816,
      "rewards/ReflectionBalance/std": 0.23240601271390915,
      "step": 770,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 462.65234375,
      "completions/min_length": 252.0,
      "epoch": 4.16711590296496,
      "grad_norm": 3.95216907796308,
      "kl": 1.302734375,
      "learning_rate": 6.384324763835814e-07,
      "loss": 0.1356891393661499,
      "memory(GiB)": 75.26,
      "reward": 1.962436556816101,
      "reward_std": 0.41970644891262054,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2255912721157074,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.39205317199230194,
      "rewards/ReflectionBalance/mean": 0.8179052472114563,
      "rewards/ReflectionBalance/std": 0.220673106610775,
      "step": 771,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 455.50390625,
      "completions/min_length": 302.5,
      "epoch": 4.172506738544475,
      "grad_norm": 3.3166785193794546,
      "kl": 0.962890625,
      "learning_rate": 6.376079190761821e-07,
      "loss": 0.09437630325555801,
      "memory(GiB)": 75.26,
      "reward": 1.9729199409484863,
      "reward_std": 0.32918494939804077,
      "rewards/Format/mean": 0.96875,
      "rewards/Format/std": 0.1746762990951538,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3734995424747467,
      "rewards/ReflectionBalance/mean": 0.8362011611461639,
      "rewards/ReflectionBalance/std": 0.18464774638414383,
      "step": 772,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 473.59765625,
      "completions/min_length": 226.0,
      "epoch": 4.177897574123989,
      "grad_norm": 2.9262566309115425,
      "kl": 1.62109375,
      "learning_rate": 6.367829566649265e-07,
      "loss": 0.1567833572626114,
      "memory(GiB)": 75.26,
      "reward": 1.849560558795929,
      "reward_std": 0.4843897223472595,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.32345911115407944,
      "rewards/ReflectionBalance/mean": 0.783154308795929,
      "rewards/ReflectionBalance/std": 0.2901112586259842,
      "step": 773,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 972.5,
      "completions/mean_length": 462.7578125,
      "completions/min_length": 267.5,
      "epoch": 4.183288409703504,
      "grad_norm": 3.574288061316224,
      "kl": 1.26953125,
      "learning_rate": 6.359575915784204e-07,
      "loss": 0.129658505320549,
      "memory(GiB)": 75.26,
      "reward": 2.0255126953125,
      "reward_std": 0.4527505189180374,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.20884444564580917,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42921192944049835,
      "rewards/ReflectionBalance/mean": 0.8302001953125,
      "rewards/ReflectionBalance/std": 0.22601629048585892,
      "step": 774,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.5,
      "completions/mean_length": 468.58203125,
      "completions/min_length": 272.5,
      "epoch": 4.188679245283019,
      "grad_norm": 2.798994836647241,
      "kl": 2.125,
      "learning_rate": 6.351318262464558e-07,
      "loss": 0.1868050992488861,
      "memory(GiB)": 75.26,
      "reward": 2.0232739448547363,
      "reward_std": 0.582982748746872,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.33203125,
      "rewards/MultiModalAccuracyORM/std": 0.46967945992946625,
      "rewards/ReflectionBalance/mean": 0.7928051948547363,
      "rewards/ReflectionBalance/std": 0.2617168426513672,
      "step": 775,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 478.84375,
      "completions/min_length": 253.5,
      "epoch": 4.1940700808625335,
      "grad_norm": 2.7986257069307108,
      "kl": 2.328125,
      "learning_rate": 6.343056631000023e-07,
      "loss": 0.1936497688293457,
      "memory(GiB)": 75.26,
      "reward": 1.9593993425369263,
      "reward_std": 0.4753054082393646,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4168776273727417,
      "rewards/ReflectionBalance/mean": 0.8031494319438934,
      "rewards/ReflectionBalance/std": 0.23910431563854218,
      "step": 776,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 466.19140625,
      "completions/min_length": 259.0,
      "epoch": 4.199460916442049,
      "grad_norm": 6.6720350048335195,
      "kl": 2.71875,
      "learning_rate": 6.334791045712014e-07,
      "loss": 0.22571179270744324,
      "memory(GiB)": 75.26,
      "reward": 1.8275080919265747,
      "reward_std": 0.42973268032073975,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.31763872504234314,
      "rewards/ReflectionBalance/mean": 0.7884456515312195,
      "rewards/ReflectionBalance/std": 0.2772039771080017,
      "step": 777,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 467.7578125,
      "completions/min_length": 211.5,
      "epoch": 4.204851752021563,
      "grad_norm": 3.467211664764973,
      "kl": 3.90625,
      "learning_rate": 6.32652153093358e-07,
      "loss": 0.3415948748588562,
      "memory(GiB)": 75.26,
      "reward": 1.7232590913772583,
      "reward_std": 0.6353081166744232,
      "rewards/Format/mean": 0.83203125,
      "rewards/Format/std": 0.368965208530426,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.3541714698076248,
      "rewards/ReflectionBalance/mean": 0.7427903711795807,
      "rewards/ReflectionBalance/std": 0.31386415660381317,
      "step": 778,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 475.9921875,
      "completions/min_length": 258.5,
      "epoch": 4.210242587601078,
      "grad_norm": 3.0518541757105893,
      "kl": 3.5859375,
      "learning_rate": 6.318248111009341e-07,
      "loss": 0.3096701204776764,
      "memory(GiB)": 75.26,
      "reward": 1.8933886885643005,
      "reward_std": 0.6131230890750885,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.34485287964344025,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4168776273727417,
      "rewards/ReflectionBalance/mean": 0.7840136587619781,
      "rewards/ReflectionBalance/std": 0.28320200741291046,
      "step": 779,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 466.78515625,
      "completions/min_length": 245.0,
      "epoch": 4.215633423180593,
      "grad_norm": 2.8923559947908593,
      "kl": 3.4609375,
      "learning_rate": 6.309970810295411e-07,
      "loss": 0.288682758808136,
      "memory(GiB)": 75.26,
      "reward": 1.8098526000976562,
      "reward_std": 0.6270154118537903,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.35259076952934265,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.3979068696498871,
      "rewards/ReflectionBalance/mean": 0.7473525702953339,
      "rewards/ReflectionBalance/std": 0.3005444407463074,
      "step": 780,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 447.5625,
      "completions/min_length": 263.0,
      "epoch": 4.2210242587601075,
      "grad_norm": 2.8868949562643738,
      "kl": 2.7890625,
      "learning_rate": 6.30168965315933e-07,
      "loss": 0.22343483567237854,
      "memory(GiB)": 75.26,
      "reward": 1.953346312046051,
      "reward_std": 0.49674177169799805,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.43684960901737213,
      "rewards/ReflectionBalance/mean": 0.7775651216506958,
      "rewards/ReflectionBalance/std": 0.2793833166360855,
      "step": 781,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 484.9765625,
      "completions/min_length": 257.0,
      "epoch": 4.226415094339623,
      "grad_norm": 2.7043926115772763,
      "kl": 2.40625,
      "learning_rate": 6.29340466397999e-07,
      "loss": 0.20916588604450226,
      "memory(GiB)": 75.26,
      "reward": 1.8041179180145264,
      "reward_std": 0.5223739147186279,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.3029807060956955,
      "rewards/ReflectionBalance/mean": 0.7963053286075592,
      "rewards/ReflectionBalance/std": 0.2783872038125992,
      "step": 782,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 469.9609375,
      "completions/min_length": 258.0,
      "epoch": 4.231805929919138,
      "grad_norm": 2.97738226772157,
      "kl": 2.8671875,
      "learning_rate": 6.285115867147566e-07,
      "loss": 0.21623092889785767,
      "memory(GiB)": 75.26,
      "reward": 1.924332618713379,
      "reward_std": 0.5134974271059036,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.4176374822854996,
      "rewards/ReflectionBalance/mean": 0.7954264283180237,
      "rewards/ReflectionBalance/std": 0.2672404795885086,
      "step": 783,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 487.2734375,
      "completions/min_length": 259.0,
      "epoch": 4.237196765498652,
      "grad_norm": 2.6148108268744896,
      "kl": 1.73046875,
      "learning_rate": 6.276823287063442e-07,
      "loss": 0.1583385467529297,
      "memory(GiB)": 75.26,
      "reward": 1.957057237625122,
      "reward_std": 0.47566425800323486,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.2379390001296997,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.39950306713581085,
      "rewards/ReflectionBalance/mean": 0.8203385174274445,
      "rewards/ReflectionBalance/std": 0.2379763200879097,
      "step": 784,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 485.16796875,
      "completions/min_length": 284.0,
      "epoch": 4.242587601078167,
      "grad_norm": 3.356513591595178,
      "kl": 2.8984375,
      "learning_rate": 6.268526948140136e-07,
      "loss": 0.24296095967292786,
      "memory(GiB)": 75.26,
      "reward": 1.8756380677223206,
      "reward_std": 0.44545552134513855,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.258706197142601,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3734995424747467,
      "rewards/ReflectionBalance/mean": 0.7857942581176758,
      "rewards/ReflectionBalance/std": 0.2743857875466347,
      "step": 785,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 936.5,
      "completions/mean_length": 500.234375,
      "completions/min_length": 258.0,
      "epoch": 4.247978436657682,
      "grad_norm": 2.792083972252305,
      "kl": 1.064453125,
      "learning_rate": 6.260226874801236e-07,
      "loss": 0.07659904658794403,
      "memory(GiB)": 75.26,
      "reward": 2.0154948234558105,
      "reward_std": 0.36020347476005554,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4300411641597748,
      "rewards/ReflectionBalance/mean": 0.8279947638511658,
      "rewards/ReflectionBalance/std": 0.17639752477407455,
      "step": 786,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 512.22265625,
      "completions/min_length": 278.5,
      "epoch": 4.2533692722371965,
      "grad_norm": 9.732252843521021,
      "kl": 2.1328125,
      "learning_rate": 6.251923091481322e-07,
      "loss": 0.19808608293533325,
      "memory(GiB)": 75.26,
      "reward": 1.8704052567481995,
      "reward_std": 0.47078531980514526,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3497565984725952,
      "rewards/ReflectionBalance/mean": 0.8079052567481995,
      "rewards/ReflectionBalance/std": 0.22307805716991425,
      "step": 787,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 476.85546875,
      "completions/min_length": 267.5,
      "epoch": 4.258760107816712,
      "grad_norm": 3.455648342666831,
      "kl": 1.328125,
      "learning_rate": 6.243615622625898e-07,
      "loss": 0.14275799691677094,
      "memory(GiB)": 75.26,
      "reward": 2.0132577419281006,
      "reward_std": 0.43640996515750885,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4270860105752945,
      "rewards/ReflectionBalance/mean": 0.8257576525211334,
      "rewards/ReflectionBalance/std": 0.20222113281488419,
      "step": 788,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.0,
      "completions/mean_length": 473.89453125,
      "completions/min_length": 256.5,
      "epoch": 4.264150943396227,
      "grad_norm": 2.734642529563229,
      "kl": 1.609375,
      "learning_rate": 6.23530449269131e-07,
      "loss": 0.13836738467216492,
      "memory(GiB)": 75.26,
      "reward": 1.8893017768859863,
      "reward_std": 0.4028281718492508,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3290177956223488,
      "rewards/ReflectionBalance/mean": 0.8189892172813416,
      "rewards/ReflectionBalance/std": 0.22197586297988892,
      "step": 789,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 478.1015625,
      "completions/min_length": 257.0,
      "epoch": 4.269541778975741,
      "grad_norm": 3.0421207887924147,
      "kl": 2.890625,
      "learning_rate": 6.226989726144693e-07,
      "loss": 0.23925621807575226,
      "memory(GiB)": 75.26,
      "reward": 1.8361653685569763,
      "reward_std": 0.5018600225448608,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.2960064336657524,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35737672448158264,
      "rewards/ReflectionBalance/mean": 0.7853841185569763,
      "rewards/ReflectionBalance/std": 0.28303995728492737,
      "step": 790,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 500.44921875,
      "completions/min_length": 260.0,
      "epoch": 4.274932614555256,
      "grad_norm": 3.863274637054332,
      "kl": 3.859375,
      "learning_rate": 6.218671347463881e-07,
      "loss": 0.3163267970085144,
      "memory(GiB)": 75.26,
      "reward": 1.7925626039505005,
      "reward_std": 0.6150961816310883,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.3607245087623596,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3899272531270981,
      "rewards/ReflectionBalance/mean": 0.7495939135551453,
      "rewards/ReflectionBalance/std": 0.3167257755994797,
      "step": 791,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 497.046875,
      "completions/min_length": 257.5,
      "epoch": 4.280323450134771,
      "grad_norm": 4.638322128370218,
      "kl": 3.890625,
      "learning_rate": 6.21034938113734e-07,
      "loss": 0.28409358859062195,
      "memory(GiB)": 75.26,
      "reward": 1.6964567303657532,
      "reward_std": 0.6355946063995361,
      "rewards/Format/mean": 0.82421875,
      "rewards/Format/std": 0.3717130124568939,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3300945907831192,
      "rewards/ReflectionBalance/mean": 0.7472379505634308,
      "rewards/ReflectionBalance/std": 0.31460267305374146,
      "step": 792,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 518.5,
      "completions/min_length": 266.5,
      "epoch": 4.285714285714286,
      "grad_norm": 4.106200380436596,
      "kl": 4.1640625,
      "learning_rate": 6.202023851664106e-07,
      "loss": 0.33239221572875977,
      "memory(GiB)": 75.26,
      "reward": 1.749119520187378,
      "reward_std": 0.5234711170196533,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.36627575755119324,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.30737847089767456,
      "rewards/ReflectionBalance/mean": 0.7764632403850555,
      "rewards/ReflectionBalance/std": 0.2813785672187805,
      "step": 793,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 497.8828125,
      "completions/min_length": 179.5,
      "epoch": 4.291105121293801,
      "grad_norm": 4.683179885936009,
      "kl": 3.5234375,
      "learning_rate": 6.193694783553698e-07,
      "loss": 0.2687247693538666,
      "memory(GiB)": 75.26,
      "reward": 1.7728353142738342,
      "reward_std": 0.5009615123271942,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.3538244515657425,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.36930377781391144,
      "rewards/ReflectionBalance/mean": 0.7572102844715118,
      "rewards/ReflectionBalance/std": 0.27999334037303925,
      "step": 794,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 507.078125,
      "completions/min_length": 247.5,
      "epoch": 4.296495956873315,
      "grad_norm": 2.6048447905115286,
      "kl": 2.7578125,
      "learning_rate": 6.18536220132605e-07,
      "loss": 0.24258200824260712,
      "memory(GiB)": 75.26,
      "reward": 1.7752952575683594,
      "reward_std": 0.48120369017124176,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3285723924636841,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.310598224401474,
      "rewards/ReflectionBalance/mean": 0.7870139479637146,
      "rewards/ReflectionBalance/std": 0.26070988178253174,
      "step": 795,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1038.5,
      "completions/mean_length": 460.046875,
      "completions/min_length": 240.5,
      "epoch": 4.30188679245283,
      "grad_norm": 3.0221188695350474,
      "kl": 1.77734375,
      "learning_rate": 6.17702612951145e-07,
      "loss": 0.16408011317253113,
      "memory(GiB)": 75.26,
      "reward": 1.9576969146728516,
      "reward_std": 0.5292099863290787,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2801405265927315,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.4383239597082138,
      "rewards/ReflectionBalance/mean": 0.7897281646728516,
      "rewards/ReflectionBalance/std": 0.2549917474389076,
      "step": 796,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.5,
      "completions/mean_length": 491.25,
      "completions/min_length": 264.5,
      "epoch": 4.307277628032345,
      "grad_norm": 3.280631723465185,
      "kl": 1.76171875,
      "learning_rate": 6.168686592650452e-07,
      "loss": 0.17057280242443085,
      "memory(GiB)": 75.26,
      "reward": 1.9815998673439026,
      "reward_std": 0.48923808336257935,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4181321859359741,
      "rewards/ReflectionBalance/mean": 0.8058186769485474,
      "rewards/ReflectionBalance/std": 0.2259996458888054,
      "step": 797,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 953.0,
      "completions/mean_length": 478.50390625,
      "completions/min_length": 250.0,
      "epoch": 4.3126684636118595,
      "grad_norm": 3.6047632527308537,
      "kl": 1.5,
      "learning_rate": 6.160343615293808e-07,
      "loss": 0.16039308905601501,
      "memory(GiB)": 75.26,
      "reward": 1.8261601328849792,
      "reward_std": 0.4781358242034912,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3131764084100723,
      "rewards/ReflectionBalance/mean": 0.7870976626873016,
      "rewards/ReflectionBalance/std": 0.30194342136383057,
      "step": 798,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 486.6953125,
      "completions/min_length": 278.0,
      "epoch": 4.318059299191375,
      "grad_norm": 2.9095833132998594,
      "kl": 1.546875,
      "learning_rate": 6.151997222002412e-07,
      "loss": 0.14855068922042847,
      "memory(GiB)": 75.26,
      "reward": 1.8815918564796448,
      "reward_std": 0.487909659743309,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.35719405859708786,
      "rewards/ReflectionBalance/mean": 0.7839355766773224,
      "rewards/ReflectionBalance/std": 0.2519425228238106,
      "step": 799,
      "train_speed(iter/s)": 0.007106
    },
    {
      "epoch": 4.32345013477089,
      "grad_norm": 3.4241292019810703,
      "learning_rate": 6.1436474373472e-07,
      "loss": 0.20145705342292786,
      "memory(GiB)": 75.26,
      "step": 800,
      "train_speed(iter/s)": 0.007107
    },
    {
      "epoch": 4.32345013477089,
      "eval_clip_ratio": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 1024.75,
      "eval_completions/mean_length": 494.203125,
      "eval_completions/min_length": 216.5,
      "eval_kl": 1.71875,
      "eval_loss": 0.13286332786083221,
      "eval_reward": 1.8578100800514221,
      "eval_reward_std": 0.48047374188899994,
      "eval_rewards/Format/mean": 0.90234375,
      "eval_rewards/Format/std": 0.2948278598487377,
      "eval_rewards/MultiModalAccuracyORM/mean": 0.146484375,
      "eval_rewards/MultiModalAccuracyORM/std": 0.3498135879635811,
      "eval_rewards/ReflectionBalance/mean": 0.8089819252490997,
      "eval_rewards/ReflectionBalance/std": 0.25548624992370605,
      "eval_runtime": 248.4289,
      "eval_samples_per_second": 0.237,
      "eval_steps_per_second": 0.004,
      "step": 800
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 488.908203125,
      "completions/min_length": 254.25,
      "epoch": 4.328840970350404,
      "grad_norm": 4.225192931254194,
      "kl": 2.396484375,
      "learning_rate": 6.135294285909098e-07,
      "loss": 0.24621298909187317,
      "memory(GiB)": 75.26,
      "reward": 1.8602062463760376,
      "reward_std": 0.5635207891464233,
      "rewards/Format/mean": 0.896484375,
      "rewards/Format/std": 0.3027462214231491,
      "rewards/MultiModalAccuracyORM/mean": 0.177734375,
      "rewards/MultiModalAccuracyORM/std": 0.37800271064043045,
      "rewards/ReflectionBalance/mean": 0.78598752617836,
      "rewards/ReflectionBalance/std": 0.26880309358239174,
      "step": 801,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.5,
      "completions/mean_length": 517.4296875,
      "completions/min_length": 245.5,
      "epoch": 4.334231805929919,
      "grad_norm": 3.2856071540765024,
      "kl": 3.703125,
      "learning_rate": 6.126937792278944e-07,
      "loss": 0.3106701970100403,
      "memory(GiB)": 75.26,
      "reward": 1.7590267062187195,
      "reward_std": 0.6290370225906372,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3473474234342575,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3422359675168991,
      "rewards/ReflectionBalance/mean": 0.7590267062187195,
      "rewards/ReflectionBalance/std": 0.28692011535167694,
      "step": 802,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 536.734375,
      "completions/min_length": 265.0,
      "epoch": 4.339622641509434,
      "grad_norm": 3.2240375722700603,
      "kl": 3.59375,
      "learning_rate": 6.11857798105741e-07,
      "loss": 0.30309006571769714,
      "memory(GiB)": 75.26,
      "reward": 1.7604979872703552,
      "reward_std": 0.6327281892299652,
      "rewards/Format/mean": 0.828125,
      "rewards/Format/std": 0.37818245589733124,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3804178833961487,
      "rewards/ReflectionBalance/mean": 0.756591796875,
      "rewards/ReflectionBalance/std": 0.30654262006282806,
      "step": 803,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 490.57421875,
      "completions/min_length": 264.5,
      "epoch": 4.345013477088949,
      "grad_norm": 3.076116600697986,
      "kl": 3.546875,
      "learning_rate": 6.110214876854942e-07,
      "loss": 0.30490878224372864,
      "memory(GiB)": 75.26,
      "reward": 1.9066845178604126,
      "reward_std": 0.5368396639823914,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3208130896091461,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.4176374822854996,
      "rewards/ReflectionBalance/mean": 0.801215797662735,
      "rewards/ReflectionBalance/std": 0.22955965250730515,
      "step": 804,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 524.359375,
      "completions/min_length": 248.0,
      "epoch": 4.350404312668464,
      "grad_norm": 5.633549406497026,
      "kl": 4.1875,
      "learning_rate": 6.101848504291676e-07,
      "loss": 0.33758270740509033,
      "memory(GiB)": 75.26,
      "reward": 1.7638306021690369,
      "reward_std": 0.6149934232234955,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.3607245087623596,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.35641810297966003,
      "rewards/ReflectionBalance/mean": 0.7599243223667145,
      "rewards/ReflectionBalance/std": 0.2893780916929245,
      "step": 805,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 505.26171875,
      "completions/min_length": 170.5,
      "epoch": 4.355795148247978,
      "grad_norm": 5.9552911197933245,
      "kl": 5.796875,
      "learning_rate": 6.093478887997371e-07,
      "loss": 0.45394831895828247,
      "memory(GiB)": 75.26,
      "reward": 1.675464689731598,
      "reward_std": 0.6671134531497955,
      "rewards/Format/mean": 0.8046875,
      "rewards/Format/std": 0.3948954641819,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.365008607506752,
      "rewards/ReflectionBalance/mean": 0.7106209099292755,
      "rewards/ReflectionBalance/std": 0.3603398948907852,
      "step": 806,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 488.90234375,
      "completions/min_length": 177.5,
      "epoch": 4.361185983827493,
      "grad_norm": 2.8470103522746513,
      "kl": 2.3046875,
      "learning_rate": 6.085106052611333e-07,
      "loss": 0.19583657383918762,
      "memory(GiB)": 75.26,
      "reward": 1.7583788633346558,
      "reward_std": 0.47335731983184814,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.07421875,
      "rewards/MultiModalAccuracyORM/std": 0.2621928080916405,
      "rewards/ReflectionBalance/mean": 0.7857226729393005,
      "rewards/ReflectionBalance/std": 0.27189527451992035,
      "step": 807,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 463.62890625,
      "completions/min_length": 259.5,
      "epoch": 4.366576819407008,
      "grad_norm": 2.9831285283873687,
      "kl": 1.31640625,
      "learning_rate": 6.076730022782345e-07,
      "loss": 0.1380525827407837,
      "memory(GiB)": 75.26,
      "reward": 2.0125166177749634,
      "reward_std": 0.3549502193927765,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21875878423452377,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.42273683845996857,
      "rewards/ReflectionBalance/mean": 0.8289228677749634,
      "rewards/ReflectionBalance/std": 0.19050735980272293,
      "step": 808,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 489.453125,
      "completions/min_length": 293.0,
      "epoch": 4.3719676549865225,
      "grad_norm": 2.8876232103904167,
      "kl": 2.5078125,
      "learning_rate": 6.068350823168597e-07,
      "loss": 0.21213190257549286,
      "memory(GiB)": 75.26,
      "reward": 1.9036905765533447,
      "reward_std": 0.5540053248405457,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.41096827387809753,
      "rewards/ReflectionBalance/mean": 0.7904093265533447,
      "rewards/ReflectionBalance/std": 0.25887925922870636,
      "step": 809,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 496.75390625,
      "completions/min_length": 284.5,
      "epoch": 4.377358490566038,
      "grad_norm": 3.0106799570626803,
      "kl": 3.0234375,
      "learning_rate": 6.059968478437612e-07,
      "loss": 0.25954198837280273,
      "memory(GiB)": 75.26,
      "reward": 1.92239248752594,
      "reward_std": 0.6007936298847198,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.32267695665359497,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4322105348110199,
      "rewards/ReflectionBalance/mean": 0.7934863269329071,
      "rewards/ReflectionBalance/std": 0.2389661744236946,
      "step": 810,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 476.97265625,
      "completions/min_length": 272.5,
      "epoch": 4.382749326145553,
      "grad_norm": 3.367372226201481,
      "kl": 3.015625,
      "learning_rate": 6.051583013266163e-07,
      "loss": 0.2507515847682953,
      "memory(GiB)": 75.26,
      "reward": 1.9208300113677979,
      "reward_std": 0.5268263071775436,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.30921147763729095,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.42014752328395844,
      "rewards/ReflectionBalance/mean": 0.8036425709724426,
      "rewards/ReflectionBalance/std": 0.24784217029809952,
      "step": 811,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 454.3828125,
      "completions/min_length": 266.5,
      "epoch": 4.388140161725067,
      "grad_norm": 3.1223023498453726,
      "kl": 1.6640625,
      "learning_rate": 6.043194452340219e-07,
      "loss": 0.1634920835494995,
      "memory(GiB)": 75.26,
      "reward": 2.05588960647583,
      "reward_std": 0.4849771559238434,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.32421875,
      "rewards/MultiModalAccuracyORM/std": 0.4694543182849884,
      "rewards/ReflectionBalance/mean": 0.8058894872665405,
      "rewards/ReflectionBalance/std": 0.2556629702448845,
      "step": 812,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 516.8515625,
      "completions/min_length": 285.0,
      "epoch": 4.393530997304582,
      "grad_norm": 35.61279605309034,
      "kl": 3.4375,
      "learning_rate": 6.034802820354857e-07,
      "loss": 0.28302836418151855,
      "memory(GiB)": 75.26,
      "reward": 1.9213981628417969,
      "reward_std": 0.5016423463821411,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.40567632019519806,
      "rewards/ReflectionBalance/mean": 0.7807731032371521,
      "rewards/ReflectionBalance/std": 0.28716662526130676,
      "step": 813,
      "train_speed(iter/s)": 0.007086
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 512.84375,
      "completions/min_length": 287.5,
      "epoch": 4.398921832884097,
      "grad_norm": 3.218478942145696,
      "kl": 2.8671875,
      "learning_rate": 6.026408142014197e-07,
      "loss": 0.2481497824192047,
      "memory(GiB)": 75.26,
      "reward": 1.8598014116287231,
      "reward_std": 0.5725324004888535,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.332206666469574,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.40016549825668335,
      "rewards/ReflectionBalance/mean": 0.7816764712333679,
      "rewards/ReflectionBalance/std": 0.2645088881254196,
      "step": 814,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 476.1328125,
      "completions/min_length": 188.5,
      "epoch": 4.404312668463612,
      "grad_norm": 2.9469016975463957,
      "kl": 2.7421875,
      "learning_rate": 6.018010442031325e-07,
      "loss": 0.22356575727462769,
      "memory(GiB)": 75.26,
      "reward": 1.8148159980773926,
      "reward_std": 0.4878773093223572,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.2960064336657524,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.3191515803337097,
      "rewards/ReflectionBalance/mean": 0.7991911172866821,
      "rewards/ReflectionBalance/std": 0.28405311703681946,
      "step": 815,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 553.390625,
      "completions/min_length": 302.0,
      "epoch": 4.409703504043127,
      "grad_norm": 4.114929556992096,
      "kl": 4.6953125,
      "learning_rate": 6.009609745128228e-07,
      "loss": 0.36697277426719666,
      "memory(GiB)": 75.26,
      "reward": 1.7951879501342773,
      "reward_std": 0.6688352525234222,
      "rewards/Format/mean": 0.81640625,
      "rewards/Format/std": 0.382988765835762,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.43583033978939056,
      "rewards/ReflectionBalance/mean": 0.7209692597389221,
      "rewards/ReflectionBalance/std": 0.376392662525177,
      "step": 816,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 488.29296875,
      "completions/min_length": 278.0,
      "epoch": 4.415094339622642,
      "grad_norm": 2.8883997239951054,
      "kl": 3.06640625,
      "learning_rate": 6.001206076035709e-07,
      "loss": 0.24283218383789062,
      "memory(GiB)": 75.26,
      "reward": 1.833985984325409,
      "reward_std": 0.602806568145752,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.33063435554504395,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4053537845611572,
      "rewards/ReflectionBalance/mean": 0.7597672343254089,
      "rewards/ReflectionBalance/std": 0.31671927869319916,
      "step": 817,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 493.23828125,
      "completions/min_length": 250.5,
      "epoch": 4.420485175202156,
      "grad_norm": 2.7953684662804505,
      "kl": 2.8515625,
      "learning_rate": 5.992799459493323e-07,
      "loss": 0.2564397156238556,
      "memory(GiB)": 75.26,
      "reward": 1.9096028804779053,
      "reward_std": 0.5418815016746521,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42204391956329346,
      "rewards/ReflectionBalance/mean": 0.7806966304779053,
      "rewards/ReflectionBalance/std": 0.27775293588638306,
      "step": 818,
      "train_speed(iter/s)": 0.007087
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 502.15234375,
      "completions/min_length": 259.5,
      "epoch": 4.425876010781671,
      "grad_norm": 3.173901320571143,
      "kl": 3.3515625,
      "learning_rate": 5.984389920249304e-07,
      "loss": 0.2783427834510803,
      "memory(GiB)": 75.26,
      "reward": 1.7504565119743347,
      "reward_std": 0.523772805929184,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.3562464565038681,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.30204106867313385,
      "rewards/ReflectionBalance/mean": 0.7738940119743347,
      "rewards/ReflectionBalance/std": 0.2914837598800659,
      "step": 819,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 495.9375,
      "completions/min_length": 266.0,
      "epoch": 4.431266846361186,
      "grad_norm": 2.9301468955514745,
      "kl": 1.9765625,
      "learning_rate": 5.975977483060488e-07,
      "loss": 0.16525936126708984,
      "memory(GiB)": 75.26,
      "reward": 1.867882490158081,
      "reward_std": 0.44739237427711487,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3764529526233673,
      "rewards/ReflectionBalance/mean": 0.805382490158081,
      "rewards/ReflectionBalance/std": 0.22999028116464615,
      "step": 820,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 514.640625,
      "completions/min_length": 181.5,
      "epoch": 4.436657681940701,
      "grad_norm": 3.365322663390618,
      "kl": 2.7734375,
      "learning_rate": 5.967562172692243e-07,
      "loss": 0.24412773549556732,
      "memory(GiB)": 75.26,
      "reward": 1.7565526962280273,
      "reward_std": 0.6113215386867523,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3488178104162216,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3678421527147293,
      "rewards/ReflectionBalance/mean": 0.7370214760303497,
      "rewards/ReflectionBalance/std": 0.36009447276592255,
      "step": 821,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 472.65625,
      "completions/min_length": 217.0,
      "epoch": 4.442048517520216,
      "grad_norm": 3.7247710980403834,
      "kl": 1.9140625,
      "learning_rate": 5.959144013918397e-07,
      "loss": 0.18911483883857727,
      "memory(GiB)": 75.26,
      "reward": 1.9110571146011353,
      "reward_std": 0.4595441371202469,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3946705758571625,
      "rewards/ReflectionBalance/mean": 0.8016821444034576,
      "rewards/ReflectionBalance/std": 0.2339339703321457,
      "step": 822,
      "train_speed(iter/s)": 0.007088
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.0,
      "completions/mean_length": 494.73828125,
      "completions/min_length": 286.0,
      "epoch": 4.44743935309973,
      "grad_norm": 3.156214671309645,
      "kl": 3.15625,
      "learning_rate": 5.95072303152116e-07,
      "loss": 0.2808067798614502,
      "memory(GiB)": 75.26,
      "reward": 1.77934730052948,
      "reward_std": 0.5347539484500885,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32746291160583496,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3285723924636841,
      "rewards/ReflectionBalance/mean": 0.7754411101341248,
      "rewards/ReflectionBalance/std": 0.28824564814567566,
      "step": 823,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 494.9609375,
      "completions/min_length": 299.0,
      "epoch": 4.452830188679245,
      "grad_norm": 3.57237238105082,
      "kl": 3.421875,
      "learning_rate": 5.942299250291054e-07,
      "loss": 0.2656380832195282,
      "memory(GiB)": 75.26,
      "reward": 1.8963786363601685,
      "reward_std": 0.47690533101558685,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3300945907831192,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.38827909529209137,
      "rewards/ReflectionBalance/mean": 0.7909098267555237,
      "rewards/ReflectionBalance/std": 0.255343422293663,
      "step": 824,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 490.23828125,
      "completions/min_length": 251.0,
      "epoch": 4.45822102425876,
      "grad_norm": 3.0769304464325544,
      "kl": 4.0390625,
      "learning_rate": 5.933872695026847e-07,
      "loss": 0.337727427482605,
      "memory(GiB)": 75.26,
      "reward": 1.77492356300354,
      "reward_std": 0.6375865936279297,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.3475525379180908,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3842122554779053,
      "rewards/ReflectionBalance/mean": 0.7397672533988953,
      "rewards/ReflectionBalance/std": 0.3587194234132767,
      "step": 825,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 543.21875,
      "completions/min_length": 282.5,
      "epoch": 4.463611859838275,
      "grad_norm": 8.331584563876158,
      "kl": 5.921875,
      "learning_rate": 5.925443390535467e-07,
      "loss": 0.42557430267333984,
      "memory(GiB)": 75.26,
      "reward": 1.6370182037353516,
      "reward_std": 0.6757181584835052,
      "rewards/Format/mean": 0.7734375,
      "rewards/Format/std": 0.4193098843097687,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.33148839324712753,
      "rewards/ReflectionBalance/mean": 0.711236983537674,
      "rewards/ReflectionBalance/std": 0.33973895013332367,
      "step": 826,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 531.421875,
      "completions/min_length": 251.0,
      "epoch": 4.46900269541779,
      "grad_norm": 6.165609326429914,
      "kl": 6.21875,
      "learning_rate": 5.917011361631938e-07,
      "loss": 0.49438828229904175,
      "memory(GiB)": 75.26,
      "reward": 1.55027174949646,
      "reward_std": 0.715304046869278,
      "rewards/Format/mean": 0.75390625,
      "rewards/Format/std": 0.4324023276567459,
      "rewards/MultiModalAccuracyORM/mean": 0.09375,
      "rewards/MultiModalAccuracyORM/std": 0.2923160344362259,
      "rewards/ReflectionBalance/mean": 0.7026155591011047,
      "rewards/ReflectionBalance/std": 0.3470167964696884,
      "step": 827,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 467.375,
      "completions/min_length": 249.0,
      "epoch": 4.474393530997305,
      "grad_norm": 3.045060543409473,
      "kl": 2.8984375,
      "learning_rate": 5.908576633139303e-07,
      "loss": 0.25250035524368286,
      "memory(GiB)": 75.26,
      "reward": 1.8193570971488953,
      "reward_std": 0.6114144325256348,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.339918777346611,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3787541687488556,
      "rewards/ReflectionBalance/mean": 0.7802945971488953,
      "rewards/ReflectionBalance/std": 0.28073012828826904,
      "step": 828,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 476.765625,
      "completions/min_length": 276.5,
      "epoch": 4.479784366576819,
      "grad_norm": 4.157031998986716,
      "kl": 1.73828125,
      "learning_rate": 5.900139229888553e-07,
      "loss": 0.13888299465179443,
      "memory(GiB)": 75.26,
      "reward": 1.912843406200409,
      "reward_std": 0.4603542685508728,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.8034684360027313,
      "rewards/ReflectionBalance/std": 0.25225500762462616,
      "step": 829,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 486.5625,
      "completions/min_length": 266.5,
      "epoch": 4.485175202156334,
      "grad_norm": 3.2194857989059478,
      "kl": 2.05859375,
      "learning_rate": 5.891699176718557e-07,
      "loss": 0.20219770073890686,
      "memory(GiB)": 75.26,
      "reward": 1.9497118592262268,
      "reward_std": 0.4407026171684265,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2524137645959854,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39171716570854187,
      "rewards/ReflectionBalance/mean": 0.8325243890285492,
      "rewards/ReflectionBalance/std": 0.2005026936531067,
      "step": 830,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 968.0,
      "completions/mean_length": 465.828125,
      "completions/min_length": 288.5,
      "epoch": 4.490566037735849,
      "grad_norm": 2.8266699055559923,
      "kl": 1.212890625,
      "learning_rate": 5.883256498475977e-07,
      "loss": 0.10135012865066528,
      "memory(GiB)": 75.26,
      "reward": 1.942359983921051,
      "reward_std": 0.3600105792284012,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20146947354078293,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3511015325784683,
      "rewards/ReflectionBalance/mean": 0.832984983921051,
      "rewards/ReflectionBalance/std": 0.2001471370458603,
      "step": 831,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 483.78515625,
      "completions/min_length": 282.0,
      "epoch": 4.495956873315364,
      "grad_norm": 2.4253952364224136,
      "kl": 1.3046875,
      "learning_rate": 5.874811220015212e-07,
      "loss": 0.10224082320928574,
      "memory(GiB)": 75.26,
      "reward": 1.9230387210845947,
      "reward_std": 0.3887420892715454,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21875878423452377,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3497565984725952,
      "rewards/ReflectionBalance/mean": 0.8136637806892395,
      "rewards/ReflectionBalance/std": 0.23369840532541275,
      "step": 832,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 450.65234375,
      "completions/min_length": 272.0,
      "epoch": 4.501347708894879,
      "grad_norm": 3.1370439294459485,
      "kl": 0.962890625,
      "learning_rate": 5.866363366198308e-07,
      "loss": 0.0981811136007309,
      "memory(GiB)": 75.26,
      "reward": 1.9964436888694763,
      "reward_std": 0.3468167781829834,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.1969647891819477,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.40395987033843994,
      "rewards/ReflectionBalance/mean": 0.8323811888694763,
      "rewards/ReflectionBalance/std": 0.17761079967021942,
      "step": 833,
      "train_speed(iter/s)": 0.007089
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1016.0,
      "completions/mean_length": 477.5546875,
      "completions/min_length": 265.5,
      "epoch": 4.506738544474393,
      "grad_norm": 4.312259652687326,
      "kl": 1.32421875,
      "learning_rate": 5.857912961894897e-07,
      "loss": 0.13310474157333374,
      "memory(GiB)": 75.26,
      "reward": 1.999298632144928,
      "reward_std": 0.4739428460597992,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2813730239868164,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.4301380217075348,
      "rewards/ReflectionBalance/mean": 0.8313297629356384,
      "rewards/ReflectionBalance/std": 0.22337717562913895,
      "step": 834,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 486.234375,
      "completions/min_length": 240.5,
      "epoch": 4.512129380053908,
      "grad_norm": 2.9519580922561253,
      "kl": 1.3984375,
      "learning_rate": 5.849460031982118e-07,
      "loss": 0.1299504041671753,
      "memory(GiB)": 75.26,
      "reward": 1.9166048765182495,
      "reward_std": 0.4113256186246872,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2255912721157074,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3533225357532501,
      "rewards/ReflectionBalance/mean": 0.8111360669136047,
      "rewards/ReflectionBalance/std": 0.2366596758365631,
      "step": 835,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 485.8671875,
      "completions/min_length": 277.0,
      "epoch": 4.517520215633423,
      "grad_norm": 3.120501167112198,
      "kl": 1.70703125,
      "learning_rate": 5.841004601344545e-07,
      "loss": 0.1574326753616333,
      "memory(GiB)": 75.26,
      "reward": 1.9720442295074463,
      "reward_std": 0.41246461868286133,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4059968739748001,
      "rewards/ReflectionBalance/mean": 0.8118880391120911,
      "rewards/ReflectionBalance/std": 0.2085045650601387,
      "step": 836,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 507.921875,
      "completions/min_length": 273.0,
      "epoch": 4.5229110512129385,
      "grad_norm": 4.991142360037928,
      "kl": 3.859375,
      "learning_rate": 5.832546694874116e-07,
      "loss": 0.31956207752227783,
      "memory(GiB)": 75.26,
      "reward": 1.8677783608436584,
      "reward_std": 0.58795365691185,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.33660952746868134,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.4131145179271698,
      "rewards/ReflectionBalance/mean": 0.7818408310413361,
      "rewards/ReflectionBalance/std": 0.2924097031354904,
      "step": 837,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 504.171875,
      "completions/min_length": 235.5,
      "epoch": 4.528301886792453,
      "grad_norm": 6.396330634328752,
      "kl": 6.515625,
      "learning_rate": 5.824086337470057e-07,
      "loss": 0.5003724694252014,
      "memory(GiB)": 75.26,
      "reward": 1.7010954022407532,
      "reward_std": 0.663966953754425,
      "rewards/Format/mean": 0.80078125,
      "rewards/Format/std": 0.3985297381877899,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.36039385199546814,
      "rewards/ReflectionBalance/mean": 0.7479703724384308,
      "rewards/ReflectionBalance/std": 0.32058069109916687,
      "step": 838,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.0,
      "completions/mean_length": 530.15625,
      "completions/min_length": 183.5,
      "epoch": 4.533692722371968,
      "grad_norm": 5.955915711306295,
      "kl": 6.40625,
      "learning_rate": 5.815623554038804e-07,
      "loss": 0.482199490070343,
      "memory(GiB)": 75.26,
      "reward": 1.7401334047317505,
      "reward_std": 0.7515813112258911,
      "rewards/Format/mean": 0.7890625,
      "rewards/Format/std": 0.4077630192041397,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.3996908515691757,
      "rewards/ReflectionBalance/mean": 0.7362272143363953,
      "rewards/ReflectionBalance/std": 0.33883802592754364,
      "step": 839,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1039.5,
      "completions/mean_length": 499.47265625,
      "completions/min_length": 242.5,
      "epoch": 4.539083557951482,
      "grad_norm": 6.1594030983596095,
      "kl": 6.1875,
      "learning_rate": 5.807158369493942e-07,
      "loss": 0.4744071662425995,
      "memory(GiB)": 75.26,
      "reward": 1.6971549987792969,
      "reward_std": 0.684869647026062,
      "rewards/Format/mean": 0.8125,
      "rewards/Format/std": 0.3897709399461746,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.3562464565038681,
      "rewards/ReflectionBalance/mean": 0.7362174391746521,
      "rewards/ReflectionBalance/std": 0.35831402242183685,
      "step": 840,
      "train_speed(iter/s)": 0.00709
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 491.9453125,
      "completions/min_length": 278.0,
      "epoch": 4.544474393530997,
      "grad_norm": 4.078610638589151,
      "kl": 4.1484375,
      "learning_rate": 5.798690808756122e-07,
      "loss": 0.3036493957042694,
      "memory(GiB)": 75.26,
      "reward": 1.7045800685882568,
      "reward_std": 0.5925968289375305,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.35737672448158264,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.30072054266929626,
      "rewards/ReflectionBalance/mean": 0.7553613483905792,
      "rewards/ReflectionBalance/std": 0.310154527425766,
      "step": 841,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.5,
      "completions/mean_length": 477.3984375,
      "completions/min_length": 232.5,
      "epoch": 4.549865229110512,
      "grad_norm": 3.0887970404286675,
      "kl": 3.8984375,
      "learning_rate": 5.790220896752989e-07,
      "loss": 0.3218621015548706,
      "memory(GiB)": 75.26,
      "reward": 1.8689875602722168,
      "reward_std": 0.6062009334564209,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3229459524154663,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.4122922718524933,
      "rewards/ReflectionBalance/mean": 0.771331399679184,
      "rewards/ReflectionBalance/std": 0.29048556089401245,
      "step": 842,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 465.19140625,
      "completions/min_length": 158.5,
      "epoch": 4.555256064690027,
      "grad_norm": 2.7691927612767286,
      "kl": 2.078125,
      "learning_rate": 5.781748658419111e-07,
      "loss": 0.17913295328617096,
      "memory(GiB)": 75.26,
      "reward": 1.8541797399520874,
      "reward_std": 0.4667727053165436,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3597298264503479,
      "rewards/ReflectionBalance/mean": 0.7838671505451202,
      "rewards/ReflectionBalance/std": 0.2934577018022537,
      "step": 843,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 975.5,
      "completions/mean_length": 464.50390625,
      "completions/min_length": 275.0,
      "epoch": 4.560646900269542,
      "grad_norm": 3.175846282284482,
      "kl": 1.59765625,
      "learning_rate": 5.773274118695906e-07,
      "loss": 0.09941941499710083,
      "memory(GiB)": 75.26,
      "reward": 1.9288866519927979,
      "reward_std": 0.45045146346092224,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.38315413892269135,
      "rewards/ReflectionBalance/mean": 0.8077929615974426,
      "rewards/ReflectionBalance/std": 0.2604124844074249,
      "step": 844,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1018.0,
      "completions/mean_length": 499.609375,
      "completions/min_length": 274.5,
      "epoch": 4.566037735849057,
      "grad_norm": 2.6378770953501274,
      "kl": 1.58984375,
      "learning_rate": 5.764797302531563e-07,
      "loss": 0.14153708517551422,
      "memory(GiB)": 75.26,
      "reward": 2.0038037300109863,
      "reward_std": 0.42964567244052887,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4270860105752945,
      "rewards/ReflectionBalance/mean": 0.8202099502086639,
      "rewards/ReflectionBalance/std": 0.24450118839740753,
      "step": 845,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 488.171875,
      "completions/min_length": 273.0,
      "epoch": 4.571428571428571,
      "grad_norm": 3.4983614243639645,
      "kl": 2.11328125,
      "learning_rate": 5.756318234880978e-07,
      "loss": 0.21402300894260406,
      "memory(GiB)": 75.26,
      "reward": 1.8329492211341858,
      "reward_std": 0.4844176471233368,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36387722194194794,
      "rewards/ReflectionBalance/mean": 0.7821679711341858,
      "rewards/ReflectionBalance/std": 0.25409915298223495,
      "step": 846,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 469.20703125,
      "completions/min_length": 243.0,
      "epoch": 4.576819407008086,
      "grad_norm": 3.014348885527928,
      "kl": 2.26171875,
      "learning_rate": 5.747836940705671e-07,
      "loss": 0.2059607058763504,
      "memory(GiB)": 75.26,
      "reward": 1.904000699520111,
      "reward_std": 0.5211733877658844,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2829600051045418,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.39950306713581085,
      "rewards/ReflectionBalance/mean": 0.7946256399154663,
      "rewards/ReflectionBalance/std": 0.25296274572610855,
      "step": 847,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 971.5,
      "completions/mean_length": 480.94140625,
      "completions/min_length": 234.5,
      "epoch": 4.5822102425876015,
      "grad_norm": 2.8917012779337314,
      "kl": 3.16015625,
      "learning_rate": 5.739353444973718e-07,
      "loss": 0.2482750415802002,
      "memory(GiB)": 75.26,
      "reward": 1.7678157091140747,
      "reward_std": 0.5099923312664032,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3247780203819275,
      "rewards/MultiModalAccuracyORM/mean": 0.09375,
      "rewards/MultiModalAccuracyORM/std": 0.2897924929857254,
      "rewards/ReflectionBalance/mean": 0.7951595187187195,
      "rewards/ReflectionBalance/std": 0.24734507501125336,
      "step": 848,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 486.59765625,
      "completions/min_length": 284.5,
      "epoch": 4.587601078167116,
      "grad_norm": 3.0487977706812446,
      "kl": 3.7734375,
      "learning_rate": 5.730867772659677e-07,
      "loss": 0.327017605304718,
      "memory(GiB)": 75.26,
      "reward": 1.7714745998382568,
      "reward_std": 0.629648745059967,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.35737672448158264,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3752707988023758,
      "rewards/ReflectionBalance/mean": 0.7558495998382568,
      "rewards/ReflectionBalance/std": 0.3290872573852539,
      "step": 849,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 474.4296875,
      "completions/min_length": 192.5,
      "epoch": 4.592991913746631,
      "grad_norm": 3.131272183710625,
      "kl": 3.1640625,
      "learning_rate": 5.72237994874451e-07,
      "loss": 0.2587100863456726,
      "memory(GiB)": 75.26,
      "reward": 1.7993649244308472,
      "reward_std": 0.52084881067276,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.3541714698076248,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.365008607506752,
      "rewards/ReflectionBalance/mean": 0.7876461744308472,
      "rewards/ReflectionBalance/std": 0.2772560641169548,
      "step": 850,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 489.03515625,
      "completions/min_length": 295.5,
      "epoch": 4.598382749326145,
      "grad_norm": 4.065095710640235,
      "kl": 2.32421875,
      "learning_rate": 5.713889998215517e-07,
      "loss": 0.19479072093963623,
      "memory(GiB)": 75.26,
      "reward": 1.930607795715332,
      "reward_std": 0.48108622431755066,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.405468225479126,
      "rewards/ReflectionBalance/mean": 0.7938889861106873,
      "rewards/ReflectionBalance/std": 0.2644951567053795,
      "step": 851,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 458.46484375,
      "completions/min_length": 246.0,
      "epoch": 4.60377358490566,
      "grad_norm": 2.8665435527556613,
      "kl": 2.5625,
      "learning_rate": 5.705397946066262e-07,
      "loss": 0.22995394468307495,
      "memory(GiB)": 75.26,
      "reward": 1.8207438588142395,
      "reward_std": 0.4742000997066498,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.34826576709747314,
      "rewards/ReflectionBalance/mean": 0.7855875492095947,
      "rewards/ReflectionBalance/std": 0.24587953090667725,
      "step": 852,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 490.4296875,
      "completions/min_length": 156.5,
      "epoch": 4.609164420485175,
      "grad_norm": 2.771569034290791,
      "kl": 2.6484375,
      "learning_rate": 5.696903817296485e-07,
      "loss": 0.2436894178390503,
      "memory(GiB)": 75.26,
      "reward": 1.8551017045974731,
      "reward_std": 0.5607021450996399,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31333550810813904,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3636700212955475,
      "rewards/ReflectionBalance/mean": 0.7926017045974731,
      "rewards/ReflectionBalance/std": 0.282028965651989,
      "step": 853,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 464.4921875,
      "completions/min_length": 248.0,
      "epoch": 4.6145552560646905,
      "grad_norm": 4.11870059137701,
      "kl": 2.2578125,
      "learning_rate": 5.68840763691205e-07,
      "loss": 0.2166653275489807,
      "memory(GiB)": 75.26,
      "reward": 1.9212183356285095,
      "reward_std": 0.45215919613838196,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38047872483730316,
      "rewards/ReflectionBalance/mean": 0.8235620260238647,
      "rewards/ReflectionBalance/std": 0.2154494971036911,
      "step": 854,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 449.9296875,
      "completions/min_length": 281.5,
      "epoch": 4.619946091644205,
      "grad_norm": 3.0664410222560132,
      "kl": 2.00390625,
      "learning_rate": 5.679909429924853e-07,
      "loss": 0.1570252776145935,
      "memory(GiB)": 75.26,
      "reward": 1.9344434142112732,
      "reward_std": 0.413120299577713,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3761351853609085,
      "rewards/ReflectionBalance/mean": 0.8211621344089508,
      "rewards/ReflectionBalance/std": 0.22251149266958237,
      "step": 855,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 495.6640625,
      "completions/min_length": 250.0,
      "epoch": 4.62533692722372,
      "grad_norm": 3.5564225941692498,
      "kl": 3.34375,
      "learning_rate": 5.671409221352762e-07,
      "loss": 0.28932756185531616,
      "memory(GiB)": 75.26,
      "reward": 1.9617415070533752,
      "reward_std": 0.5276496708393097,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31518664956092834,
      "rewards/MultiModalAccuracyORM/mean": 0.265625,
      "rewards/MultiModalAccuracyORM/std": 0.4302283823490143,
      "rewards/ReflectionBalance/mean": 0.8093977868556976,
      "rewards/ReflectionBalance/std": 0.23434662073850632,
      "step": 856,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 481.19140625,
      "completions/min_length": 258.0,
      "epoch": 4.630727762803234,
      "grad_norm": 3.2889565288674936,
      "kl": 3.15625,
      "learning_rate": 5.662907036219535e-07,
      "loss": 0.2385699301958084,
      "memory(GiB)": 75.26,
      "reward": 1.852148413658142,
      "reward_std": 0.5423721075057983,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.32267695665359497,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.382988765835762,
      "rewards/ReflectionBalance/mean": 0.7857421338558197,
      "rewards/ReflectionBalance/std": 0.24924977123737335,
      "step": 857,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 485.8359375,
      "completions/min_length": 250.0,
      "epoch": 4.636118598382749,
      "grad_norm": 2.7919010693413164,
      "kl": 2.30859375,
      "learning_rate": 5.654402899554748e-07,
      "loss": 0.2022714614868164,
      "memory(GiB)": 75.26,
      "reward": 1.9095654487609863,
      "reward_std": 0.5230866074562073,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.39169713854789734,
      "rewards/ReflectionBalance/mean": 0.8080029487609863,
      "rewards/ReflectionBalance/std": 0.2627064660191536,
      "step": 858,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 480.359375,
      "completions/min_length": 272.5,
      "epoch": 4.6415094339622645,
      "grad_norm": 2.8125775925566505,
      "kl": 2.734375,
      "learning_rate": 5.645896836393725e-07,
      "loss": 0.2278643399477005,
      "memory(GiB)": 75.26,
      "reward": 1.9464349746704102,
      "reward_std": 0.5022943913936615,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41328008472919464,
      "rewards/ReflectionBalance/mean": 0.817528635263443,
      "rewards/ReflectionBalance/std": 0.2073710858821869,
      "step": 859,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 507.33984375,
      "completions/min_length": 252.5,
      "epoch": 4.646900269541779,
      "grad_norm": 2.919815759858281,
      "kl": 2.9296875,
      "learning_rate": 5.637388871777454e-07,
      "loss": 0.2438318133354187,
      "memory(GiB)": 75.26,
      "reward": 1.827833652496338,
      "reward_std": 0.508381724357605,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.36627575755119324,
      "rewards/ReflectionBalance/mean": 0.7731461524963379,
      "rewards/ReflectionBalance/std": 0.3022155612707138,
      "step": 860,
      "train_speed(iter/s)": 0.007091
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 487.4375,
      "completions/min_length": 278.5,
      "epoch": 4.652291105121294,
      "grad_norm": 2.8193657611144967,
      "kl": 2.7109375,
      "learning_rate": 5.628879030752533e-07,
      "loss": 0.2283935248851776,
      "memory(GiB)": 75.26,
      "reward": 1.8406786918640137,
      "reward_std": 0.46378882229328156,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3023834079504013,
      "rewards/ReflectionBalance/mean": 0.7977099418640137,
      "rewards/ReflectionBalance/std": 0.2832223027944565,
      "step": 861,
      "train_speed(iter/s)": 0.007092
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 481.42578125,
      "completions/min_length": 259.5,
      "epoch": 4.657681940700809,
      "grad_norm": 2.884304573456543,
      "kl": 2.6328125,
      "learning_rate": 5.620367338371072e-07,
      "loss": 0.22753658890724182,
      "memory(GiB)": 75.26,
      "reward": 2.02730393409729,
      "reward_std": 0.49427630007267,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.296875,
      "rewards/MultiModalAccuracyORM/std": 0.439965158700943,
      "rewards/ReflectionBalance/mean": 0.8163663446903229,
      "rewards/ReflectionBalance/std": 0.19638164341449738,
      "step": 862,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 471.6328125,
      "completions/min_length": 265.5,
      "epoch": 4.663072776280323,
      "grad_norm": 3.18020667024505,
      "kl": 2.0,
      "learning_rate": 5.611853819690639e-07,
      "loss": 0.19285070896148682,
      "memory(GiB)": 75.26,
      "reward": 1.945275068283081,
      "reward_std": 0.3982952833175659,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3996165096759796,
      "rewards/ReflectionBalance/mean": 0.8163687884807587,
      "rewards/ReflectionBalance/std": 0.20899714529514313,
      "step": 863,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 940.5,
      "completions/mean_length": 460.9765625,
      "completions/min_length": 210.5,
      "epoch": 4.668463611859838,
      "grad_norm": 3.078310442576591,
      "kl": 1.9296875,
      "learning_rate": 5.603338499774175e-07,
      "loss": 0.16869688034057617,
      "memory(GiB)": 75.26,
      "reward": 1.8972184658050537,
      "reward_std": 0.445875346660614,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2813730239868164,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.36930377781391144,
      "rewards/ReflectionBalance/mean": 0.8112809360027313,
      "rewards/ReflectionBalance/std": 0.2607366591691971,
      "step": 864,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 444.08203125,
      "completions/min_length": 187.5,
      "epoch": 4.6738544474393535,
      "grad_norm": 4.230744960569248,
      "kl": 2.8203125,
      "learning_rate": 5.594821403689924e-07,
      "loss": 0.20444530248641968,
      "memory(GiB)": 75.26,
      "reward": 1.8870524168014526,
      "reward_std": 0.49711906909942627,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.37514132261276245,
      "rewards/ReflectionBalance/mean": 0.8089273869991302,
      "rewards/ReflectionBalance/std": 0.21181246638298035,
      "step": 865,
      "train_speed(iter/s)": 0.007093
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 471.36328125,
      "completions/min_length": 277.5,
      "epoch": 4.679245283018868,
      "grad_norm": 3.0983773245718567,
      "kl": 2.8125,
      "learning_rate": 5.586302556511358e-07,
      "loss": 0.236981600522995,
      "memory(GiB)": 75.26,
      "reward": 1.8795596957206726,
      "reward_std": 0.5102112144231796,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3711828291416168,
      "rewards/ReflectionBalance/mean": 0.801434725522995,
      "rewards/ReflectionBalance/std": 0.23099344223737717,
      "step": 866,
      "train_speed(iter/s)": 0.007094
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 478.02734375,
      "completions/min_length": 285.5,
      "epoch": 4.684636118598383,
      "grad_norm": 3.052807333308475,
      "kl": 2.7578125,
      "learning_rate": 5.577781983317107e-07,
      "loss": 0.24154877662658691,
      "memory(GiB)": 75.26,
      "reward": 1.958310604095459,
      "reward_std": 0.5093065053224564,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2721100151538849,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.42014752328395844,
      "rewards/ReflectionBalance/mean": 0.8176855444908142,
      "rewards/ReflectionBalance/std": 0.21858232468366623,
      "step": 867,
      "train_speed(iter/s)": 0.007094
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 495.62890625,
      "completions/min_length": 243.5,
      "epoch": 4.690026954177897,
      "grad_norm": 4.133325296106946,
      "kl": 4.4921875,
      "learning_rate": 5.569259709190881e-07,
      "loss": 0.35316798090934753,
      "memory(GiB)": 75.26,
      "reward": 1.7497265338897705,
      "reward_std": 0.577100545167923,
      "rewards/Format/mean": 0.8359375,
      "rewards/Format/std": 0.36935214698314667,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.34447427093982697,
      "rewards/ReflectionBalance/mean": 0.7770703136920929,
      "rewards/ReflectionBalance/std": 0.2780933231115341,
      "step": 868,
      "train_speed(iter/s)": 0.007094
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 484.01171875,
      "completions/min_length": 248.0,
      "epoch": 4.695417789757412,
      "grad_norm": 6.424202075681465,
      "kl": 3.40625,
      "learning_rate": 5.560735759221397e-07,
      "loss": 0.2695506513118744,
      "memory(GiB)": 75.26,
      "reward": 1.9414128065109253,
      "reward_std": 0.4863686114549637,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3300945907831192,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.4473883658647537,
      "rewards/ReflectionBalance/mean": 0.7929752469062805,
      "rewards/ReflectionBalance/std": 0.29590317606925964,
      "step": 869,
      "train_speed(iter/s)": 0.007094
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 471.03125,
      "completions/min_length": 277.5,
      "epoch": 4.7008086253369274,
      "grad_norm": 2.9241721193818155,
      "kl": 4.328125,
      "learning_rate": 5.552210158502301e-07,
      "loss": 0.3827281892299652,
      "memory(GiB)": 75.26,
      "reward": 1.8969385027885437,
      "reward_std": 0.6474882960319519,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3265802264213562,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4323427081108093,
      "rewards/ReflectionBalance/mean": 0.7719385027885437,
      "rewards/ReflectionBalance/std": 0.27685680240392685,
      "step": 870,
      "train_speed(iter/s)": 0.007095
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 477.546875,
      "completions/min_length": 196.0,
      "epoch": 4.706199460916442,
      "grad_norm": 4.723129341984367,
      "kl": 4.0,
      "learning_rate": 5.543682932132108e-07,
      "loss": 0.276172399520874,
      "memory(GiB)": 75.26,
      "reward": 1.791220784187317,
      "reward_std": 0.5722101032733917,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3229459524154663,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3358502835035324,
      "rewards/ReflectionBalance/mean": 0.7677832245826721,
      "rewards/ReflectionBalance/std": 0.3178507536649704,
      "step": 871,
      "train_speed(iter/s)": 0.007094
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 481.7109375,
      "completions/min_length": 264.0,
      "epoch": 4.711590296495957,
      "grad_norm": 3.5853927559950267,
      "kl": 3.7578125,
      "learning_rate": 5.535154105214111e-07,
      "loss": 0.2947383224964142,
      "memory(GiB)": 75.26,
      "reward": 1.8505321741104126,
      "reward_std": 0.5685722231864929,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.33569003641605377,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4094643145799637,
      "rewards/ReflectionBalance/mean": 0.772407203912735,
      "rewards/ReflectionBalance/std": 0.28176042437553406,
      "step": 872,
      "train_speed(iter/s)": 0.007095
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1037.0,
      "completions/mean_length": 470.5234375,
      "completions/min_length": 236.0,
      "epoch": 4.716981132075472,
      "grad_norm": 3.087879923973926,
      "kl": 3.08203125,
      "learning_rate": 5.526623702856319e-07,
      "loss": 0.24861985445022583,
      "memory(GiB)": 75.26,
      "reward": 1.9577831029891968,
      "reward_std": 0.5466713607311249,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.43559013307094574,
      "rewards/ReflectionBalance/mean": 0.7937207221984863,
      "rewards/ReflectionBalance/std": 0.2801681458950043,
      "step": 873,
      "train_speed(iter/s)": 0.007095
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 459.66796875,
      "completions/min_length": 203.0,
      "epoch": 4.722371967654986,
      "grad_norm": 2.897398459986377,
      "kl": 2.546875,
      "learning_rate": 5.518091750171378e-07,
      "loss": 0.20426079630851746,
      "memory(GiB)": 75.26,
      "reward": 1.8113444447517395,
      "reward_std": 0.4690440148115158,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.32267695665359497,
      "rewards/ReflectionBalance/mean": 0.7918131649494171,
      "rewards/ReflectionBalance/std": 0.23838139325380325,
      "step": 874,
      "train_speed(iter/s)": 0.007095
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 465.94140625,
      "completions/min_length": 305.5,
      "epoch": 4.727762803234501,
      "grad_norm": 4.997465430635667,
      "kl": 1.98046875,
      "learning_rate": 5.509558272276496e-07,
      "loss": 0.1845010221004486,
      "memory(GiB)": 75.26,
      "reward": 1.9853352904319763,
      "reward_std": 0.4345288574695587,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.43684960901737213,
      "rewards/ReflectionBalance/mean": 0.8017415404319763,
      "rewards/ReflectionBalance/std": 0.20090371370315552,
      "step": 875,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 474.6484375,
      "completions/min_length": 244.0,
      "epoch": 4.7331536388140165,
      "grad_norm": 2.88048877261916,
      "kl": 2.1796875,
      "learning_rate": 5.501023294293371e-07,
      "loss": 0.20574472844600677,
      "memory(GiB)": 75.26,
      "reward": 1.9462158679962158,
      "reward_std": 0.41098515689373016,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24081215262413025,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39068229496479034,
      "rewards/ReflectionBalance/mean": 0.821215808391571,
      "rewards/ReflectionBalance/std": 0.22181612998247147,
      "step": 876,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 929.5,
      "completions/mean_length": 448.7890625,
      "completions/min_length": 229.0,
      "epoch": 4.738544474393531,
      "grad_norm": 3.4768897764333113,
      "kl": 1.546875,
      "learning_rate": 5.492486841348121e-07,
      "loss": 0.14119285345077515,
      "memory(GiB)": 75.26,
      "reward": 1.9260985851287842,
      "reward_std": 0.4292839467525482,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38325364887714386,
      "rewards/ReflectionBalance/mean": 0.8128173649311066,
      "rewards/ReflectionBalance/std": 0.21476807445287704,
      "step": 877,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1017.5,
      "completions/mean_length": 449.671875,
      "completions/min_length": 267.5,
      "epoch": 4.743935309973046,
      "grad_norm": 2.6242218258911616,
      "kl": 1.5546875,
      "learning_rate": 5.483948938571207e-07,
      "loss": 0.13459956645965576,
      "memory(GiB)": 75.26,
      "reward": 1.9720655679702759,
      "reward_std": 0.43391281366348267,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42010872066020966,
      "rewards/ReflectionBalance/mean": 0.8080029487609863,
      "rewards/ReflectionBalance/std": 0.2023930326104164,
      "step": 878,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 460.61328125,
      "completions/min_length": 249.0,
      "epoch": 4.74932614555256,
      "grad_norm": 3.0499605404841392,
      "kl": 1.98046875,
      "learning_rate": 5.47540961109735e-07,
      "loss": 0.19393965601921082,
      "memory(GiB)": 75.26,
      "reward": 1.9602652788162231,
      "reward_std": 0.3611680269241333,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.2156769335269928,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3854355216026306,
      "rewards/ReflectionBalance/mean": 0.8313590884208679,
      "rewards/ReflectionBalance/std": 0.2101699486374855,
      "step": 879,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 507.2421875,
      "completions/min_length": 221.5,
      "epoch": 4.754716981132075,
      "grad_norm": 3.0221806420541473,
      "kl": 3.0078125,
      "learning_rate": 5.466868884065471e-07,
      "loss": 0.2607947587966919,
      "memory(GiB)": 75.26,
      "reward": 1.7814356088638306,
      "reward_std": 0.5294299721717834,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3219822645187378,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.29317550361156464,
      "rewards/ReflectionBalance/mean": 0.7853417992591858,
      "rewards/ReflectionBalance/std": 0.24657197296619415,
      "step": 880,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 465.921875,
      "completions/min_length": 278.5,
      "epoch": 4.7601078167115904,
      "grad_norm": 3.1911712643262953,
      "kl": 2.6171875,
      "learning_rate": 5.458326782618612e-07,
      "loss": 0.22862690687179565,
      "memory(GiB)": 75.26,
      "reward": 1.8797802925109863,
      "reward_std": 0.4291805326938629,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2801405265927315,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.370050385594368,
      "rewards/ReflectionBalance/mean": 0.7860302925109863,
      "rewards/ReflectionBalance/std": 0.2670898735523224,
      "step": 881,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 467.86328125,
      "completions/min_length": 263.5,
      "epoch": 4.765498652291106,
      "grad_norm": 3.1641163424315835,
      "kl": 2.0,
      "learning_rate": 5.449783331903858e-07,
      "loss": 0.1652194857597351,
      "memory(GiB)": 75.26,
      "reward": 1.9977636337280273,
      "reward_std": 0.3991062194108963,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24678732454776764,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.4176374822854996,
      "rewards/ReflectionBalance/mean": 0.8415136933326721,
      "rewards/ReflectionBalance/std": 0.18034931272268295,
      "step": 882,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 483.18359375,
      "completions/min_length": 273.5,
      "epoch": 4.77088948787062,
      "grad_norm": 2.783279527421608,
      "kl": 2.109375,
      "learning_rate": 5.441238557072268e-07,
      "loss": 0.1984342634677887,
      "memory(GiB)": 75.26,
      "reward": 1.8521558046340942,
      "reward_std": 0.3731606602668762,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.08984375,
      "rewards/MultiModalAccuracyORM/std": 0.27175262570381165,
      "rewards/ReflectionBalance/mean": 0.8209057450294495,
      "rewards/ReflectionBalance/std": 0.23284751921892166,
      "step": 883,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 486.42578125,
      "completions/min_length": 146.5,
      "epoch": 4.776280323450135,
      "grad_norm": 3.8635588018356875,
      "kl": 2.21875,
      "learning_rate": 5.432692483278801e-07,
      "loss": 0.15823489427566528,
      "memory(GiB)": 75.26,
      "reward": 1.8828954696655273,
      "reward_std": 0.4790632724761963,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3680133819580078,
      "rewards/ReflectionBalance/mean": 0.8047704696655273,
      "rewards/ReflectionBalance/std": 0.2505815252661705,
      "step": 884,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 489.1796875,
      "completions/min_length": 244.5,
      "epoch": 4.781671159029649,
      "grad_norm": 5.876347752488259,
      "kl": 2.265625,
      "learning_rate": 5.424145135682234e-07,
      "loss": 0.18764522671699524,
      "memory(GiB)": 75.26,
      "reward": 1.988483190536499,
      "reward_std": 0.5736586153507233,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3219822645187378,
      "rewards/MultiModalAccuracyORM/mean": 0.30078125,
      "rewards/MultiModalAccuracyORM/std": 0.4531385749578476,
      "rewards/ReflectionBalance/mean": 0.8048893809318542,
      "rewards/ReflectionBalance/std": 0.2672622948884964,
      "step": 885,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 486.375,
      "completions/min_length": 245.5,
      "epoch": 4.787061994609164,
      "grad_norm": 2.917621573030625,
      "kl": 3.25,
      "learning_rate": 5.415596539445098e-07,
      "loss": 0.266912579536438,
      "memory(GiB)": 75.26,
      "reward": 1.882665991783142,
      "reward_std": 0.566809892654419,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3973046988248825,
      "rewards/ReflectionBalance/mean": 0.7850097417831421,
      "rewards/ReflectionBalance/std": 0.24810382723808289,
      "step": 886,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 490.48046875,
      "completions/min_length": 263.0,
      "epoch": 4.7924528301886795,
      "grad_norm": 5.154651646437166,
      "kl": 4.3671875,
      "learning_rate": 5.407046719733604e-07,
      "loss": 0.3221602439880371,
      "memory(GiB)": 75.26,
      "reward": 1.7795442342758179,
      "reward_std": 0.5868231058120728,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.35190291702747345,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3764529526233673,
      "rewards/ReflectionBalance/mean": 0.7522004842758179,
      "rewards/ReflectionBalance/std": 0.3279687315225601,
      "step": 887,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 504.60546875,
      "completions/min_length": 161.5,
      "epoch": 4.797843665768194,
      "grad_norm": 5.367388223655434,
      "kl": 5.453125,
      "learning_rate": 5.398495701717553e-07,
      "loss": 0.4299966096878052,
      "memory(GiB)": 75.26,
      "reward": 1.7149434089660645,
      "reward_std": 0.5973266363143921,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.3541714698076248,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.32612966001033783,
      "rewards/ReflectionBalance/mean": 0.7422870993614197,
      "rewards/ReflectionBalance/std": 0.330439954996109,
      "step": 888,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.5,
      "completions/mean_length": 516.078125,
      "completions/min_length": 282.0,
      "epoch": 4.803234501347709,
      "grad_norm": 4.51969902176299,
      "kl": 4.9375,
      "learning_rate": 5.389943510570286e-07,
      "loss": 0.3775109052658081,
      "memory(GiB)": 75.26,
      "reward": 1.7355590462684631,
      "reward_std": 0.6546122431755066,
      "rewards/Format/mean": 0.8359375,
      "rewards/Format/std": 0.3711828291416168,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3681531548500061,
      "rewards/ReflectionBalance/mean": 0.7394653260707855,
      "rewards/ReflectionBalance/std": 0.31959444284439087,
      "step": 889,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 484.19140625,
      "completions/min_length": 273.5,
      "epoch": 4.808625336927224,
      "grad_norm": 2.958808545638775,
      "kl": 3.7734375,
      "learning_rate": 5.381390171468593e-07,
      "loss": 0.3387000560760498,
      "memory(GiB)": 75.26,
      "reward": 1.845081388950348,
      "reward_std": 0.5521328449249268,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3270204961299896,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3733299523591995,
      "rewards/ReflectionBalance/mean": 0.7825813889503479,
      "rewards/ReflectionBalance/std": 0.27482734620571136,
      "step": 890,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 970.0,
      "completions/mean_length": 475.63671875,
      "completions/min_length": 182.0,
      "epoch": 4.814016172506738,
      "grad_norm": 2.759553668129206,
      "kl": 2.08984375,
      "learning_rate": 5.372835709592641e-07,
      "loss": 0.1587265133857727,
      "memory(GiB)": 75.26,
      "reward": 1.9550145864486694,
      "reward_std": 0.5144891887903214,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2897924929857254,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.43581391870975494,
      "rewards/ReflectionBalance/mean": 0.7948583960533142,
      "rewards/ReflectionBalance/std": 0.26581860333681107,
      "step": 891,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 512.9453125,
      "completions/min_length": 225.5,
      "epoch": 4.819407008086253,
      "grad_norm": 3.1934898216235177,
      "kl": 3.4921875,
      "learning_rate": 5.364280150125902e-07,
      "loss": 0.31112271547317505,
      "memory(GiB)": 75.26,
      "reward": 1.7124390006065369,
      "reward_std": 0.5404866337776184,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.3410066068172455,
      "rewards/MultiModalAccuracyORM/mean": 0.07421875,
      "rewards/MultiModalAccuracyORM/std": 0.2487918585538864,
      "rewards/ReflectionBalance/mean": 0.7749390006065369,
      "rewards/ReflectionBalance/std": 0.27122431993484497,
      "step": 892,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 485.4140625,
      "completions/min_length": 183.0,
      "epoch": 4.824797843665769,
      "grad_norm": 2.74086765156177,
      "kl": 2.71484375,
      "learning_rate": 5.355723518255087e-07,
      "loss": 0.22628025710582733,
      "memory(GiB)": 75.26,
      "reward": 1.8966748118400574,
      "reward_std": 0.583963081240654,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.2899702489376068,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4011208266019821,
      "rewards/ReflectionBalance/mean": 0.7716748118400574,
      "rewards/ReflectionBalance/std": 0.2722165137529373,
      "step": 893,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 462.109375,
      "completions/min_length": 241.5,
      "epoch": 4.830188679245283,
      "grad_norm": 2.6514556916201575,
      "kl": 1.4921875,
      "learning_rate": 5.347165839170054e-07,
      "loss": 0.11051445454359055,
      "memory(GiB)": 75.26,
      "reward": 1.9982844591140747,
      "reward_std": 0.4015735983848572,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.423653244972229,
      "rewards/ReflectionBalance/mean": 0.8303157389163971,
      "rewards/ReflectionBalance/std": 0.18264127522706985,
      "step": 894,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 494.875,
      "completions/min_length": 118.0,
      "epoch": 4.835579514824798,
      "grad_norm": 3.0598365069415063,
      "kl": 2.6640625,
      "learning_rate": 5.338607138063752e-07,
      "loss": 0.20448321104049683,
      "memory(GiB)": 75.26,
      "reward": 1.9155542850494385,
      "reward_std": 0.48825326561927795,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4077630192041397,
      "rewards/ReflectionBalance/mean": 0.8100854456424713,
      "rewards/ReflectionBalance/std": 0.2660404071211815,
      "step": 895,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 487.94140625,
      "completions/min_length": 180.0,
      "epoch": 4.840970350404312,
      "grad_norm": 3.0973562557079766,
      "kl": 2.33203125,
      "learning_rate": 5.330047440132133e-07,
      "loss": 0.21273356676101685,
      "memory(GiB)": 75.26,
      "reward": 1.8719555139541626,
      "reward_std": 0.4964786767959595,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26276856660842896,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36387722194194794,
      "rewards/ReflectionBalance/mean": 0.793830543756485,
      "rewards/ReflectionBalance/std": 0.276008278131485,
      "step": 896,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 509.8359375,
      "completions/min_length": 229.0,
      "epoch": 4.846361185983827,
      "grad_norm": 16.651985606998217,
      "kl": 2.6640625,
      "learning_rate": 5.321486770574085e-07,
      "loss": 0.21309539675712585,
      "memory(GiB)": 75.26,
      "reward": 1.8339551091194153,
      "reward_std": 0.5243542343378067,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.33029159903526306,
      "rewards/ReflectionBalance/mean": 0.8105175495147705,
      "rewards/ReflectionBalance/std": 0.25606028735637665,
      "step": 897,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 520.94921875,
      "completions/min_length": 277.5,
      "epoch": 4.8517520215633425,
      "grad_norm": 12.263895193053877,
      "kl": 3.1796875,
      "learning_rate": 5.312925154591359e-07,
      "loss": 0.26612067222595215,
      "memory(GiB)": 75.26,
      "reward": 1.8726741075515747,
      "reward_std": 0.49393393099308014,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3764529526233673,
      "rewards/ReflectionBalance/mean": 0.8101741373538971,
      "rewards/ReflectionBalance/std": 0.23713238537311554,
      "step": 898,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 507.1171875,
      "completions/min_length": 265.5,
      "epoch": 4.857142857142857,
      "grad_norm": 3.479717404324453,
      "kl": 2.8046875,
      "learning_rate": 5.304362617388489e-07,
      "loss": 0.1826322078704834,
      "memory(GiB)": 75.26,
      "reward": 1.918190062046051,
      "reward_std": 0.48221801221370697,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2488820105791092,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.39750634133815765,
      "rewards/ReflectionBalance/mean": 0.7931900918483734,
      "rewards/ReflectionBalance/std": 0.2795345336198807,
      "step": 899,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.0,
      "completions/mean_length": 481.1328125,
      "completions/min_length": 231.0,
      "epoch": 4.862533692722372,
      "grad_norm": 2.956470934770589,
      "kl": 1.671875,
      "learning_rate": 5.295799184172723e-07,
      "loss": 0.11108535528182983,
      "memory(GiB)": 75.26,
      "reward": 2.1000585556030273,
      "reward_std": 0.46122439205646515,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.36328125,
      "rewards/MultiModalAccuracyORM/std": 0.48240408301353455,
      "rewards/ReflectionBalance/mean": 0.8188085556030273,
      "rewards/ReflectionBalance/std": 0.22673916816711426,
      "step": 900,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 462.89453125,
      "completions/min_length": 223.5,
      "epoch": 4.867924528301887,
      "grad_norm": 2.9657342854488657,
      "kl": 1.65234375,
      "learning_rate": 5.287234880153947e-07,
      "loss": 0.1316501796245575,
      "memory(GiB)": 75.26,
      "reward": 1.8293562531471252,
      "reward_std": 0.41888174414634705,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.09375,
      "rewards/MultiModalAccuracyORM/std": 0.28448406606912613,
      "rewards/ReflectionBalance/mean": 0.8098250329494476,
      "rewards/ReflectionBalance/std": 0.21797652542591095,
      "step": 901,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 513.83984375,
      "completions/min_length": 315.5,
      "epoch": 4.873315363881401,
      "grad_norm": 8.600681323978208,
      "kl": 3.8671875,
      "learning_rate": 5.278669730544607e-07,
      "loss": 0.3363153636455536,
      "memory(GiB)": 75.26,
      "reward": 1.6907087564468384,
      "reward_std": 0.6705391705036163,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.3630705028772354,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.31812146306037903,
      "rewards/ReflectionBalance/mean": 0.7336775958538055,
      "rewards/ReflectionBalance/std": 0.36902929842472076,
      "step": 902,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1036.5,
      "completions/mean_length": 492.94921875,
      "completions/min_length": 274.5,
      "epoch": 4.878706199460916,
      "grad_norm": 2.9250139578447607,
      "kl": 2.0234375,
      "learning_rate": 5.270103760559645e-07,
      "loss": 0.17019984126091003,
      "memory(GiB)": 75.26,
      "reward": 2.0090137124061584,
      "reward_std": 0.5159815847873688,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.4530755430459976,
      "rewards/ReflectionBalance/mean": 0.8058886528015137,
      "rewards/ReflectionBalance/std": 0.2524958699941635,
      "step": 903,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1015.0,
      "completions/mean_length": 472.20703125,
      "completions/min_length": 269.0,
      "epoch": 4.884097035040432,
      "grad_norm": 2.633407910846185,
      "kl": 1.69921875,
      "learning_rate": 5.261536995416408e-07,
      "loss": 0.14314177632331848,
      "memory(GiB)": 75.26,
      "reward": 1.9143490195274353,
      "reward_std": 0.4313753545284271,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.26755452156066895,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.38336627185344696,
      "rewards/ReflectionBalance/mean": 0.8088802099227905,
      "rewards/ReflectionBalance/std": 0.22013384103775024,
      "step": 904,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 465.69921875,
      "completions/min_length": 255.5,
      "epoch": 4.889487870619946,
      "grad_norm": 2.734786198301732,
      "kl": 2.3203125,
      "learning_rate": 5.252969460334595e-07,
      "loss": 0.2095758318901062,
      "memory(GiB)": 75.26,
      "reward": 1.9359700679779053,
      "reward_std": 0.5042671263217926,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27079907804727554,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4091253876686096,
      "rewards/ReflectionBalance/mean": 0.8070638179779053,
      "rewards/ReflectionBalance/std": 0.22574155032634735,
      "step": 905,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 506.33984375,
      "completions/min_length": 222.0,
      "epoch": 4.894878706199461,
      "grad_norm": 2.835827010801843,
      "kl": 3.1484375,
      "learning_rate": 5.244401180536164e-07,
      "loss": 0.26218292117118835,
      "memory(GiB)": 75.26,
      "reward": 1.9079101085662842,
      "reward_std": 0.5739483833312988,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.322907418012619,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.4155515879392624,
      "rewards/ReflectionBalance/mean": 0.7829101383686066,
      "rewards/ReflectionBalance/std": 0.30147483944892883,
      "step": 906,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 491.2421875,
      "completions/min_length": 285.5,
      "epoch": 4.900269541778976,
      "grad_norm": 2.777731805269924,
      "kl": 1.91015625,
      "learning_rate": 5.235832181245271e-07,
      "loss": 0.15867307782173157,
      "memory(GiB)": 75.26,
      "reward": 1.977817416191101,
      "reward_std": 0.446430042386055,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.41230008006095886,
      "rewards/ReflectionBalance/mean": 0.8293798863887787,
      "rewards/ReflectionBalance/std": 0.20509956032037735,
      "step": 907,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 505.578125,
      "completions/min_length": 253.0,
      "epoch": 4.90566037735849,
      "grad_norm": 3.615947039160455,
      "kl": 2.875,
      "learning_rate": 5.227262487688183e-07,
      "loss": 0.22828185558319092,
      "memory(GiB)": 75.26,
      "reward": 1.8291361927986145,
      "reward_std": 0.5125332474708557,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.3131764084100723,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.34826864302158356,
      "rewards/ReflectionBalance/mean": 0.8017924427986145,
      "rewards/ReflectionBalance/std": 0.23123575001955032,
      "step": 908,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 496.484375,
      "completions/min_length": 274.0,
      "epoch": 4.9110512129380055,
      "grad_norm": 3.068564556469436,
      "kl": 2.7109375,
      "learning_rate": 5.218692125093216e-07,
      "loss": 0.21442458033561707,
      "memory(GiB)": 75.26,
      "reward": 1.8901236653327942,
      "reward_std": 0.49129490554332733,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3778642416000366,
      "rewards/ReflectionBalance/mean": 0.8041861653327942,
      "rewards/ReflectionBalance/std": 0.2564494162797928,
      "step": 909,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 465.68359375,
      "completions/min_length": 289.5,
      "epoch": 4.916442048517521,
      "grad_norm": 3.820338318063427,
      "kl": 2.4921875,
      "learning_rate": 5.210121118690657e-07,
      "loss": 0.19993022084236145,
      "memory(GiB)": 75.26,
      "reward": 1.8116260170936584,
      "reward_std": 0.43536585569381714,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.3191515803337097,
      "rewards/ReflectionBalance/mean": 0.7764697372913361,
      "rewards/ReflectionBalance/std": 0.279092475771904,
      "step": 910,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 497.3828125,
      "completions/min_length": 247.0,
      "epoch": 4.921832884097035,
      "grad_norm": 2.876488826798022,
      "kl": 3.046875,
      "learning_rate": 5.201549493712684e-07,
      "loss": 0.2509831190109253,
      "memory(GiB)": 75.26,
      "reward": 1.8731298446655273,
      "reward_std": 0.4573062062263489,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3774644732475281,
      "rewards/ReflectionBalance/mean": 0.8067236542701721,
      "rewards/ReflectionBalance/std": 0.22993776947259903,
      "step": 911,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 522.4609375,
      "completions/min_length": 209.5,
      "epoch": 4.92722371967655,
      "grad_norm": 3.9180388839139537,
      "kl": 2.234375,
      "learning_rate": 5.192977275393296e-07,
      "loss": 0.17245277762413025,
      "memory(GiB)": 75.26,
      "reward": 1.9505810141563416,
      "reward_std": 0.4242044687271118,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41635647416114807,
      "rewards/ReflectionBalance/mean": 0.8216748237609863,
      "rewards/ReflectionBalance/std": 0.20984261482954025,
      "step": 912,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 487.609375,
      "completions/min_length": 199.0,
      "epoch": 4.932614555256064,
      "grad_norm": 2.947332350282165,
      "kl": 2.6953125,
      "learning_rate": 5.184404488968243e-07,
      "loss": 0.22200565040111542,
      "memory(GiB)": 75.26,
      "reward": 1.8904981017112732,
      "reward_std": 0.4711449146270752,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.37402482330799103,
      "rewards/ReflectionBalance/mean": 0.7928417921066284,
      "rewards/ReflectionBalance/std": 0.2791221961379051,
      "step": 913,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 504.82421875,
      "completions/min_length": 275.0,
      "epoch": 4.938005390835579,
      "grad_norm": 2.808702188590559,
      "kl": 3.015625,
      "learning_rate": 5.175831159674942e-07,
      "loss": 0.25255048274993896,
      "memory(GiB)": 75.26,
      "reward": 1.8428955078125,
      "reward_std": 0.5261431336402893,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3219822645187378,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.37157969176769257,
      "rewards/ReflectionBalance/mean": 0.7882079780101776,
      "rewards/ReflectionBalance/std": 0.2620268166065216,
      "step": 914,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 488.96875,
      "completions/min_length": 234.0,
      "epoch": 4.943396226415095,
      "grad_norm": 2.91244615049306,
      "kl": 1.72265625,
      "learning_rate": 5.167257312752416e-07,
      "loss": 0.14320898056030273,
      "memory(GiB)": 75.26,
      "reward": 1.8806836009025574,
      "reward_std": 0.4406384229660034,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26276856660842896,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.34263932704925537,
      "rewards/ReflectionBalance/mean": 0.8142773509025574,
      "rewards/ReflectionBalance/std": 0.21430551260709763,
      "step": 915,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 481.60546875,
      "completions/min_length": 259.0,
      "epoch": 4.948787061994609,
      "grad_norm": 3.831413702395032,
      "kl": 1.953125,
      "learning_rate": 5.158682973441203e-07,
      "loss": 0.1954513043165207,
      "memory(GiB)": 75.26,
      "reward": 1.9899088740348816,
      "reward_std": 0.4151270389556885,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.38827909529209137,
      "rewards/ReflectionBalance/mean": 0.8258463442325592,
      "rewards/ReflectionBalance/std": 0.19705737382173538,
      "step": 916,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 518.21484375,
      "completions/min_length": 293.5,
      "epoch": 4.954177897574124,
      "grad_norm": 2.866219958142994,
      "kl": 1.9765625,
      "learning_rate": 5.150108166983295e-07,
      "loss": 0.17024341225624084,
      "memory(GiB)": 75.26,
      "reward": 1.8019042611122131,
      "reward_std": 0.41844911873340607,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.078125,
      "rewards/MultiModalAccuracyORM/std": 0.2690252959728241,
      "rewards/ReflectionBalance/mean": 0.7940918207168579,
      "rewards/ReflectionBalance/std": 0.2614835277199745,
      "step": 917,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 502.8515625,
      "completions/min_length": 259.5,
      "epoch": 4.959568733153639,
      "grad_norm": 2.7108352221305814,
      "kl": 1.859375,
      "learning_rate": 5.141532918622062e-07,
      "loss": 0.16055947542190552,
      "memory(GiB)": 75.26,
      "reward": 1.9069417119026184,
      "reward_std": 0.5675084590911865,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.42273683845996857,
      "rewards/ReflectionBalance/mean": 0.7858479917049408,
      "rewards/ReflectionBalance/std": 0.26143015176057816,
      "step": 918,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 495.92578125,
      "completions/min_length": 283.5,
      "epoch": 4.964959568733153,
      "grad_norm": 3.0086713674693586,
      "kl": 3.5078125,
      "learning_rate": 5.132957253602166e-07,
      "loss": 0.2882932722568512,
      "memory(GiB)": 75.26,
      "reward": 1.893114447593689,
      "reward_std": 0.55246701836586,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4120787978172302,
      "rewards/ReflectionBalance/mean": 0.7915519177913666,
      "rewards/ReflectionBalance/std": 0.2780344635248184,
      "step": 919,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 489.5703125,
      "completions/min_length": 288.5,
      "epoch": 4.9703504043126685,
      "grad_norm": 3.198373980602321,
      "kl": 2.1484375,
      "learning_rate": 5.124381197169505e-07,
      "loss": 0.18432870507240295,
      "memory(GiB)": 75.26,
      "reward": 1.9744662046432495,
      "reward_std": 0.4916386902332306,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4229802340269089,
      "rewards/ReflectionBalance/mean": 0.8104036152362823,
      "rewards/ReflectionBalance/std": 0.2667698711156845,
      "step": 920,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 499.83203125,
      "completions/min_length": 259.0,
      "epoch": 4.975741239892184,
      "grad_norm": 6.450103907124579,
      "kl": 5.4296875,
      "learning_rate": 5.115804774571127e-07,
      "loss": 0.41187000274658203,
      "memory(GiB)": 75.26,
      "reward": 1.7564030289649963,
      "reward_std": 0.572637289762497,
      "rewards/Format/mean": 0.828125,
      "rewards/Format/std": 0.37352070212364197,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3723055571317673,
      "rewards/ReflectionBalance/mean": 0.7603092789649963,
      "rewards/ReflectionBalance/std": 0.3050413429737091,
      "step": 921,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 504.5234375,
      "completions/min_length": 248.5,
      "epoch": 4.981132075471698,
      "grad_norm": 3.9089619460643745,
      "kl": 4.890625,
      "learning_rate": 5.107228011055151e-07,
      "loss": 0.3853417932987213,
      "memory(GiB)": 75.26,
      "reward": 1.8033837676048279,
      "reward_std": 0.7350716888904572,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.35675957798957825,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.395822137594223,
      "rewards/ReflectionBalance/mean": 0.7526025474071503,
      "rewards/ReflectionBalance/std": 0.3514195829629898,
      "step": 922,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 489.89453125,
      "completions/min_length": 177.0,
      "epoch": 4.986522911051213,
      "grad_norm": 3.817877875487387,
      "kl": 3.578125,
      "learning_rate": 5.09865093187071e-07,
      "loss": 0.3031080961227417,
      "memory(GiB)": 75.26,
      "reward": 1.8426074981689453,
      "reward_std": 0.5507866442203522,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3206050544977188,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.37120403349399567,
      "rewards/ReflectionBalance/mean": 0.7840136587619781,
      "rewards/ReflectionBalance/std": 0.280614972114563,
      "step": 923,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 486.98046875,
      "completions/min_length": 281.5,
      "epoch": 4.991913746630727,
      "grad_norm": 3.397912248788555,
      "kl": 3.2578125,
      "learning_rate": 5.090073562267859e-07,
      "loss": 0.2586286664009094,
      "memory(GiB)": 75.26,
      "reward": 1.8557844758033752,
      "reward_std": 0.5368054658174515,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3540267199277878,
      "rewards/ReflectionBalance/mean": 0.7854719758033752,
      "rewards/ReflectionBalance/std": 0.276516318321228,
      "step": 924,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 507.30078125,
      "completions/min_length": 231.5,
      "epoch": 4.997304582210242,
      "grad_norm": 3.096753572638547,
      "kl": 3.6796875,
      "learning_rate": 5.081495927497511e-07,
      "loss": 0.33137989044189453,
      "memory(GiB)": 75.26,
      "reward": 1.8574788570404053,
      "reward_std": 0.5506605058908463,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.2999713644385338,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3946705758571625,
      "rewards/ReflectionBalance/mean": 0.7715413570404053,
      "rewards/ReflectionBalance/std": 0.3024737611413002,
      "step": 925,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 501.43359375,
      "completions/min_length": 265.5,
      "epoch": 5.005390835579515,
      "grad_norm": 3.0056417841010434,
      "kl": 2.6328125,
      "learning_rate": 5.07291805281136e-07,
      "loss": 0.22609765827655792,
      "memory(GiB)": 75.26,
      "reward": 1.9631317257881165,
      "reward_std": 0.5352988988161087,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.4418751001358032,
      "rewards/ReflectionBalance/mean": 0.7990692257881165,
      "rewards/ReflectionBalance/std": 0.25205428153276443,
      "step": 926,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 517.51953125,
      "completions/min_length": 294.5,
      "epoch": 5.010781671159029,
      "grad_norm": 2.433562623577067,
      "kl": 1.58203125,
      "learning_rate": 5.064339963461805e-07,
      "loss": 0.1365469992160797,
      "memory(GiB)": 75.26,
      "reward": 1.9523738622665405,
      "reward_std": 0.4559640437364578,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4089515954256058,
      "rewards/ReflectionBalance/mean": 0.8156551122665405,
      "rewards/ReflectionBalance/std": 0.2506321966648102,
      "step": 927,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 486.89453125,
      "completions/min_length": 243.5,
      "epoch": 5.0161725067385445,
      "grad_norm": 2.8373994376666243,
      "kl": 1.73046875,
      "learning_rate": 5.055761684701875e-07,
      "loss": 0.14249110221862793,
      "memory(GiB)": 75.26,
      "reward": 1.96651691198349,
      "reward_std": 0.3999321460723877,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.4013945013284683,
      "rewards/ReflectionBalance/mean": 0.81026691198349,
      "rewards/ReflectionBalance/std": 0.23177257925271988,
      "step": 928,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 470.57421875,
      "completions/min_length": 246.0,
      "epoch": 5.02156334231806,
      "grad_norm": 4.220353698176484,
      "kl": 1.89453125,
      "learning_rate": 5.047183241785162e-07,
      "loss": 0.1767013669013977,
      "memory(GiB)": 75.26,
      "reward": 1.9870182871818542,
      "reward_std": 0.37786756455898285,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20146947354078293,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.396200567483902,
      "rewards/ReflectionBalance/mean": 0.8268619775772095,
      "rewards/ReflectionBalance/std": 0.20368531346321106,
      "step": 929,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 499.515625,
      "completions/min_length": 265.0,
      "epoch": 5.026954177897574,
      "grad_norm": 3.4792245798270938,
      "kl": 1.5859375,
      "learning_rate": 5.038604659965735e-07,
      "loss": 0.16070500016212463,
      "memory(GiB)": 75.26,
      "reward": 1.929397702217102,
      "reward_std": 0.4545392245054245,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3774644732475281,
      "rewards/ReflectionBalance/mean": 0.8356477916240692,
      "rewards/ReflectionBalance/std": 0.2204756960272789,
      "step": 930,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 493.203125,
      "completions/min_length": 308.0,
      "epoch": 5.032345013477089,
      "grad_norm": 3.112103648996934,
      "kl": 1.75,
      "learning_rate": 5.030025964498078e-07,
      "loss": 0.16933470964431763,
      "memory(GiB)": 75.26,
      "reward": 1.9623063206672668,
      "reward_std": 0.4773615002632141,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.42527204751968384,
      "rewards/ReflectionBalance/mean": 0.8099625706672668,
      "rewards/ReflectionBalance/std": 0.23516283184289932,
      "step": 931,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 472.125,
      "completions/min_length": 283.5,
      "epoch": 5.037735849056604,
      "grad_norm": 2.8906218895852476,
      "kl": 1.88671875,
      "learning_rate": 5.021447180637004e-07,
      "loss": 0.16483071446418762,
      "memory(GiB)": 75.26,
      "reward": 1.9542708992958069,
      "reward_std": 0.43805335462093353,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39238542318344116,
      "rewards/ReflectionBalance/mean": 0.8136458396911621,
      "rewards/ReflectionBalance/std": 0.22655467689037323,
      "step": 932,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 500.703125,
      "completions/min_length": 288.0,
      "epoch": 5.0431266846361185,
      "grad_norm": 2.828543234463282,
      "kl": 1.96875,
      "learning_rate": 5.012868333637591e-07,
      "loss": 0.16563042998313904,
      "memory(GiB)": 75.26,
      "reward": 2.013330101966858,
      "reward_std": 0.445388525724411,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22204948961734772,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4312489330768585,
      "rewards/ReflectionBalance/mean": 0.8219238221645355,
      "rewards/ReflectionBalance/std": 0.22393427044153214,
      "step": 933,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 519.24609375,
      "completions/min_length": 172.0,
      "epoch": 5.048517520215634,
      "grad_norm": 4.901836023900393,
      "kl": 3.9375,
      "learning_rate": 5.0042894487551e-07,
      "loss": 0.29775094985961914,
      "memory(GiB)": 75.26,
      "reward": 1.69266676902771,
      "reward_std": 0.5674404501914978,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.3586205691099167,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.29794247448444366,
      "rewards/ReflectionBalance/mean": 0.7512605786323547,
      "rewards/ReflectionBalance/std": 0.333877831697464,
      "step": 934,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1033.0,
      "completions/mean_length": 512.734375,
      "completions/min_length": 265.5,
      "epoch": 5.053908355795148,
      "grad_norm": 5.224544635125138,
      "kl": 4.78125,
      "learning_rate": 4.9957105512449e-07,
      "loss": 0.34499865770339966,
      "memory(GiB)": 75.26,
      "reward": 1.705992877483368,
      "reward_std": 0.5654759705066681,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.36039385199546814,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3177158683538437,
      "rewards/ReflectionBalance/mean": 0.7372428476810455,
      "rewards/ReflectionBalance/std": 0.3469189256429672,
      "step": 935,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 458.125,
      "completions/min_length": 263.5,
      "epoch": 5.059299191374663,
      "grad_norm": 3.0006421663473146,
      "kl": 3.0,
      "learning_rate": 4.987131666362411e-07,
      "loss": 0.24346964061260223,
      "memory(GiB)": 75.26,
      "reward": 1.879702091217041,
      "reward_std": 0.5669302344322205,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3285723924636841,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.423653244972229,
      "rewards/ReflectionBalance/mean": 0.7703271508216858,
      "rewards/ReflectionBalance/std": 0.29181136935949326,
      "step": 936,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 506.125,
      "completions/min_length": 273.0,
      "epoch": 5.064690026954178,
      "grad_norm": 4.380807539846468,
      "kl": 4.15625,
      "learning_rate": 4.978552819362997e-07,
      "loss": 0.33284398913383484,
      "memory(GiB)": 75.26,
      "reward": 1.8202473521232605,
      "reward_std": 0.5019660592079163,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3318064510822296,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.36238694190979004,
      "rewards/ReflectionBalance/mean": 0.7694661319255829,
      "rewards/ReflectionBalance/std": 0.28939288854599,
      "step": 937,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 514.3203125,
      "completions/min_length": 235.5,
      "epoch": 5.070080862533692,
      "grad_norm": 4.472857027370233,
      "kl": 3.9140625,
      "learning_rate": 4.969974035501922e-07,
      "loss": 0.28041431307792664,
      "memory(GiB)": 75.26,
      "reward": 1.8796297907829285,
      "reward_std": 0.6063815355300903,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.36039385199546814,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4378891587257385,
      "rewards/ReflectionBalance/mean": 0.7546297311782837,
      "rewards/ReflectionBalance/std": 0.3211827278137207,
      "step": 938,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 484.1796875,
      "completions/min_length": 265.0,
      "epoch": 5.0754716981132075,
      "grad_norm": 3.0485141382511487,
      "kl": 2.1640625,
      "learning_rate": 4.961395340034265e-07,
      "loss": 0.2187737226486206,
      "memory(GiB)": 75.26,
      "reward": 1.9598909616470337,
      "reward_std": 0.516411155462265,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4077630192041397,
      "rewards/ReflectionBalance/mean": 0.8153597116470337,
      "rewards/ReflectionBalance/std": 0.2705325335264206,
      "step": 939,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 499.4609375,
      "completions/min_length": 241.0,
      "epoch": 5.080862533692723,
      "grad_norm": 2.8268884545688038,
      "kl": 2.7578125,
      "learning_rate": 4.952816758214839e-07,
      "loss": 0.23688599467277527,
      "memory(GiB)": 75.26,
      "reward": 1.8380810022354126,
      "reward_std": 0.5112306028604507,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35538628697395325,
      "rewards/ReflectionBalance/mean": 0.7833935916423798,
      "rewards/ReflectionBalance/std": 0.25957293063402176,
      "step": 940,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 526.44140625,
      "completions/min_length": 298.0,
      "epoch": 5.086253369272237,
      "grad_norm": 2.6684099926488325,
      "kl": 2.4765625,
      "learning_rate": 4.944238315298124e-07,
      "loss": 0.21450906991958618,
      "memory(GiB)": 75.26,
      "reward": 1.9245174527168274,
      "reward_std": 0.4914025366306305,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.40343451499938965,
      "rewards/ReflectionBalance/mean": 0.803423672914505,
      "rewards/ReflectionBalance/std": 0.25835853070020676,
      "step": 941,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 501.77734375,
      "completions/min_length": 279.5,
      "epoch": 5.091644204851752,
      "grad_norm": 2.782231826080779,
      "kl": 2.08203125,
      "learning_rate": 4.935660036538196e-07,
      "loss": 0.16921880841255188,
      "memory(GiB)": 75.26,
      "reward": 1.88136225938797,
      "reward_std": 0.4954480826854706,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3607245087623596,
      "rewards/ReflectionBalance/mean": 0.8071435689926147,
      "rewards/ReflectionBalance/std": 0.24353709071874619,
      "step": 942,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 522.515625,
      "completions/min_length": 284.0,
      "epoch": 5.097035040431267,
      "grad_norm": 45.14085230303388,
      "kl": 1.9296875,
      "learning_rate": 4.92708194718864e-07,
      "loss": 0.17453360557556152,
      "memory(GiB)": 75.26,
      "reward": 1.865465521812439,
      "reward_std": 0.4238096624612808,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.31232215464115143,
      "rewards/ReflectionBalance/mean": 0.8264029920101166,
      "rewards/ReflectionBalance/std": 0.22849631309509277,
      "step": 943,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 502.0078125,
      "completions/min_length": 247.5,
      "epoch": 5.1024258760107815,
      "grad_norm": 2.962449031377768,
      "kl": 1.2734375,
      "learning_rate": 4.918504072502488e-07,
      "loss": 0.12570443749427795,
      "memory(GiB)": 75.26,
      "reward": 1.9348127841949463,
      "reward_std": 0.37158599495887756,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20146947354078293,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.34263932704925537,
      "rewards/ReflectionBalance/mean": 0.8332503139972687,
      "rewards/ReflectionBalance/std": 0.22638894617557526,
      "step": 944,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 516.640625,
      "completions/min_length": 313.0,
      "epoch": 5.107816711590297,
      "grad_norm": 2.638224841439153,
      "kl": 1.6484375,
      "learning_rate": 4.909926437732142e-07,
      "loss": 0.16739949584007263,
      "memory(GiB)": 75.26,
      "reward": 1.9717594385147095,
      "reward_std": 0.41468556225299835,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23443959653377533,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4111319035291672,
      "rewards/ReflectionBalance/mean": 0.7998844683170319,
      "rewards/ReflectionBalance/std": 0.250386118888855,
      "step": 945,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 516.88671875,
      "completions/min_length": 286.0,
      "epoch": 5.113207547169812,
      "grad_norm": 2.5584494695983073,
      "kl": 1.75390625,
      "learning_rate": 4.90134906812929e-07,
      "loss": 0.12660595774650574,
      "memory(GiB)": 75.26,
      "reward": 1.883447289466858,
      "reward_std": 0.45769593119621277,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3425614684820175,
      "rewards/ReflectionBalance/mean": 0.8248535394668579,
      "rewards/ReflectionBalance/std": 0.23982493579387665,
      "step": 946,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 501.79296875,
      "completions/min_length": 265.0,
      "epoch": 5.118598382749326,
      "grad_norm": 3.180382981507496,
      "kl": 2.1484375,
      "learning_rate": 4.892771988944848e-07,
      "loss": 0.1725103259086609,
      "memory(GiB)": 75.26,
      "reward": 1.954820990562439,
      "reward_std": 0.48066429793834686,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.4164551943540573,
      "rewards/ReflectionBalance/mean": 0.8063834309577942,
      "rewards/ReflectionBalance/std": 0.25241515040397644,
      "step": 947,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 501.796875,
      "completions/min_length": 253.5,
      "epoch": 5.123989218328841,
      "grad_norm": 2.719224523658143,
      "kl": 3.171875,
      "learning_rate": 4.884195225428875e-07,
      "loss": 0.28733018040657043,
      "memory(GiB)": 75.26,
      "reward": 1.8735999464988708,
      "reward_std": 0.5329952090978622,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3979988098144531,
      "rewards/ReflectionBalance/mean": 0.7876624464988708,
      "rewards/ReflectionBalance/std": 0.27402766793966293,
      "step": 948,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 499.37890625,
      "completions/min_length": 244.5,
      "epoch": 5.129380053908355,
      "grad_norm": 3.848432800304319,
      "kl": 3.5078125,
      "learning_rate": 4.875618802830495e-07,
      "loss": 0.2730265259742737,
      "memory(GiB)": 75.26,
      "reward": 1.9165421724319458,
      "reward_std": 0.5459921658039093,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.34826864302158356,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.4497225433588028,
      "rewards/ReflectionBalance/mean": 0.7720109224319458,
      "rewards/ReflectionBalance/std": 0.30995962023735046,
      "step": 949,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 518.47265625,
      "completions/min_length": 308.5,
      "epoch": 5.1347708894878705,
      "grad_norm": 2.8654702074144978,
      "kl": 2.5703125,
      "learning_rate": 4.867042746397833e-07,
      "loss": 0.19767898321151733,
      "memory(GiB)": 75.26,
      "reward": 1.9137874245643616,
      "reward_std": 0.4773190766572952,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.40301957726478577,
      "rewards/ReflectionBalance/mean": 0.7965999245643616,
      "rewards/ReflectionBalance/std": 0.2628019005060196,
      "step": 950,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 519.50390625,
      "completions/min_length": 226.0,
      "epoch": 5.140161725067386,
      "grad_norm": 3.217984663179285,
      "kl": 4.421875,
      "learning_rate": 4.858467081377939e-07,
      "loss": 0.3633212447166443,
      "memory(GiB)": 75.26,
      "reward": 1.8145947456359863,
      "reward_std": 0.6414875388145447,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.35062892735004425,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.40284374356269836,
      "rewards/ReflectionBalance/mean": 0.7599072158336639,
      "rewards/ReflectionBalance/std": 0.30140796303749084,
      "step": 951,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 555.3203125,
      "completions/min_length": 298.0,
      "epoch": 5.1455525606469,
      "grad_norm": 3.904194630736911,
      "kl": 4.421875,
      "learning_rate": 4.849891833016704e-07,
      "loss": 0.32330167293548584,
      "memory(GiB)": 75.26,
      "reward": 1.7359415292739868,
      "reward_std": 0.5944961905479431,
      "rewards/Format/mean": 0.828125,
      "rewards/Format/std": 0.3787541687488556,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.36145633459091187,
      "rewards/ReflectionBalance/mean": 0.7476603388786316,
      "rewards/ReflectionBalance/std": 0.29225048422813416,
      "step": 952,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 488.42578125,
      "completions/min_length": 282.0,
      "epoch": 5.150943396226415,
      "grad_norm": 4.41340907952126,
      "kl": 3.8203125,
      "learning_rate": 4.841317026558796e-07,
      "loss": 0.2737470269203186,
      "memory(GiB)": 75.26,
      "reward": 1.8019987344741821,
      "reward_std": 0.622925341129303,
      "rewards/Format/mean": 0.828125,
      "rewards/Format/std": 0.3786114752292633,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4067413955926895,
      "rewards/ReflectionBalance/mean": 0.7668424546718597,
      "rewards/ReflectionBalance/std": 0.2884276956319809,
      "step": 953,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 515.984375,
      "completions/min_length": 288.0,
      "epoch": 5.15633423180593,
      "grad_norm": 2.752005652547196,
      "kl": 1.98046875,
      "learning_rate": 4.832742687247585e-07,
      "loss": 0.14331839978694916,
      "memory(GiB)": 75.26,
      "reward": 1.9056798815727234,
      "reward_std": 0.4550665020942688,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3679620176553726,
      "rewards/ReflectionBalance/mean": 0.819742351770401,
      "rewards/ReflectionBalance/std": 0.21974540501832962,
      "step": 954,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 519.33203125,
      "completions/min_length": 225.5,
      "epoch": 5.1617250673854445,
      "grad_norm": 3.0897934884393936,
      "kl": 1.89453125,
      "learning_rate": 4.824168840325057e-07,
      "loss": 0.12964248657226562,
      "memory(GiB)": 75.26,
      "reward": 1.9135539531707764,
      "reward_std": 0.5387692451477051,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38201163709163666,
      "rewards/ReflectionBalance/mean": 0.8237101435661316,
      "rewards/ReflectionBalance/std": 0.25365862995386124,
      "step": 955,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 497.1484375,
      "completions/min_length": 293.5,
      "epoch": 5.16711590296496,
      "grad_norm": 2.7032892835811024,
      "kl": 2.5234375,
      "learning_rate": 4.815595511031758e-07,
      "loss": 0.22530411183834076,
      "memory(GiB)": 75.26,
      "reward": 1.9153043627738953,
      "reward_std": 0.4972437620162964,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4169354736804962,
      "rewards/ReflectionBalance/mean": 0.7785856127738953,
      "rewards/ReflectionBalance/std": 0.29499369859695435,
      "step": 956,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 503.44140625,
      "completions/min_length": 262.0,
      "epoch": 5.172506738544475,
      "grad_norm": 2.9147863962992018,
      "kl": 1.859375,
      "learning_rate": 4.807022724606705e-07,
      "loss": 0.1673278510570526,
      "memory(GiB)": 75.26,
      "reward": 1.9327212572097778,
      "reward_std": 0.39646777510643005,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2566775679588318,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3973046988248825,
      "rewards/ReflectionBalance/mean": 0.803815096616745,
      "rewards/ReflectionBalance/std": 0.22911345958709717,
      "step": 957,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 498.3515625,
      "completions/min_length": 273.5,
      "epoch": 5.177897574123989,
      "grad_norm": 3.5971840456468995,
      "kl": 2.21484375,
      "learning_rate": 4.798450506287317e-07,
      "loss": 0.19960224628448486,
      "memory(GiB)": 75.26,
      "reward": 1.8521793484687805,
      "reward_std": 0.4381720870733261,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3460448384284973,
      "rewards/ReflectionBalance/mean": 0.7974918782711029,
      "rewards/ReflectionBalance/std": 0.2649541348218918,
      "step": 958,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 968.5,
      "completions/mean_length": 479.609375,
      "completions/min_length": 223.5,
      "epoch": 5.183288409703504,
      "grad_norm": 2.766961742725553,
      "kl": 2.46875,
      "learning_rate": 4.789878881309342e-07,
      "loss": 0.20307603478431702,
      "memory(GiB)": 75.26,
      "reward": 1.861254870891571,
      "reward_std": 0.44945380091667175,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3589489758014679,
      "rewards/ReflectionBalance/mean": 0.810473620891571,
      "rewards/ReflectionBalance/std": 0.23144222050905228,
      "step": 959,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 512.4921875,
      "completions/min_length": 244.5,
      "epoch": 5.188679245283019,
      "grad_norm": 2.9551621897941023,
      "kl": 2.6875,
      "learning_rate": 4.781307874906785e-07,
      "loss": 0.24248546361923218,
      "memory(GiB)": 75.26,
      "reward": 1.8434887528419495,
      "reward_std": 0.47596970200538635,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3475525379180908,
      "rewards/ReflectionBalance/mean": 0.8083325326442719,
      "rewards/ReflectionBalance/std": 0.2476644068956375,
      "step": 960,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 491.19921875,
      "completions/min_length": 225.5,
      "epoch": 5.1940700808625335,
      "grad_norm": 2.671159052716046,
      "kl": 2.7578125,
      "learning_rate": 4.772737512311818e-07,
      "loss": 0.21527443826198578,
      "memory(GiB)": 75.26,
      "reward": 1.815942406654358,
      "reward_std": 0.5463589727878571,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3390307128429413,
      "rewards/ReflectionBalance/mean": 0.7885985970497131,
      "rewards/ReflectionBalance/std": 0.2513812705874443,
      "step": 961,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 506.80078125,
      "completions/min_length": 291.0,
      "epoch": 5.199460916442049,
      "grad_norm": 64.57786615247981,
      "kl": 3.1796875,
      "learning_rate": 4.764167818754731e-07,
      "loss": 0.2829570770263672,
      "memory(GiB)": 75.26,
      "reward": 1.8548323512077332,
      "reward_std": 0.5075044333934784,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.35641810297966003,
      "rewards/ReflectionBalance/mean": 0.7923323512077332,
      "rewards/ReflectionBalance/std": 0.2517765313386917,
      "step": 962,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 528.45703125,
      "completions/min_length": 246.5,
      "epoch": 5.204851752021563,
      "grad_norm": 3.9165598328487095,
      "kl": 3.8125,
      "learning_rate": 4.7555988194638356e-07,
      "loss": 0.2936355173587799,
      "memory(GiB)": 75.26,
      "reward": 1.7777799367904663,
      "reward_std": 0.6047524809837341,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.3586205691099167,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3778642416000366,
      "rewards/ReflectionBalance/mean": 0.7582487165927887,
      "rewards/ReflectionBalance/std": 0.29159463196992874,
      "step": 963,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 483.80078125,
      "completions/min_length": 265.0,
      "epoch": 5.210242587601078,
      "grad_norm": 3.2802635964674356,
      "kl": 3.40625,
      "learning_rate": 4.747030539665405e-07,
      "loss": 0.28107497096061707,
      "memory(GiB)": 75.26,
      "reward": 1.9190266728401184,
      "reward_std": 0.4764997363090515,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4120787978172302,
      "rewards/ReflectionBalance/mean": 0.8096517026424408,
      "rewards/ReflectionBalance/std": 0.24375157803297043,
      "step": 964,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 510.99609375,
      "completions/min_length": 263.5,
      "epoch": 5.215633423180593,
      "grad_norm": 2.7836224630970503,
      "kl": 3.53125,
      "learning_rate": 4.7384630045835933e-07,
      "loss": 0.2910456359386444,
      "memory(GiB)": 75.26,
      "reward": 1.8564062118530273,
      "reward_std": 0.6061035394668579,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3375232368707657,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4044223427772522,
      "rewards/ReflectionBalance/mean": 0.7821875214576721,
      "rewards/ReflectionBalance/std": 0.27522820234298706,
      "step": 965,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 490.171875,
      "completions/min_length": 243.0,
      "epoch": 5.2210242587601075,
      "grad_norm": 2.8924847151106263,
      "kl": 3.046875,
      "learning_rate": 4.7298962394403565e-07,
      "loss": 0.23871290683746338,
      "memory(GiB)": 75.26,
      "reward": 1.8929972052574158,
      "reward_std": 0.5428414046764374,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.4145888388156891,
      "rewards/ReflectionBalance/mean": 0.7797159850597382,
      "rewards/ReflectionBalance/std": 0.27103306353092194,
      "step": 966,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 492.5,
      "completions/min_length": 301.0,
      "epoch": 5.226415094339623,
      "grad_norm": 2.7801809733249736,
      "kl": 2.48828125,
      "learning_rate": 4.7213302694553917e-07,
      "loss": 0.20588065683841705,
      "memory(GiB)": 75.26,
      "reward": 1.9137126207351685,
      "reward_std": 0.42982017993927,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3809676319360733,
      "rewards/ReflectionBalance/mean": 0.8160563111305237,
      "rewards/ReflectionBalance/std": 0.22029581665992737,
      "step": 967,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1047.5,
      "completions/mean_length": 462.42578125,
      "completions/min_length": 273.5,
      "epoch": 5.231805929919138,
      "grad_norm": 2.7921540427012856,
      "kl": 1.71875,
      "learning_rate": 4.712765119846054e-07,
      "loss": 0.16232672333717346,
      "memory(GiB)": 75.26,
      "reward": 1.9744076132774353,
      "reward_std": 0.43109340965747833,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24301259219646454,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.40759414434432983,
      "rewards/ReflectionBalance/mean": 0.8220638334751129,
      "rewards/ReflectionBalance/std": 0.2354016900062561,
      "step": 968,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1020.0,
      "completions/mean_length": 464.140625,
      "completions/min_length": 252.0,
      "epoch": 5.237196765498652,
      "grad_norm": 3.1920364836426187,
      "kl": 2.05078125,
      "learning_rate": 4.704200815827277e-07,
      "loss": 0.17029055953025818,
      "memory(GiB)": 75.26,
      "reward": 1.9216032028198242,
      "reward_std": 0.4635567367076874,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.38336627185344696,
      "rewards/ReflectionBalance/mean": 0.8161344230175018,
      "rewards/ReflectionBalance/std": 0.26472581177949905,
      "step": 969,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 512.390625,
      "completions/min_length": 276.0,
      "epoch": 5.242587601078167,
      "grad_norm": 2.561288120288948,
      "kl": 2.3203125,
      "learning_rate": 4.6956373826115104e-07,
      "loss": 0.20064587891101837,
      "memory(GiB)": 75.26,
      "reward": 1.8619710206985474,
      "reward_std": 0.48040513694286346,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.35081249475479126,
      "rewards/ReflectionBalance/mean": 0.8072835206985474,
      "rewards/ReflectionBalance/std": 0.25403153896331787,
      "step": 970,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 919.5,
      "completions/mean_length": 497.02734375,
      "completions/min_length": 283.0,
      "epoch": 5.247978436657682,
      "grad_norm": 2.4891878486700105,
      "kl": 1.73046875,
      "learning_rate": 4.687074845408641e-07,
      "loss": 0.1518992781639099,
      "memory(GiB)": 75.26,
      "reward": 2.0237565636634827,
      "reward_std": 0.44956181943416595,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.4313432276248932,
      "rewards/ReflectionBalance/mean": 0.8284440338611603,
      "rewards/ReflectionBalance/std": 0.23654012382030487,
      "step": 971,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 492.23828125,
      "completions/min_length": 257.0,
      "epoch": 5.2533692722371965,
      "grad_norm": 2.8001695355236587,
      "kl": 1.98828125,
      "learning_rate": 4.678513229425914e-07,
      "loss": 0.1700238287448883,
      "memory(GiB)": 75.26,
      "reward": 1.89101243019104,
      "reward_std": 0.4627368897199631,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2829600051045418,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39238542318344116,
      "rewards/ReflectionBalance/mean": 0.7894499003887177,
      "rewards/ReflectionBalance/std": 0.2684919759631157,
      "step": 972,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 479.99609375,
      "completions/min_length": 278.5,
      "epoch": 5.258760107816712,
      "grad_norm": 2.6570555302110943,
      "kl": 1.7890625,
      "learning_rate": 4.669952559867867e-07,
      "loss": 0.13997794687747955,
      "memory(GiB)": 75.26,
      "reward": 1.9793756604194641,
      "reward_std": 0.39850306510925293,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2566775679588318,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42970049381256104,
      "rewards/ReflectionBalance/mean": 0.8075006306171417,
      "rewards/ReflectionBalance/std": 0.22673679888248444,
      "step": 973,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1016.0,
      "completions/mean_length": 473.51953125,
      "completions/min_length": 244.0,
      "epoch": 5.264150943396227,
      "grad_norm": 3.2235381795767037,
      "kl": 1.5859375,
      "learning_rate": 4.6613928619362487e-07,
      "loss": 0.14470383524894714,
      "memory(GiB)": 75.26,
      "reward": 1.9456738233566284,
      "reward_std": 0.498891681432724,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4206998348236084,
      "rewards/ReflectionBalance/mean": 0.8128613829612732,
      "rewards/ReflectionBalance/std": 0.23550230264663696,
      "step": 974,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 482.3125,
      "completions/min_length": 240.5,
      "epoch": 5.269541778975741,
      "grad_norm": 3.078059702918594,
      "kl": 1.505859375,
      "learning_rate": 4.652834160829945e-07,
      "loss": 0.14467835426330566,
      "memory(GiB)": 75.26,
      "reward": 1.913857340812683,
      "reward_std": 0.4065873622894287,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3334042727947235,
      "rewards/ReflectionBalance/mean": 0.8435449600219727,
      "rewards/ReflectionBalance/std": 0.2125740423798561,
      "step": 975,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 494.36328125,
      "completions/min_length": 266.5,
      "epoch": 5.274932614555256,
      "grad_norm": 3.2351638179061997,
      "kl": 1.3828125,
      "learning_rate": 4.6442764817449125e-07,
      "loss": 0.11820222437381744,
      "memory(GiB)": 75.26,
      "reward": 2.0466763973236084,
      "reward_std": 0.47354641556739807,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.296875,
      "rewards/MultiModalAccuracyORM/std": 0.45835477113723755,
      "rewards/ReflectionBalance/mean": 0.8162077069282532,
      "rewards/ReflectionBalance/std": 0.23820403218269348,
      "step": 976,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 485.05859375,
      "completions/min_length": 252.5,
      "epoch": 5.280323450134771,
      "grad_norm": 3.090174402818805,
      "kl": 1.7265625,
      "learning_rate": 4.6357198498740966e-07,
      "loss": 0.1479678452014923,
      "memory(GiB)": 75.26,
      "reward": 1.9066601991653442,
      "reward_std": 0.444008469581604,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.368965208530426,
      "rewards/ReflectionBalance/mean": 0.8129101693630219,
      "rewards/ReflectionBalance/std": 0.23023273050785065,
      "step": 977,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 485.35546875,
      "completions/min_length": 268.5,
      "epoch": 5.285714285714286,
      "grad_norm": 3.1432315244795506,
      "kl": 3.3046875,
      "learning_rate": 4.6271642904073595e-07,
      "loss": 0.25826871395111084,
      "memory(GiB)": 75.26,
      "reward": 1.7823389172554016,
      "reward_std": 0.5200996398925781,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3405093401670456,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.3274291008710861,
      "rewards/ReflectionBalance/mean": 0.7862451076507568,
      "rewards/ReflectionBalance/std": 0.2708112597465515,
      "step": 978,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 490.93359375,
      "completions/min_length": 272.5,
      "epoch": 5.291105121293801,
      "grad_norm": 2.8930461946915376,
      "kl": 3.140625,
      "learning_rate": 4.618609828531406e-07,
      "loss": 0.2679440379142761,
      "memory(GiB)": 75.26,
      "reward": 1.8666454553604126,
      "reward_std": 0.4772118330001831,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3437131494283676,
      "rewards/ReflectionBalance/mean": 0.8041455149650574,
      "rewards/ReflectionBalance/std": 0.2646660655736923,
      "step": 979,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 504.83984375,
      "completions/min_length": 245.5,
      "epoch": 5.296495956873315,
      "grad_norm": 4.385605882596703,
      "kl": 4.390625,
      "learning_rate": 4.6100564894297135e-07,
      "loss": 0.3254101276397705,
      "memory(GiB)": 75.26,
      "reward": 1.8297672867774963,
      "reward_std": 0.5815662741661072,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.3437131494283676,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3786114752292633,
      "rewards/ReflectionBalance/mean": 0.7946110367774963,
      "rewards/ReflectionBalance/std": 0.27852198481559753,
      "step": 980,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 478.55078125,
      "completions/min_length": 267.0,
      "epoch": 5.30188679245283,
      "grad_norm": 3.872592628782537,
      "kl": 4.3359375,
      "learning_rate": 4.6015042982824467e-07,
      "loss": 0.3394821882247925,
      "memory(GiB)": 75.26,
      "reward": 1.861695945262909,
      "reward_std": 0.5951652228832245,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.3425614684820175,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4053537845611572,
      "rewards/ReflectionBalance/mean": 0.7913834750652313,
      "rewards/ReflectionBalance/std": 0.29557858407497406,
      "step": 981,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 471.9453125,
      "completions/min_length": 244.5,
      "epoch": 5.307277628032345,
      "grad_norm": 3.206898362559652,
      "kl": 3.953125,
      "learning_rate": 4.592953280266397e-07,
      "loss": 0.3091103434562683,
      "memory(GiB)": 75.26,
      "reward": 1.84085613489151,
      "reward_std": 0.5580008924007416,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3405093401670456,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3978758305311203,
      "rewards/ReflectionBalance/mean": 0.7783561050891876,
      "rewards/ReflectionBalance/std": 0.3032599240541458,
      "step": 982,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 526.91796875,
      "completions/min_length": 263.5,
      "epoch": 5.3126684636118595,
      "grad_norm": 4.18744074664047,
      "kl": 5.0625,
      "learning_rate": 4.584403460554901e-07,
      "loss": 0.3970310091972351,
      "memory(GiB)": 75.26,
      "reward": 1.696495771408081,
      "reward_std": 0.6446752548217773,
      "rewards/Format/mean": 0.796875,
      "rewards/Format/std": 0.3980395793914795,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.31873683631420135,
      "rewards/ReflectionBalance/mean": 0.758995771408081,
      "rewards/ReflectionBalance/std": 0.29662343859672546,
      "step": 983,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 474.44140625,
      "completions/min_length": 283.5,
      "epoch": 5.318059299191375,
      "grad_norm": 3.0080604921299248,
      "kl": 2.12109375,
      "learning_rate": 4.575854864317766e-07,
      "loss": 0.18355149030685425,
      "memory(GiB)": 75.26,
      "reward": 1.88645339012146,
      "reward_std": 0.4416904002428055,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3636700212955475,
      "rewards/ReflectionBalance/mean": 0.8044221997261047,
      "rewards/ReflectionBalance/std": 0.23698677122592926,
      "step": 984,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 483.6015625,
      "completions/min_length": 253.5,
      "epoch": 5.32345013477089,
      "grad_norm": 3.5214210989235935,
      "kl": 4.671875,
      "learning_rate": 4.567307516721198e-07,
      "loss": 0.3794962763786316,
      "memory(GiB)": 75.26,
      "reward": 1.887706696987152,
      "reward_std": 0.6428169310092926,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.35675957798957825,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4346197694540024,
      "rewards/ReflectionBalance/mean": 0.7861441969871521,
      "rewards/ReflectionBalance/std": 0.270446315407753,
      "step": 985,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1051.5,
      "completions/mean_length": 493.47265625,
      "completions/min_length": 228.5,
      "epoch": 5.328840970350404,
      "grad_norm": 4.385741972970283,
      "kl": 3.03125,
      "learning_rate": 4.5587614429277314e-07,
      "loss": 0.29231148958206177,
      "memory(GiB)": 75.26,
      "reward": 1.7732958793640137,
      "reward_std": 0.6282331347465515,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3247780203819275,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.30740001052618027,
      "rewards/ReflectionBalance/mean": 0.7811083793640137,
      "rewards/ReflectionBalance/std": 0.318006232380867,
      "step": 986,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1013.0,
      "completions/mean_length": 493.10546875,
      "completions/min_length": 271.0,
      "epoch": 5.334231805929919,
      "grad_norm": 2.877464583948656,
      "kl": 2.3203125,
      "learning_rate": 4.5502166680961417e-07,
      "loss": 0.17865127325057983,
      "memory(GiB)": 75.26,
      "reward": 1.8576123118400574,
      "reward_std": 0.47951413691043854,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.36039385199546814,
      "rewards/ReflectionBalance/mean": 0.7912060618400574,
      "rewards/ReflectionBalance/std": 0.28153979778289795,
      "step": 987,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 503.33203125,
      "completions/min_length": 299.5,
      "epoch": 5.339622641509434,
      "grad_norm": 3.930424039402012,
      "kl": 2.26953125,
      "learning_rate": 4.541673217381389e-07,
      "loss": 0.1979553997516632,
      "memory(GiB)": 75.26,
      "reward": 1.8826342225074768,
      "reward_std": 0.5900042653083801,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.32267695665359497,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41742460429668427,
      "rewards/ReflectionBalance/mean": 0.7771655023097992,
      "rewards/ReflectionBalance/std": 0.2925662249326706,
      "step": 988,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 471.890625,
      "completions/min_length": 269.0,
      "epoch": 5.345013477088949,
      "grad_norm": 3.8869210042920823,
      "kl": 2.16015625,
      "learning_rate": 4.533131115934529e-07,
      "loss": 0.16823910176753998,
      "memory(GiB)": 75.26,
      "reward": 1.9046451449394226,
      "reward_std": 0.4063161462545395,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2524137645959854,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36045311391353607,
      "rewards/ReflectionBalance/mean": 0.8187077045440674,
      "rewards/ReflectionBalance/std": 0.20323744416236877,
      "step": 989,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 488.7109375,
      "completions/min_length": 253.0,
      "epoch": 5.350404312668464,
      "grad_norm": 3.0848987689423213,
      "kl": 2.234375,
      "learning_rate": 4.5245903889026505e-07,
      "loss": 0.1794462502002716,
      "memory(GiB)": 75.26,
      "reward": 1.8317675590515137,
      "reward_std": 0.5184991955757141,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.310598224401474,
      "rewards/ReflectionBalance/mean": 0.8044238090515137,
      "rewards/ReflectionBalance/std": 0.2649243175983429,
      "step": 990,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 490.625,
      "completions/min_length": 156.0,
      "epoch": 5.355795148247978,
      "grad_norm": 2.6551086854339188,
      "kl": 1.828125,
      "learning_rate": 4.5160510614287946e-07,
      "loss": 0.16107822954654694,
      "memory(GiB)": 75.26,
      "reward": 1.9961628913879395,
      "reward_std": 0.5123337805271149,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.4313432276248932,
      "rewards/ReflectionBalance/mean": 0.8125691413879395,
      "rewards/ReflectionBalance/std": 0.25278910249471664,
      "step": 991,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 519.44140625,
      "completions/min_length": 271.0,
      "epoch": 5.361185983827493,
      "grad_norm": 2.783982292466803,
      "kl": 2.6171875,
      "learning_rate": 4.507513158651878e-07,
      "loss": 0.2459128201007843,
      "memory(GiB)": 75.26,
      "reward": 1.9013875126838684,
      "reward_std": 0.5352421402931213,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.8037313222885132,
      "rewards/ReflectionBalance/std": 0.25228727608919144,
      "step": 992,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 484.2578125,
      "completions/min_length": 281.0,
      "epoch": 5.366576819407008,
      "grad_norm": 2.5628034535908806,
      "kl": 2.6875,
      "learning_rate": 4.498976705706629e-07,
      "loss": 0.23376615345478058,
      "memory(GiB)": 75.26,
      "reward": 1.9514746069908142,
      "reward_std": 0.49126332998275757,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29262590408325195,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.4098135679960251,
      "rewards/ReflectionBalance/mean": 0.8225683867931366,
      "rewards/ReflectionBalance/std": 0.21780095994472504,
      "step": 993,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 497.9296875,
      "completions/min_length": 175.5,
      "epoch": 5.3719676549865225,
      "grad_norm": 44.00057749321763,
      "kl": 2.8828125,
      "learning_rate": 4.4904417277235063e-07,
      "loss": 0.2531748116016388,
      "memory(GiB)": 75.26,
      "reward": 1.809690773487091,
      "reward_std": 0.47952674329280853,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.0859375,
      "rewards/MultiModalAccuracyORM/std": 0.2799683064222336,
      "rewards/ReflectionBalance/mean": 0.8135970234870911,
      "rewards/ReflectionBalance/std": 0.25928621739149094,
      "step": 994,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 519.22265625,
      "completions/min_length": 302.0,
      "epoch": 5.377358490566038,
      "grad_norm": 2.5962277709947035,
      "kl": 2.0859375,
      "learning_rate": 4.481908249828623e-07,
      "loss": 0.13909736275672913,
      "memory(GiB)": 75.26,
      "reward": 1.8946109414100647,
      "reward_std": 0.395372673869133,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3645188808441162,
      "rewards/ReflectionBalance/mean": 0.8125797808170319,
      "rewards/ReflectionBalance/std": 0.2250400111079216,
      "step": 995,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 463.19921875,
      "completions/min_length": 240.0,
      "epoch": 5.382749326145553,
      "grad_norm": 2.868738062913623,
      "kl": 2.40625,
      "learning_rate": 4.4733762971436806e-07,
      "loss": 0.20962689816951752,
      "memory(GiB)": 75.26,
      "reward": 2.0359463691711426,
      "reward_std": 0.5148088335990906,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.28125,
      "rewards/MultiModalAccuracyORM/std": 0.45103853940963745,
      "rewards/ReflectionBalance/mean": 0.8250089883804321,
      "rewards/ReflectionBalance/std": 0.23447829484939575,
      "step": 996,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 470.453125,
      "completions/min_length": 255.0,
      "epoch": 5.388140161725067,
      "grad_norm": 5.530134805713455,
      "kl": 1.62890625,
      "learning_rate": 4.46484589478589e-07,
      "loss": 0.15668877959251404,
      "memory(GiB)": 75.26,
      "reward": 1.9842448234558105,
      "reward_std": 0.4415425658226013,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.41096827387809753,
      "rewards/ReflectionBalance/mean": 0.8319010734558105,
      "rewards/ReflectionBalance/std": 0.17939862608909607,
      "step": 997,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 461.96484375,
      "completions/min_length": 273.0,
      "epoch": 5.393530997304582,
      "grad_norm": 2.8359140078736584,
      "kl": 2.08203125,
      "learning_rate": 4.456317067867892e-07,
      "loss": 0.16934701800346375,
      "memory(GiB)": 75.26,
      "reward": 2.060257911682129,
      "reward_std": 0.4391156733036041,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.3125,
      "rewards/MultiModalAccuracyORM/std": 0.4023359566926956,
      "rewards/ReflectionBalance/mean": 0.8258829712867737,
      "rewards/ReflectionBalance/std": 0.22354914247989655,
      "step": 998,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 483.14453125,
      "completions/min_length": 268.0,
      "epoch": 5.398921832884097,
      "grad_norm": 3.735231365936788,
      "kl": 2.71875,
      "learning_rate": 4.447789841497698e-07,
      "loss": 0.1983504593372345,
      "memory(GiB)": 75.26,
      "reward": 2.0563061237335205,
      "reward_std": 0.4738968014717102,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.34375,
      "rewards/MultiModalAccuracyORM/std": 0.47221899032592773,
      "rewards/ReflectionBalance/mean": 0.8141186535358429,
      "rewards/ReflectionBalance/std": 0.24025752395391464,
      "step": 999,
      "train_speed(iter/s)": 0.007111
    },
    {
      "epoch": 5.404312668463612,
      "grad_norm": 2.6313250287041994,
      "learning_rate": 4.439264240778605e-07,
      "loss": 0.18168425559997559,
      "memory(GiB)": 75.26,
      "step": 1000,
      "train_speed(iter/s)": 0.00711
    },
    {
      "epoch": 5.404312668463612,
      "eval_clip_ratio": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 1027.0,
      "eval_completions/mean_length": 505.822265625,
      "eval_completions/min_length": 263.75,
      "eval_kl": 2.314453125,
      "eval_loss": 0.17796842753887177,
      "eval_reward": 1.8821634650230408,
      "eval_reward_std": 0.4273792579770088,
      "eval_rewards/Format/mean": 0.921875,
      "eval_rewards/Format/std": 0.26386574655771255,
      "eval_rewards/MultiModalAccuracyORM/mean": 0.15625,
      "eval_rewards/MultiModalAccuracyORM/std": 0.35492296516895294,
      "eval_rewards/ReflectionBalance/mean": 0.8040384948253632,
      "eval_rewards/ReflectionBalance/std": 0.24849547818303108,
      "eval_runtime": 247.7887,
      "eval_samples_per_second": 0.238,
      "eval_steps_per_second": 0.004,
      "step": 1000
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 959.75,
      "completions/mean_length": 465.17578125,
      "completions/min_length": 254.25,
      "epoch": 5.409703504043127,
      "grad_norm": 3.073286899359538,
      "kl": 1.7939453125,
      "learning_rate": 4.4307402908091183e-07,
      "loss": 0.12224527448415756,
      "memory(GiB)": 75.26,
      "reward": 1.9330696165561676,
      "reward_std": 0.4176693633198738,
      "rewards/Format/mean": 0.939453125,
      "rewards/Format/std": 0.23328739032149315,
      "rewards/MultiModalAccuracyORM/mean": 0.177734375,
      "rewards/MultiModalAccuracyORM/std": 0.38121476024389267,
      "rewards/ReflectionBalance/mean": 0.8158821612596512,
      "rewards/ReflectionBalance/std": 0.2222069390118122,
      "step": 1001,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 487.3125,
      "completions/min_length": 275.5,
      "epoch": 5.415094339622642,
      "grad_norm": 2.6447229060711206,
      "kl": 1.796875,
      "learning_rate": 4.4222180166828916e-07,
      "loss": 0.14248600602149963,
      "memory(GiB)": 75.26,
      "reward": 1.9549494981765747,
      "reward_std": 0.35187068581581116,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3749772012233734,
      "rewards/ReflectionBalance/mean": 0.8377620577812195,
      "rewards/ReflectionBalance/std": 0.205422542989254,
      "step": 1002,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 472.68359375,
      "completions/min_length": 269.0,
      "epoch": 5.420485175202156,
      "grad_norm": 2.7630323446635914,
      "kl": 1.8828125,
      "learning_rate": 4.4136974434886414e-07,
      "loss": 0.16685184836387634,
      "memory(GiB)": 75.26,
      "reward": 1.9346240758895874,
      "reward_std": 0.47219882905483246,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39110109210014343,
      "rewards/ReflectionBalance/mean": 0.821342796087265,
      "rewards/ReflectionBalance/std": 0.23144350200891495,
      "step": 1003,
      "train_speed(iter/s)": 0.007095
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 513.4609375,
      "completions/min_length": 269.5,
      "epoch": 5.425876010781671,
      "grad_norm": 2.7081485139643497,
      "kl": 2.96875,
      "learning_rate": 4.405178596310076e-07,
      "loss": 0.2523692548274994,
      "memory(GiB)": 75.26,
      "reward": 1.9300211668014526,
      "reward_std": 0.5103950053453445,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4202718883752823,
      "rewards/ReflectionBalance/mean": 0.7933024168014526,
      "rewards/ReflectionBalance/std": 0.2641519010066986,
      "step": 1004,
      "train_speed(iter/s)": 0.007095
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 490.53125,
      "completions/min_length": 269.0,
      "epoch": 5.431266846361186,
      "grad_norm": 4.100051462329319,
      "kl": 3.2734375,
      "learning_rate": 4.3966615002258246e-07,
      "loss": 0.2627768814563751,
      "memory(GiB)": 75.26,
      "reward": 1.8883366584777832,
      "reward_std": 0.5428081154823303,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3940901756286621,
      "rewards/ReflectionBalance/mean": 0.794586569070816,
      "rewards/ReflectionBalance/std": 0.27117758989334106,
      "step": 1005,
      "train_speed(iter/s)": 0.007095
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 452.59375,
      "completions/min_length": 235.0,
      "epoch": 5.436657681940701,
      "grad_norm": 3.0852998450204527,
      "kl": 2.390625,
      "learning_rate": 4.3881461803093615e-07,
      "loss": 0.23164069652557373,
      "memory(GiB)": 75.26,
      "reward": 1.9293432235717773,
      "reward_std": 0.4421408772468567,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.4122922718524933,
      "rewards/ReflectionBalance/mean": 0.7965307533740997,
      "rewards/ReflectionBalance/std": 0.23606126755475998,
      "step": 1006,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 478.4921875,
      "completions/min_length": 175.0,
      "epoch": 5.442048517520216,
      "grad_norm": 3.482373061141679,
      "kl": 2.71875,
      "learning_rate": 4.3796326616289283e-07,
      "loss": 0.20886391401290894,
      "memory(GiB)": 75.26,
      "reward": 1.82905113697052,
      "reward_std": 0.5095824301242828,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.30204106867313385,
      "rewards/ReflectionBalance/mean": 0.80561363697052,
      "rewards/ReflectionBalance/std": 0.26038606464862823,
      "step": 1007,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 869.5,
      "completions/mean_length": 471.0625,
      "completions/min_length": 214.5,
      "epoch": 5.44743935309973,
      "grad_norm": 2.6851087439577803,
      "kl": 1.92578125,
      "learning_rate": 4.371120969247467e-07,
      "loss": 0.14434146881103516,
      "memory(GiB)": 75.26,
      "reward": 2.0150275230407715,
      "reward_std": 0.4395807534456253,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.4420119971036911,
      "rewards/ReflectionBalance/mean": 0.8158089518547058,
      "rewards/ReflectionBalance/std": 0.21946798264980316,
      "step": 1008,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 480.15625,
      "completions/min_length": 258.0,
      "epoch": 5.452830188679245,
      "grad_norm": 2.950743243022282,
      "kl": 2.5078125,
      "learning_rate": 4.362611128222545e-07,
      "loss": 0.21892637014389038,
      "memory(GiB)": 75.26,
      "reward": 1.9792170524597168,
      "reward_std": 0.491214781999588,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.43905749917030334,
      "rewards/ReflectionBalance/mean": 0.8112483620643616,
      "rewards/ReflectionBalance/std": 0.28042128682136536,
      "step": 1009,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 508.41015625,
      "completions/min_length": 260.5,
      "epoch": 5.45822102425876,
      "grad_norm": 2.915535471621165,
      "kl": 3.2890625,
      "learning_rate": 4.3541031636062765e-07,
      "loss": 0.2834300994873047,
      "memory(GiB)": 75.26,
      "reward": 1.8816129565238953,
      "reward_std": 0.5440923869609833,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3037657365202904,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3940746486186981,
      "rewards/ReflectionBalance/mean": 0.7917692065238953,
      "rewards/ReflectionBalance/std": 0.3048642724752426,
      "step": 1010,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 471.06640625,
      "completions/min_length": 253.0,
      "epoch": 5.463611859838275,
      "grad_norm": 3.653577464293899,
      "kl": 2.671875,
      "learning_rate": 4.3455971004452514e-07,
      "loss": 0.20274123549461365,
      "memory(GiB)": 75.26,
      "reward": 1.8420817255973816,
      "reward_std": 0.41846001148223877,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3270204961299896,
      "rewards/ReflectionBalance/mean": 0.8069254457950592,
      "rewards/ReflectionBalance/std": 0.25598932057619095,
      "step": 1011,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 485.0703125,
      "completions/min_length": 221.0,
      "epoch": 5.46900269541779,
      "grad_norm": 4.420765736086836,
      "kl": 3.09375,
      "learning_rate": 4.337092963780466e-07,
      "loss": 0.22313722968101501,
      "memory(GiB)": 75.26,
      "reward": 1.9137532711029053,
      "reward_std": 0.5429005920886993,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41742460429668427,
      "rewards/ReflectionBalance/mean": 0.7692219913005829,
      "rewards/ReflectionBalance/std": 0.32413023710250854,
      "step": 1012,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 487.390625,
      "completions/min_length": 283.0,
      "epoch": 5.474393530997305,
      "grad_norm": 2.868232239781523,
      "kl": 2.8359375,
      "learning_rate": 4.328590778647238e-07,
      "loss": 0.24617686867713928,
      "memory(GiB)": 75.26,
      "reward": 1.8167716264724731,
      "reward_std": 0.46751822531223297,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32746291160583496,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.32267695665359497,
      "rewards/ReflectionBalance/mean": 0.8206779062747955,
      "rewards/ReflectionBalance/std": 0.23221654444932938,
      "step": 1013,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 490.08984375,
      "completions/min_length": 245.5,
      "epoch": 5.479784366576819,
      "grad_norm": 2.686928773827305,
      "kl": 2.35546875,
      "learning_rate": 4.3200905700751475e-07,
      "loss": 0.19833803176879883,
      "memory(GiB)": 75.26,
      "reward": 1.9397296905517578,
      "reward_std": 0.531268909573555,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.4300905168056488,
      "rewards/ReflectionBalance/mean": 0.8069173097610474,
      "rewards/ReflectionBalance/std": 0.25268255919218063,
      "step": 1014,
      "train_speed(iter/s)": 0.007096
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 488.46875,
      "completions/min_length": 282.5,
      "epoch": 5.485175202156334,
      "grad_norm": 3.1910759081586906,
      "kl": 2.8359375,
      "learning_rate": 4.3115923630879516e-07,
      "loss": 0.23374611139297485,
      "memory(GiB)": 75.26,
      "reward": 1.8713117837905884,
      "reward_std": 0.5755088925361633,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32746291160583496,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.4013945013284683,
      "rewards/ReflectionBalance/mean": 0.7736555933952332,
      "rewards/ReflectionBalance/std": 0.30653662979602814,
      "step": 1015,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 469.4453125,
      "completions/min_length": 253.0,
      "epoch": 5.490566037735849,
      "grad_norm": 2.916580983962604,
      "kl": 1.87109375,
      "learning_rate": 4.3030961827035146e-07,
      "loss": 0.14305320382118225,
      "memory(GiB)": 75.26,
      "reward": 1.918530285358429,
      "reward_std": 0.4340951591730118,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38201163709163666,
      "rewards/ReflectionBalance/mean": 0.813061535358429,
      "rewards/ReflectionBalance/std": 0.2417687252163887,
      "step": 1016,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 465.50390625,
      "completions/min_length": 264.5,
      "epoch": 5.495956873315364,
      "grad_norm": 3.0921205879262987,
      "kl": 1.6875,
      "learning_rate": 4.29460205393374e-07,
      "loss": 0.12438015639781952,
      "memory(GiB)": 75.26,
      "reward": 1.9492627382278442,
      "reward_std": 0.39012014865875244,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3636700212955475,
      "rewards/ReflectionBalance/mean": 0.8320751786231995,
      "rewards/ReflectionBalance/std": 0.1918376088142395,
      "step": 1017,
      "train_speed(iter/s)": 0.007097
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 479.21875,
      "completions/min_length": 228.0,
      "epoch": 5.501347708894879,
      "grad_norm": 2.7757822463841393,
      "kl": 1.94140625,
      "learning_rate": 4.2861100017844824e-07,
      "loss": 0.17755410075187683,
      "memory(GiB)": 75.26,
      "reward": 1.8997184038162231,
      "reward_std": 0.4177911728620529,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21875878423452377,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35358233749866486,
      "rewards/ReflectionBalance/mean": 0.7981559336185455,
      "rewards/ReflectionBalance/std": 0.2751380205154419,
      "step": 1018,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 492.5390625,
      "completions/min_length": 288.5,
      "epoch": 5.506738544474393,
      "grad_norm": 3.3898031968395053,
      "kl": 1.93359375,
      "learning_rate": 4.2776200512554906e-07,
      "loss": 0.17083631455898285,
      "memory(GiB)": 75.26,
      "reward": 1.886829435825348,
      "reward_std": 0.3637940585613251,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.2156769335269928,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.3077666610479355,
      "rewards/ReflectionBalance/mean": 0.8321419060230255,
      "rewards/ReflectionBalance/std": 0.19118332117795944,
      "step": 1019,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 463.20703125,
      "completions/min_length": 266.0,
      "epoch": 5.512129380053908,
      "grad_norm": 3.0238865261587224,
      "kl": 2.12109375,
      "learning_rate": 4.269132227340325e-07,
      "loss": 0.17304888367652893,
      "memory(GiB)": 75.26,
      "reward": 1.8842138051986694,
      "reward_std": 0.463442862033844,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3786114752292633,
      "rewards/ReflectionBalance/mean": 0.8099951148033142,
      "rewards/ReflectionBalance/std": 0.23586026579141617,
      "step": 1020,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 492.6796875,
      "completions/min_length": 287.0,
      "epoch": 5.517520215633423,
      "grad_norm": 3.3479062748091457,
      "kl": 3.15625,
      "learning_rate": 4.260646555026283e-07,
      "loss": 0.24409841001033783,
      "memory(GiB)": 75.26,
      "reward": 1.8204785585403442,
      "reward_std": 0.5429192781448364,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36624228954315186,
      "rewards/ReflectionBalance/mean": 0.7696972489356995,
      "rewards/ReflectionBalance/std": 0.30520886182785034,
      "step": 1021,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 518.81640625,
      "completions/min_length": 266.5,
      "epoch": 5.5229110512129385,
      "grad_norm": 3.037172350426495,
      "kl": 3.1015625,
      "learning_rate": 4.252163059294329e-07,
      "loss": 0.2547650933265686,
      "memory(GiB)": 75.26,
      "reward": 1.8252084255218506,
      "reward_std": 0.5813723802566528,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33513282239437103,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3778642416000366,
      "rewards/ReflectionBalance/mean": 0.778333306312561,
      "rewards/ReflectionBalance/std": 0.314938485622406,
      "step": 1022,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 505.3515625,
      "completions/min_length": 205.5,
      "epoch": 5.528301886792453,
      "grad_norm": 3.656502466835161,
      "kl": 3.796875,
      "learning_rate": 4.2436817651190233e-07,
      "loss": 0.27591508626937866,
      "memory(GiB)": 75.26,
      "reward": 1.871534824371338,
      "reward_std": 0.5166817903518677,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3940746486186981,
      "rewards/ReflectionBalance/mean": 0.7816911041736603,
      "rewards/ReflectionBalance/std": 0.271314412355423,
      "step": 1023,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1033.0,
      "completions/mean_length": 519.06640625,
      "completions/min_length": 266.5,
      "epoch": 5.533692722371968,
      "grad_norm": 4.346504525572958,
      "kl": 5.328125,
      "learning_rate": 4.235202697468437e-07,
      "loss": 0.41078734397888184,
      "memory(GiB)": 75.26,
      "reward": 1.7745572924613953,
      "reward_std": 0.643744707107544,
      "rewards/Format/mean": 0.83203125,
      "rewards/Format/std": 0.3723055571317673,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.39205317199230194,
      "rewards/ReflectionBalance/mean": 0.7433072924613953,
      "rewards/ReflectionBalance/std": 0.3098786622285843,
      "step": 1024,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 497.421875,
      "completions/min_length": 244.0,
      "epoch": 5.539083557951482,
      "grad_norm": 4.579377314647455,
      "kl": 4.46875,
      "learning_rate": 4.226725881304094e-07,
      "loss": 0.31835809350013733,
      "memory(GiB)": 75.26,
      "reward": 1.7714192867279053,
      "reward_std": 0.6965040564537048,
      "rewards/Format/mean": 0.8359375,
      "rewards/Format/std": 0.36417925357818604,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.395822137594223,
      "rewards/ReflectionBalance/mean": 0.7362629771232605,
      "rewards/ReflectionBalance/std": 0.3473730683326721,
      "step": 1025,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 478.28515625,
      "completions/min_length": 242.5,
      "epoch": 5.544474393530997,
      "grad_norm": 3.5038883943369505,
      "kl": 4.15625,
      "learning_rate": 4.2182513415808895e-07,
      "loss": 0.31748902797698975,
      "memory(GiB)": 75.26,
      "reward": 1.801820456981659,
      "reward_std": 0.580028161406517,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.336664155125618,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38325364887714386,
      "rewards/ReflectionBalance/mean": 0.7627579569816589,
      "rewards/ReflectionBalance/std": 0.28415723145008087,
      "step": 1026,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 480.0625,
      "completions/min_length": 261.0,
      "epoch": 5.549865229110512,
      "grad_norm": 3.1961298340928077,
      "kl": 3.0546875,
      "learning_rate": 4.209779103247012e-07,
      "loss": 0.257101833820343,
      "memory(GiB)": 75.26,
      "reward": 1.8996044397354126,
      "reward_std": 0.5138194113969803,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4067413955926895,
      "rewards/ReflectionBalance/mean": 0.7863232493400574,
      "rewards/ReflectionBalance/std": 0.2760773301124573,
      "step": 1027,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 515.86328125,
      "completions/min_length": 310.0,
      "epoch": 5.555256064690027,
      "grad_norm": 2.9448014523626713,
      "kl": 2.3984375,
      "learning_rate": 4.201309191243878e-07,
      "loss": 0.2141065001487732,
      "memory(GiB)": 75.26,
      "reward": 1.850073218345642,
      "reward_std": 0.5143498927354813,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.298691526055336,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35872696340084076,
      "rewards/ReflectionBalance/mean": 0.7992919981479645,
      "rewards/ReflectionBalance/std": 0.2726438343524933,
      "step": 1028,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 904.5,
      "completions/mean_length": 468.10546875,
      "completions/min_length": 195.5,
      "epoch": 5.560646900269542,
      "grad_norm": 2.6540633723753255,
      "kl": 1.298828125,
      "learning_rate": 4.192841630506058e-07,
      "loss": 0.06666888296604156,
      "memory(GiB)": 75.26,
      "reward": 1.9338786602020264,
      "reward_std": 0.3979673385620117,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.21388297155499458,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3774644732475281,
      "rewards/ReflectionBalance/mean": 0.8205972909927368,
      "rewards/ReflectionBalance/std": 0.21054252982139587,
      "step": 1029,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 476.80859375,
      "completions/min_length": 251.0,
      "epoch": 5.566037735849057,
      "grad_norm": 2.9270492622932878,
      "kl": 1.56640625,
      "learning_rate": 4.1843764459611956e-07,
      "loss": 0.15450584888458252,
      "memory(GiB)": 75.26,
      "reward": 1.9467464089393616,
      "reward_std": 0.43826138973236084,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.40343451499938965,
      "rewards/ReflectionBalance/mean": 0.8100276589393616,
      "rewards/ReflectionBalance/std": 0.24868986755609512,
      "step": 1030,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 488.91015625,
      "completions/min_length": 292.5,
      "epoch": 5.571428571428571,
      "grad_norm": 2.975008232450915,
      "kl": 1.228515625,
      "learning_rate": 4.175913662529943e-07,
      "loss": 0.11760483682155609,
      "memory(GiB)": 75.26,
      "reward": 2.0466967821121216,
      "reward_std": 0.38885362446308136,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20146947354078293,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.4219265580177307,
      "rewards/ReflectionBalance/mean": 0.8357592523097992,
      "rewards/ReflectionBalance/std": 0.20758689939975739,
      "step": 1031,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 490.3984375,
      "completions/min_length": 272.5,
      "epoch": 5.576819407008086,
      "grad_norm": 2.737979800286932,
      "kl": 1.91015625,
      "learning_rate": 4.1674533051258836e-07,
      "loss": 0.1736629605293274,
      "memory(GiB)": 75.26,
      "reward": 1.9787695407867432,
      "reward_std": 0.5000540763139725,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4234771430492401,
      "rewards/ReflectionBalance/mean": 0.8147070109844208,
      "rewards/ReflectionBalance/std": 0.24708415567874908,
      "step": 1032,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 519.171875,
      "completions/min_length": 290.5,
      "epoch": 5.5822102425876015,
      "grad_norm": 2.7446039456928717,
      "kl": 1.91015625,
      "learning_rate": 4.158995398655454e-07,
      "loss": 0.1648435890674591,
      "memory(GiB)": 75.26,
      "reward": 1.8288159370422363,
      "reward_std": 0.4338446110486984,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.26755452156066895,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.28456588089466095,
      "rewards/ReflectionBalance/mean": 0.8131909370422363,
      "rewards/ReflectionBalance/std": 0.24186423420906067,
      "step": 1033,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 504.19140625,
      "completions/min_length": 276.5,
      "epoch": 5.587601078167116,
      "grad_norm": 2.842436627457985,
      "kl": 2.578125,
      "learning_rate": 4.150539968017882e-07,
      "loss": 0.2328290343284607,
      "memory(GiB)": 75.26,
      "reward": 1.8279427289962769,
      "reward_std": 0.5269874334335327,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3265802264213562,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.35296520590782166,
      "rewards/ReflectionBalance/mean": 0.8084114789962769,
      "rewards/ReflectionBalance/std": 0.24439112842082977,
      "step": 1034,
      "train_speed(iter/s)": 0.007098
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 491.1171875,
      "completions/min_length": 282.0,
      "epoch": 5.592991913746631,
      "grad_norm": 2.540764470195395,
      "kl": 1.478515625,
      "learning_rate": 4.142087038105104e-07,
      "loss": 0.11156467348337173,
      "memory(GiB)": 75.26,
      "reward": 2.035997748374939,
      "reward_std": 0.5000487864017487,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.44320113956928253,
      "rewards/ReflectionBalance/mean": 0.8328727185726166,
      "rewards/ReflectionBalance/std": 0.24430354684591293,
      "step": 1035,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 498.28125,
      "completions/min_length": 179.5,
      "epoch": 5.598382749326145,
      "grad_norm": 2.8596884061400867,
      "kl": 2.1796875,
      "learning_rate": 4.133636633801692e-07,
      "loss": 0.18284976482391357,
      "memory(GiB)": 75.26,
      "reward": 1.8992871046066284,
      "reward_std": 0.47367119789123535,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3804178833961487,
      "rewards/ReflectionBalance/mean": 0.8016308844089508,
      "rewards/ReflectionBalance/std": 0.23722662776708603,
      "step": 1036,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 490.76953125,
      "completions/min_length": 259.5,
      "epoch": 5.60377358490566,
      "grad_norm": 2.686684231675867,
      "kl": 3.21875,
      "learning_rate": 4.125188779984787e-07,
      "loss": 0.27381980419158936,
      "memory(GiB)": 75.26,
      "reward": 1.8124365210533142,
      "reward_std": 0.5001897364854813,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3069126605987549,
      "rewards/ReflectionBalance/mean": 0.7733740508556366,
      "rewards/ReflectionBalance/std": 0.278030201792717,
      "step": 1037,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 477.6875,
      "completions/min_length": 275.5,
      "epoch": 5.609164420485175,
      "grad_norm": 3.7043442589778866,
      "kl": 2.8828125,
      "learning_rate": 4.116743501524023e-07,
      "loss": 0.20273709297180176,
      "memory(GiB)": 75.26,
      "reward": 1.9040234088897705,
      "reward_std": 0.48989836871623993,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.3979068696498871,
      "rewards/ReflectionBalance/mean": 0.7946484386920929,
      "rewards/ReflectionBalance/std": 0.23888026177883148,
      "step": 1038,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 495.24609375,
      "completions/min_length": 251.5,
      "epoch": 5.6145552560646905,
      "grad_norm": 2.9704849070495283,
      "kl": 3.21875,
      "learning_rate": 4.1083008232814434e-07,
      "loss": 0.27209770679473877,
      "memory(GiB)": 75.26,
      "reward": 1.895261287689209,
      "reward_std": 0.6093541085720062,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3270204961299896,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4225567579269409,
      "rewards/ReflectionBalance/mean": 0.7858862578868866,
      "rewards/ReflectionBalance/std": 0.2957374006509781,
      "step": 1039,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 478.80078125,
      "completions/min_length": 243.0,
      "epoch": 5.619946091644205,
      "grad_norm": 2.899753837823731,
      "kl": 2.3671875,
      "learning_rate": 4.0998607701114467e-07,
      "loss": 0.17343375086784363,
      "memory(GiB)": 75.26,
      "reward": 1.88688063621521,
      "reward_std": 0.4874877333641052,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31333550810813904,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3877098113298416,
      "rewards/ReflectionBalance/mean": 0.8048494458198547,
      "rewards/ReflectionBalance/std": 0.24460772424936295,
      "step": 1040,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 486.78125,
      "completions/min_length": 64.5,
      "epoch": 5.62533692722372,
      "grad_norm": 3.230055133528472,
      "kl": 3.765625,
      "learning_rate": 4.0914233668606977e-07,
      "loss": 0.2842435836791992,
      "memory(GiB)": 75.26,
      "reward": 1.8111824989318848,
      "reward_std": 0.5867588222026825,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.3607245087623596,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3804051876068115,
      "rewards/ReflectionBalance/mean": 0.7721199691295624,
      "rewards/ReflectionBalance/std": 0.3081153482198715,
      "step": 1041,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 517.66796875,
      "completions/min_length": 249.5,
      "epoch": 5.630727762803234,
      "grad_norm": 3.025235568961865,
      "kl": 3.390625,
      "learning_rate": 4.082988638368062e-07,
      "loss": 0.27804625034332275,
      "memory(GiB)": 75.26,
      "reward": 1.8248518705368042,
      "reward_std": 0.6091638803482056,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.35190291702747345,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3979988098144531,
      "rewards/ReflectionBalance/mean": 0.7740706205368042,
      "rewards/ReflectionBalance/std": 0.2812444642186165,
      "step": 1042,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 974.0,
      "completions/mean_length": 472.2265625,
      "completions/min_length": 227.0,
      "epoch": 5.636118598382749,
      "grad_norm": 3.100667895019686,
      "kl": 2.328125,
      "learning_rate": 4.0745566094645344e-07,
      "loss": 0.18138104677200317,
      "memory(GiB)": 75.26,
      "reward": 2.0086995363235474,
      "reward_std": 0.5347047746181488,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.2890625,
      "rewards/MultiModalAccuracyORM/std": 0.4550260454416275,
      "rewards/ReflectionBalance/mean": 0.8172932863235474,
      "rewards/ReflectionBalance/std": 0.21986351162195206,
      "step": 1043,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 482.96484375,
      "completions/min_length": 272.5,
      "epoch": 5.6415094339622645,
      "grad_norm": 2.7843267308965904,
      "kl": 1.78125,
      "learning_rate": 4.0661273049731535e-07,
      "loss": 0.14455784857273102,
      "memory(GiB)": 75.26,
      "reward": 1.9156543016433716,
      "reward_std": 0.4898591637611389,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39416566491127014,
      "rewards/ReflectionBalance/mean": 0.7984668016433716,
      "rewards/ReflectionBalance/std": 0.2561187520623207,
      "step": 1044,
      "train_speed(iter/s)": 0.007099
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 514.96875,
      "completions/min_length": 278.0,
      "epoch": 5.646900269541779,
      "grad_norm": 3.672111507939559,
      "kl": 4.0234375,
      "learning_rate": 4.0577007497089454e-07,
      "loss": 0.29834890365600586,
      "memory(GiB)": 75.26,
      "reward": 1.6595287919044495,
      "reward_std": 0.6404979825019836,
      "rewards/Format/mean": 0.82421875,
      "rewards/Format/std": 0.38181672990322113,
      "rewards/MultiModalAccuracyORM/mean": 0.0859375,
      "rewards/MultiModalAccuracyORM/std": 0.2810242921113968,
      "rewards/ReflectionBalance/mean": 0.7493725419044495,
      "rewards/ReflectionBalance/std": 0.33904363214969635,
      "step": 1045,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1019.5,
      "completions/mean_length": 492.046875,
      "completions/min_length": 242.0,
      "epoch": 5.652291105121294,
      "grad_norm": 3.9012237891321067,
      "kl": 3.109375,
      "learning_rate": 4.049276968478842e-07,
      "loss": 0.2237628698348999,
      "memory(GiB)": 75.26,
      "reward": 1.7440884709358215,
      "reward_std": 0.569090723991394,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3405093401670456,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3166656941175461,
      "rewards/ReflectionBalance/mean": 0.7636197805404663,
      "rewards/ReflectionBalance/std": 0.3059130162000656,
      "step": 1046,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1018.0,
      "completions/mean_length": 503.69921875,
      "completions/min_length": 219.5,
      "epoch": 5.657681940700809,
      "grad_norm": 3.1570174746209583,
      "kl": 2.265625,
      "learning_rate": 4.040855986081603e-07,
      "loss": 0.1812194287776947,
      "memory(GiB)": 75.26,
      "reward": 1.7919580340385437,
      "reward_std": 0.5395626425743103,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3131764084100723,
      "rewards/ReflectionBalance/mean": 0.7841455042362213,
      "rewards/ReflectionBalance/std": 0.2827666103839874,
      "step": 1047,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 468.64453125,
      "completions/min_length": 180.0,
      "epoch": 5.663072776280323,
      "grad_norm": 5.423609506894082,
      "kl": 2.4375,
      "learning_rate": 4.0324378273077557e-07,
      "loss": 0.19426479935646057,
      "memory(GiB)": 75.26,
      "reward": 1.977051556110382,
      "reward_std": 0.5721392333507538,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.45274218916893005,
      "rewards/ReflectionBalance/mean": 0.8012703359127045,
      "rewards/ReflectionBalance/std": 0.2581402212381363,
      "step": 1048,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 496.55078125,
      "completions/min_length": 264.0,
      "epoch": 5.668463611859838,
      "grad_norm": 3.1566717357035348,
      "kl": 2.125,
      "learning_rate": 4.0240225169395124e-07,
      "loss": 0.1961287558078766,
      "memory(GiB)": 75.26,
      "reward": 1.9154980778694153,
      "reward_std": 0.4648711383342743,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3853000998497009,
      "rewards/ReflectionBalance/mean": 0.8256542980670929,
      "rewards/ReflectionBalance/std": 0.21446075290441513,
      "step": 1049,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 485.65625,
      "completions/min_length": 280.5,
      "epoch": 5.6738544474393535,
      "grad_norm": 2.773786319303486,
      "kl": 2.8828125,
      "learning_rate": 4.0156100797506966e-07,
      "loss": 0.2592003643512726,
      "memory(GiB)": 75.26,
      "reward": 1.9329102039337158,
      "reward_std": 0.5656406879425049,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.43047651648521423,
      "rewards/ReflectionBalance/mean": 0.788378894329071,
      "rewards/ReflectionBalance/std": 0.2924581468105316,
      "step": 1050,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 506.4375,
      "completions/min_length": 235.0,
      "epoch": 5.679245283018868,
      "grad_norm": 2.9336163487859634,
      "kl": 3.375,
      "learning_rate": 4.007200540506677e-07,
      "loss": 0.3048962354660034,
      "memory(GiB)": 75.26,
      "reward": 1.8439486026763916,
      "reward_std": 0.552175760269165,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31518664956092834,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.36145633459091187,
      "rewards/ReflectionBalance/mean": 0.7970736026763916,
      "rewards/ReflectionBalance/std": 0.2522195354104042,
      "step": 1051,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 477.4140625,
      "completions/min_length": 167.0,
      "epoch": 5.684636118598383,
      "grad_norm": 3.065125940781264,
      "kl": 1.67578125,
      "learning_rate": 3.9987939239642923e-07,
      "loss": 0.11996852606534958,
      "memory(GiB)": 75.26,
      "reward": 1.9218376278877258,
      "reward_std": 0.47901616990566254,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4077630192041397,
      "rewards/ReflectionBalance/mean": 0.804650068283081,
      "rewards/ReflectionBalance/std": 0.24314207583665848,
      "step": 1052,
      "train_speed(iter/s)": 0.0071
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 509.875,
      "completions/min_length": 162.5,
      "epoch": 5.690026954177897,
      "grad_norm": 3.4715677773331777,
      "kl": 4.0546875,
      "learning_rate": 3.990390254871772e-07,
      "loss": 0.308590292930603,
      "memory(GiB)": 75.26,
      "reward": 1.8124104738235474,
      "reward_std": 0.6142439246177673,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.3493855148553848,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3897709399461746,
      "rewards/ReflectionBalance/mean": 0.7694417536258698,
      "rewards/ReflectionBalance/std": 0.30354034900665283,
      "step": 1053,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 935.5,
      "completions/mean_length": 481.1171875,
      "completions/min_length": 267.0,
      "epoch": 5.695417789757412,
      "grad_norm": 2.8427704271348544,
      "kl": 1.69140625,
      "learning_rate": 3.9819895579686734e-07,
      "loss": 0.11785056442022324,
      "memory(GiB)": 75.26,
      "reward": 1.8946647644042969,
      "reward_std": 0.44169798493385315,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35358233749866486,
      "rewards/ReflectionBalance/mean": 0.8165397047996521,
      "rewards/ReflectionBalance/std": 0.23981303721666336,
      "step": 1054,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1046.0,
      "completions/mean_length": 484.2734375,
      "completions/min_length": 261.5,
      "epoch": 5.7008086253369274,
      "grad_norm": 3.0713726276006064,
      "kl": 2.9921875,
      "learning_rate": 3.973591857985805e-07,
      "loss": 0.2160348743200302,
      "memory(GiB)": 75.26,
      "reward": 1.8037890195846558,
      "reward_std": 0.5965085029602051,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33595384657382965,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.36627575755119324,
      "rewards/ReflectionBalance/mean": 0.7725390791893005,
      "rewards/ReflectionBalance/std": 0.29692523181438446,
      "step": 1055,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1033.5,
      "completions/mean_length": 469.62109375,
      "completions/min_length": 238.0,
      "epoch": 5.706199460916442,
      "grad_norm": 3.6315893724510446,
      "kl": 2.59375,
      "learning_rate": 3.9651971796451444e-07,
      "loss": 0.1798919141292572,
      "memory(GiB)": 75.26,
      "reward": 1.9864501953125,
      "reward_std": 0.43548986315727234,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.4350656270980835,
      "rewards/ReflectionBalance/mean": 0.8262939155101776,
      "rewards/ReflectionBalance/std": 0.20789328962564468,
      "step": 1056,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 465.94921875,
      "completions/min_length": 74.0,
      "epoch": 5.711590296495957,
      "grad_norm": 3.9022631742924374,
      "kl": 2.8359375,
      "learning_rate": 3.9568055476597804e-07,
      "loss": 0.1854388266801834,
      "memory(GiB)": 75.26,
      "reward": 1.8687809109687805,
      "reward_std": 0.5403407961130142,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3405093401670456,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4060489982366562,
      "rewards/ReflectionBalance/mean": 0.7945621609687805,
      "rewards/ReflectionBalance/std": 0.2722105383872986,
      "step": 1057,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 469.16796875,
      "completions/min_length": 231.0,
      "epoch": 5.716981132075472,
      "grad_norm": 3.3193410523415885,
      "kl": 2.96875,
      "learning_rate": 3.948416986733838e-07,
      "loss": 0.21424338221549988,
      "memory(GiB)": 75.26,
      "reward": 1.8448339700698853,
      "reward_std": 0.5010454058647156,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.34734736382961273,
      "rewards/ReflectionBalance/mean": 0.80186527967453,
      "rewards/ReflectionBalance/std": 0.2697610706090927,
      "step": 1058,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 475.75390625,
      "completions/min_length": 252.0,
      "epoch": 5.722371967654986,
      "grad_norm": 3.994855706800224,
      "kl": 2.6015625,
      "learning_rate": 3.9400315215623894e-07,
      "loss": 0.18889038264751434,
      "memory(GiB)": 75.26,
      "reward": 1.8525211811065674,
      "reward_std": 0.48071666061878204,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3142324984073639,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3680133819580078,
      "rewards/ReflectionBalance/mean": 0.7900211811065674,
      "rewards/ReflectionBalance/std": 0.2684021666646004,
      "step": 1059,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 910.5,
      "completions/mean_length": 481.2265625,
      "completions/min_length": 270.0,
      "epoch": 5.727762803234501,
      "grad_norm": 2.6579785341617685,
      "kl": 1.302734375,
      "learning_rate": 3.931649176831401e-07,
      "loss": 0.10449575632810593,
      "memory(GiB)": 75.26,
      "reward": 2.0402848720550537,
      "reward_std": 0.43225453794002533,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.20428021997213364,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.42950694262981415,
      "rewards/ReflectionBalance/mean": 0.8410660922527313,
      "rewards/ReflectionBalance/std": 0.18001403659582138,
      "step": 1060,
      "train_speed(iter/s)": 0.007101
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 457.0234375,
      "completions/min_length": 244.5,
      "epoch": 5.7331536388140165,
      "grad_norm": 3.3480760798936005,
      "kl": 1.20703125,
      "learning_rate": 3.923269977217655e-07,
      "loss": 0.12271900475025177,
      "memory(GiB)": 75.26,
      "reward": 2.0482877492904663,
      "reward_std": 0.3561948537826538,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.194504976272583,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42921192944049835,
      "rewards/ReflectionBalance/mean": 0.8451627492904663,
      "rewards/ReflectionBalance/std": 0.16675728559494019,
      "step": 1061,
      "train_speed(iter/s)": 0.007102
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 852.5,
      "completions/mean_length": 460.06640625,
      "completions/min_length": 268.0,
      "epoch": 5.738544474393531,
      "grad_norm": 3.793652901940105,
      "kl": 1.193359375,
      "learning_rate": 3.914893947388668e-07,
      "loss": 0.10213276743888855,
      "memory(GiB)": 75.26,
      "reward": 1.9087972044944763,
      "reward_std": 0.38342294096946716,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23443959653377533,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3405093401670456,
      "rewards/ReflectionBalance/mean": 0.8345784246921539,
      "rewards/ReflectionBalance/std": 0.2104514092206955,
      "step": 1062,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 461.3671875,
      "completions/min_length": 187.5,
      "epoch": 5.743935309973046,
      "grad_norm": 2.678840787674698,
      "kl": 1.767578125,
      "learning_rate": 3.906521112002629e-07,
      "loss": 0.16818100214004517,
      "memory(GiB)": 75.26,
      "reward": 2.0157113075256348,
      "reward_std": 0.4354882836341858,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2487918585538864,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.4401901513338089,
      "rewards/ReflectionBalance/mean": 0.82821124792099,
      "rewards/ReflectionBalance/std": 0.22496244311332703,
      "step": 1063,
      "train_speed(iter/s)": 0.007103
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 483.59765625,
      "completions/min_length": 289.0,
      "epoch": 5.74932614555256,
      "grad_norm": 3.095866743554836,
      "kl": 2.015625,
      "learning_rate": 3.898151495708324e-07,
      "loss": 0.21685060858726501,
      "memory(GiB)": 75.26,
      "reward": 1.9916707277297974,
      "reward_std": 0.4143139123916626,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41857391595840454,
      "rewards/ReflectionBalance/mean": 0.8237019777297974,
      "rewards/ReflectionBalance/std": 0.19832675158977509,
      "step": 1064,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 477.796875,
      "completions/min_length": 246.0,
      "epoch": 5.754716981132075,
      "grad_norm": 2.842979827851857,
      "kl": 2.96875,
      "learning_rate": 3.8897851231450575e-07,
      "loss": 0.24890980124473572,
      "memory(GiB)": 75.26,
      "reward": 1.9525960683822632,
      "reward_std": 0.6263960897922516,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.4473883658647537,
      "rewards/ReflectionBalance/mean": 0.7924397885799408,
      "rewards/ReflectionBalance/std": 0.2927868217229843,
      "step": 1065,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 520.5546875,
      "completions/min_length": 198.0,
      "epoch": 5.7601078167115904,
      "grad_norm": 2.8200766595722473,
      "kl": 2.8125,
      "learning_rate": 3.881422018942591e-07,
      "loss": 0.224403977394104,
      "memory(GiB)": 75.26,
      "reward": 1.808218538761139,
      "reward_std": 0.4743618965148926,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29262590408325195,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3131764084100723,
      "rewards/ReflectionBalance/mean": 0.7886873185634613,
      "rewards/ReflectionBalance/std": 0.2554614469408989,
      "step": 1066,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 465.0625,
      "completions/min_length": 274.0,
      "epoch": 5.765498652291106,
      "grad_norm": 2.837654065983763,
      "kl": 2.171875,
      "learning_rate": 3.8730622077210573e-07,
      "loss": 0.1722516119480133,
      "memory(GiB)": 75.26,
      "reward": 1.935878872871399,
      "reward_std": 0.5245782881975174,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2897924929857254,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41328008472919464,
      "rewards/ReflectionBalance/mean": 0.8108789026737213,
      "rewards/ReflectionBalance/std": 0.25953952968120575,
      "step": 1067,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 526.3046875,
      "completions/min_length": 262.5,
      "epoch": 5.77088948787062,
      "grad_norm": 3.3038755555642023,
      "kl": 3.7265625,
      "learning_rate": 3.864705714090901e-07,
      "loss": 0.2854163646697998,
      "memory(GiB)": 75.26,
      "reward": 1.6986523866653442,
      "reward_std": 0.5089765191078186,
      "rewards/Format/mean": 0.84375,
      "rewards/Format/std": 0.3643587827682495,
      "rewards/MultiModalAccuracyORM/mean": 0.0859375,
      "rewards/MultiModalAccuracyORM/std": 0.27558503299951553,
      "rewards/ReflectionBalance/mean": 0.7689648568630219,
      "rewards/ReflectionBalance/std": 0.2848721593618393,
      "step": 1068,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 470.19140625,
      "completions/min_length": 250.0,
      "epoch": 5.776280323450135,
      "grad_norm": 3.4416588249507547,
      "kl": 1.953125,
      "learning_rate": 3.856352562652801e-07,
      "loss": 0.1487099528312683,
      "memory(GiB)": 75.26,
      "reward": 1.9761038422584534,
      "reward_std": 0.3836119621992111,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22826264798641205,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3949224203824997,
      "rewards/ReflectionBalance/mean": 0.839385062456131,
      "rewards/ReflectionBalance/std": 0.1828899309039116,
      "step": 1069,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 462.03125,
      "completions/min_length": 90.5,
      "epoch": 5.781671159029649,
      "grad_norm": 4.142562931421134,
      "kl": 2.05078125,
      "learning_rate": 3.848002777997588e-07,
      "loss": 0.13240408897399902,
      "memory(GiB)": 75.26,
      "reward": 1.8647655844688416,
      "reward_std": 0.42002275586128235,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.3541714698076248,
      "rewards/ReflectionBalance/mean": 0.7866406440734863,
      "rewards/ReflectionBalance/std": 0.26931317150592804,
      "step": 1070,
      "train_speed(iter/s)": 0.007104
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 469.25390625,
      "completions/min_length": 286.0,
      "epoch": 5.787061994609164,
      "grad_norm": 3.09421400588996,
      "kl": 1.7578125,
      "learning_rate": 3.8396563847061905e-07,
      "loss": 0.14799392223358154,
      "memory(GiB)": 75.26,
      "reward": 1.9869694709777832,
      "reward_std": 0.4035341888666153,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2255912721157074,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.396200567483902,
      "rewards/ReflectionBalance/mean": 0.8385319113731384,
      "rewards/ReflectionBalance/std": 0.19876490533351898,
      "step": 1071,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 502.0390625,
      "completions/min_length": 281.5,
      "epoch": 5.7924528301886795,
      "grad_norm": 3.343642706317202,
      "kl": 2.5859375,
      "learning_rate": 3.8313134073495504e-07,
      "loss": 0.21786892414093018,
      "memory(GiB)": 75.26,
      "reward": 1.936069369316101,
      "reward_std": 0.4663103073835373,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3804178833961487,
      "rewards/ReflectionBalance/mean": 0.8462255895137787,
      "rewards/ReflectionBalance/std": 0.20043328404426575,
      "step": 1072,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 462.50390625,
      "completions/min_length": 241.5,
      "epoch": 5.797843665768194,
      "grad_norm": 3.6584754447274834,
      "kl": 1.921875,
      "learning_rate": 3.82297387048855e-07,
      "loss": 0.17988011240959167,
      "memory(GiB)": 75.26,
      "reward": 1.976852297782898,
      "reward_std": 0.47112978994846344,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4256902486085892,
      "rewards/ReflectionBalance/mean": 0.8127897083759308,
      "rewards/ReflectionBalance/std": 0.22402913868427277,
      "step": 1073,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 480.3828125,
      "completions/min_length": 184.5,
      "epoch": 5.803234501347709,
      "grad_norm": 3.0835404160246638,
      "kl": 1.93359375,
      "learning_rate": 3.8146377986739485e-07,
      "loss": 0.13013380765914917,
      "memory(GiB)": 75.26,
      "reward": 1.9307421445846558,
      "reward_std": 0.3740084618330002,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.371787428855896,
      "rewards/ReflectionBalance/mean": 0.8330859541893005,
      "rewards/ReflectionBalance/std": 0.21202853322029114,
      "step": 1074,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 480.54296875,
      "completions/min_length": 233.5,
      "epoch": 5.808625336927224,
      "grad_norm": 2.9219865992545455,
      "kl": 2.046875,
      "learning_rate": 3.8063052164463037e-07,
      "loss": 0.1807490885257721,
      "memory(GiB)": 75.26,
      "reward": 1.932859718799591,
      "reward_std": 0.3953963816165924,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3842122554779053,
      "rewards/ReflectionBalance/mean": 0.8273909687995911,
      "rewards/ReflectionBalance/std": 0.24485400319099426,
      "step": 1075,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 496.7578125,
      "completions/min_length": 262.0,
      "epoch": 5.814016172506738,
      "grad_norm": 3.1312622702415887,
      "kl": 2.5859375,
      "learning_rate": 3.7979761483358934e-07,
      "loss": 0.22515949606895447,
      "memory(GiB)": 75.26,
      "reward": 1.8552181124687195,
      "reward_std": 0.49782051146030426,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35259076952934265,
      "rewards/ReflectionBalance/mean": 0.7927180826663971,
      "rewards/ReflectionBalance/std": 0.28090304136276245,
      "step": 1076,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 484.32421875,
      "completions/min_length": 264.0,
      "epoch": 5.819407008086253,
      "grad_norm": 3.2292300506422533,
      "kl": 2.7890625,
      "learning_rate": 3.7896506188626586e-07,
      "loss": 0.2195643186569214,
      "memory(GiB)": 75.26,
      "reward": 1.9970474243164062,
      "reward_std": 0.5649628043174744,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.45157207548618317,
      "rewards/ReflectionBalance/mean": 0.8056412637233734,
      "rewards/ReflectionBalance/std": 0.28322818875312805,
      "step": 1077,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 482.96484375,
      "completions/min_length": 250.0,
      "epoch": 5.824797843665769,
      "grad_norm": 4.312487243066166,
      "kl": 2.90625,
      "learning_rate": 3.7813286525361207e-07,
      "loss": 0.20477786660194397,
      "memory(GiB)": 75.26,
      "reward": 1.8707877397537231,
      "reward_std": 0.4529257118701935,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.37818245589733124,
      "rewards/ReflectionBalance/mean": 0.8043815493583679,
      "rewards/ReflectionBalance/std": 0.26021328568458557,
      "step": 1078,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 467.27734375,
      "completions/min_length": 165.5,
      "epoch": 5.830188679245283,
      "grad_norm": 4.129060530917057,
      "kl": 3.8046875,
      "learning_rate": 3.773010273855307e-07,
      "loss": 0.2807464003562927,
      "memory(GiB)": 75.26,
      "reward": 1.8161947131156921,
      "reward_std": 0.5287494659423828,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32612966001033783,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3774644732475281,
      "rewards/ReflectionBalance/mean": 0.7654134035110474,
      "rewards/ReflectionBalance/std": 0.3081733137369156,
      "step": 1079,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 506.734375,
      "completions/min_length": 274.0,
      "epoch": 5.835579514824798,
      "grad_norm": 3.054769060134687,
      "kl": 3.3203125,
      "learning_rate": 3.7646955073086893e-07,
      "loss": 0.25525039434432983,
      "memory(GiB)": 75.26,
      "reward": 1.820457398891449,
      "reward_std": 0.5676318109035492,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33513282239437103,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.37514132261276245,
      "rewards/ReflectionBalance/mean": 0.7774885892868042,
      "rewards/ReflectionBalance/std": 0.2708536311984062,
      "step": 1080,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 994.0,
      "completions/mean_length": 450.62109375,
      "completions/min_length": 235.5,
      "epoch": 5.840970350404312,
      "grad_norm": 2.8604999981335717,
      "kl": 2.56640625,
      "learning_rate": 3.7563843773741046e-07,
      "loss": 0.217007577419281,
      "memory(GiB)": 75.26,
      "reward": 1.9554699659347534,
      "reward_std": 0.46298226714134216,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.28863146156072617,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.42359209060668945,
      "rewards/ReflectionBalance/mean": 0.8070325553417206,
      "rewards/ReflectionBalance/std": 0.2128855437040329,
      "step": 1081,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 487.44921875,
      "completions/min_length": 267.5,
      "epoch": 5.846361185983827,
      "grad_norm": 2.9225935954774354,
      "kl": 3.109375,
      "learning_rate": 3.748076908518678e-07,
      "loss": 0.2626323699951172,
      "memory(GiB)": 75.26,
      "reward": 1.8897815942764282,
      "reward_std": 0.5081207752227783,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38864077627658844,
      "rewards/ReflectionBalance/mean": 0.8155627846717834,
      "rewards/ReflectionBalance/std": 0.2132440060377121,
      "step": 1082,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 480.52734375,
      "completions/min_length": 244.5,
      "epoch": 5.8517520215633425,
      "grad_norm": 3.0449903853866123,
      "kl": 2.1171875,
      "learning_rate": 3.739773125198764e-07,
      "loss": 0.17483994364738464,
      "memory(GiB)": 75.26,
      "reward": 1.9156543612480164,
      "reward_std": 0.4646923094987869,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.40395987033843994,
      "rewards/ReflectionBalance/mean": 0.8062793016433716,
      "rewards/ReflectionBalance/std": 0.26139385998249054,
      "step": 1083,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 499.4140625,
      "completions/min_length": 257.5,
      "epoch": 5.857142857142857,
      "grad_norm": 3.6392745194869187,
      "kl": 1.98046875,
      "learning_rate": 3.7314730518598646e-07,
      "loss": 0.19013091921806335,
      "memory(GiB)": 75.26,
      "reward": 1.9714552164077759,
      "reward_std": 0.504738837480545,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.4363734573125839,
      "rewards/ReflectionBalance/mean": 0.7917675673961639,
      "rewards/ReflectionBalance/std": 0.26087193936109543,
      "step": 1084,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 493.78515625,
      "completions/min_length": 262.0,
      "epoch": 5.862533692722372,
      "grad_norm": 2.8620445821473557,
      "kl": 1.73046875,
      "learning_rate": 3.723176712936558e-07,
      "loss": 0.14056755602359772,
      "memory(GiB)": 75.26,
      "reward": 1.870408535003662,
      "reward_std": 0.4693242162466049,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.36039385199546814,
      "rewards/ReflectionBalance/mean": 0.8000960648059845,
      "rewards/ReflectionBalance/std": 0.2789856120944023,
      "step": 1085,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.0,
      "completions/mean_length": 468.02734375,
      "completions/min_length": 271.0,
      "epoch": 5.867924528301887,
      "grad_norm": 2.75857114585777,
      "kl": 1.484375,
      "learning_rate": 3.714884132852433e-07,
      "loss": 0.12358416616916656,
      "memory(GiB)": 75.26,
      "reward": 1.938916027545929,
      "reward_std": 0.4281282424926758,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4053537845611572,
      "rewards/ReflectionBalance/mean": 0.8100097477436066,
      "rewards/ReflectionBalance/std": 0.2649109587073326,
      "step": 1086,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 454.0859375,
      "completions/min_length": 261.5,
      "epoch": 5.873315363881401,
      "grad_norm": 2.673236806925586,
      "kl": 1.8203125,
      "learning_rate": 3.706595336020011e-07,
      "loss": 0.1303122639656067,
      "memory(GiB)": 75.26,
      "reward": 2.0409505367279053,
      "reward_std": 0.40076132118701935,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.41688379645347595,
      "rewards/ReflectionBalance/mean": 0.8456380367279053,
      "rewards/ReflectionBalance/std": 0.1726355254650116,
      "step": 1087,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 467.2265625,
      "completions/min_length": 262.5,
      "epoch": 5.878706199460916,
      "grad_norm": 3.308949690947315,
      "kl": 1.96875,
      "learning_rate": 3.6983103468406695e-07,
      "loss": 0.17747783660888672,
      "memory(GiB)": 75.26,
      "reward": 1.9404101371765137,
      "reward_std": 0.4748189300298691,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.396200567483902,
      "rewards/ReflectionBalance/mean": 0.7997851371765137,
      "rewards/ReflectionBalance/std": 0.25684110075235367,
      "step": 1088,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 908.0,
      "completions/mean_length": 490.8671875,
      "completions/min_length": 301.5,
      "epoch": 5.884097035040432,
      "grad_norm": 2.902046841018479,
      "kl": 1.76171875,
      "learning_rate": 3.6900291897045876e-07,
      "loss": 0.1494530588388443,
      "memory(GiB)": 75.26,
      "reward": 1.9897655844688416,
      "reward_std": 0.435814693570137,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41762344539165497,
      "rewards/ReflectionBalance/mean": 0.8413281440734863,
      "rewards/ReflectionBalance/std": 0.1799631118774414,
      "step": 1089,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 479.734375,
      "completions/min_length": 242.5,
      "epoch": 5.889487870619946,
      "grad_norm": 3.400858864492478,
      "kl": 2.09375,
      "learning_rate": 3.6817518889906594e-07,
      "loss": 0.19676169753074646,
      "memory(GiB)": 75.26,
      "reward": 1.8945117592811584,
      "reward_std": 0.4555952548980713,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27079907804727554,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35872696340084076,
      "rewards/ReflectionBalance/mean": 0.8241992592811584,
      "rewards/ReflectionBalance/std": 0.22177400439977646,
      "step": 1090,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 470.43359375,
      "completions/min_length": 237.0,
      "epoch": 5.894878706199461,
      "grad_norm": 3.5851297248778686,
      "kl": 3.078125,
      "learning_rate": 3.673478469066419e-07,
      "loss": 0.23223820328712463,
      "memory(GiB)": 75.26,
      "reward": 1.8155583143234253,
      "reward_std": 0.4633020758628845,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.33116789162158966,
      "rewards/ReflectionBalance/mean": 0.8038395047187805,
      "rewards/ReflectionBalance/std": 0.25146403908729553,
      "step": 1091,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 503.609375,
      "completions/min_length": 213.5,
      "epoch": 5.900269541778976,
      "grad_norm": 3.8958001382879694,
      "kl": 4.359375,
      "learning_rate": 3.665208954287986e-07,
      "loss": 0.3442418873310089,
      "memory(GiB)": 75.26,
      "reward": 1.7960937023162842,
      "reward_std": 0.6608254313468933,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.36145633459091187,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.39750634133815765,
      "rewards/ReflectionBalance/mean": 0.760937511920929,
      "rewards/ReflectionBalance/std": 0.29232726991176605,
      "step": 1092,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 514.7421875,
      "completions/min_length": 237.5,
      "epoch": 5.90566037735849,
      "grad_norm": 3.3771068800177106,
      "kl": 4.125,
      "learning_rate": 3.656943368999977e-07,
      "loss": 0.33510857820510864,
      "memory(GiB)": 75.26,
      "reward": 1.7150276899337769,
      "reward_std": 0.6855481564998627,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.3681531548500061,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.33595384657382965,
      "rewards/ReflectionBalance/mean": 0.7462776899337769,
      "rewards/ReflectionBalance/std": 0.31067992746829987,
      "step": 1093,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 496.5859375,
      "completions/min_length": 256.0,
      "epoch": 5.9110512129380055,
      "grad_norm": 16.826885079559975,
      "kl": 3.53125,
      "learning_rate": 3.648681737535443e-07,
      "loss": 0.26287609338760376,
      "memory(GiB)": 75.26,
      "reward": 1.7841482758522034,
      "reward_std": 0.5497913956642151,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3265802264213562,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.32050570100545883,
      "rewards/ReflectionBalance/mean": 0.7802420258522034,
      "rewards/ReflectionBalance/std": 0.3034851551055908,
      "step": 1094,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 515.140625,
      "completions/min_length": 261.5,
      "epoch": 5.916442048517521,
      "grad_norm": 4.323911735248679,
      "kl": 3.8515625,
      "learning_rate": 3.6404240842157976e-07,
      "loss": 0.31237488985061646,
      "memory(GiB)": 75.26,
      "reward": 1.862244427204132,
      "reward_std": 0.540374755859375,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3208130896091461,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3778642416000366,
      "rewards/ReflectionBalance/mean": 0.8036507070064545,
      "rewards/ReflectionBalance/std": 0.24840795993804932,
      "step": 1095,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 479.96484375,
      "completions/min_length": 265.5,
      "epoch": 5.921832884097035,
      "grad_norm": 3.169258422003419,
      "kl": 2.58203125,
      "learning_rate": 3.6321704333507354e-07,
      "loss": 0.2091139554977417,
      "memory(GiB)": 75.26,
      "reward": 1.917665958404541,
      "reward_std": 0.5462016463279724,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4059968739748001,
      "rewards/ReflectionBalance/mean": 0.7887597680091858,
      "rewards/ReflectionBalance/std": 0.28952107578516006,
      "step": 1096,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 475.765625,
      "completions/min_length": 274.5,
      "epoch": 5.92722371967655,
      "grad_norm": 2.7840760976234957,
      "kl": 2.3515625,
      "learning_rate": 3.6239208092381773e-07,
      "loss": 0.18875661492347717,
      "memory(GiB)": 75.26,
      "reward": 1.993699550628662,
      "reward_std": 0.5216903388500214,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4264890104532242,
      "rewards/ReflectionBalance/mean": 0.8491683006286621,
      "rewards/ReflectionBalance/std": 0.1832566112279892,
      "step": 1097,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 469.8359375,
      "completions/min_length": 194.5,
      "epoch": 5.932614555256064,
      "grad_norm": 2.7136236098969286,
      "kl": 2.1640625,
      "learning_rate": 3.6156752361641863e-07,
      "loss": 0.161906898021698,
      "memory(GiB)": 75.26,
      "reward": 1.8786572813987732,
      "reward_std": 0.5242660641670227,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3630705028772354,
      "rewards/ReflectionBalance/mean": 0.7966259717941284,
      "rewards/ReflectionBalance/std": 0.27151770889759064,
      "step": 1098,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 497.2109375,
      "completions/min_length": 175.0,
      "epoch": 5.938005390835579,
      "grad_norm": 4.065683048988528,
      "kl": 3.140625,
      "learning_rate": 3.6074337384028966e-07,
      "loss": 0.2597503662109375,
      "memory(GiB)": 75.26,
      "reward": 1.8297948837280273,
      "reward_std": 0.5260157585144043,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3358502835035324,
      "rewards/ReflectionBalance/mean": 0.7985449135303497,
      "rewards/ReflectionBalance/std": 0.2930224984884262,
      "step": 1099,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 494.390625,
      "completions/min_length": 181.0,
      "epoch": 5.943396226415095,
      "grad_norm": 2.9737567403391183,
      "kl": 3.0625,
      "learning_rate": 3.599196340216449e-07,
      "loss": 0.23274821043014526,
      "memory(GiB)": 75.26,
      "reward": 1.7867985367774963,
      "reward_std": 0.5447893738746643,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.34447427093982697,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35259076952934265,
      "rewards/ReflectionBalance/mean": 0.7750797271728516,
      "rewards/ReflectionBalance/std": 0.2897766977548599,
      "step": 1100,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 488.65625,
      "completions/min_length": 220.5,
      "epoch": 5.948787061994609,
      "grad_norm": 3.036613050810907,
      "kl": 3.1328125,
      "learning_rate": 3.5909630658549165e-07,
      "loss": 0.2391645610332489,
      "memory(GiB)": 75.26,
      "reward": 1.9624983668327332,
      "reward_std": 0.5598341524600983,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.322907418012619,
      "rewards/MultiModalAccuracyORM/mean": 0.265625,
      "rewards/MultiModalAccuracyORM/std": 0.4430450052022934,
      "rewards/ReflectionBalance/mean": 0.8140608370304108,
      "rewards/ReflectionBalance/std": 0.2769469767808914,
      "step": 1101,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 525.140625,
      "completions/min_length": 274.5,
      "epoch": 5.954177897574124,
      "grad_norm": 2.822022316825074,
      "kl": 3.1796875,
      "learning_rate": 3.5827339395562264e-07,
      "loss": 0.2517814636230469,
      "memory(GiB)": 75.26,
      "reward": 1.8859488368034363,
      "reward_std": 0.5504364222288132,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3191515803337097,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.3912966400384903,
      "rewards/ReflectionBalance/mean": 0.7765739262104034,
      "rewards/ReflectionBalance/std": 0.2902317941188812,
      "step": 1102,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1040.0,
      "completions/mean_length": 503.2890625,
      "completions/min_length": 268.0,
      "epoch": 5.959568733153639,
      "grad_norm": 3.216429906675204,
      "kl": 2.8984375,
      "learning_rate": 3.5745089855460974e-07,
      "loss": 0.22739377617835999,
      "memory(GiB)": 75.26,
      "reward": 1.8883365988731384,
      "reward_std": 0.5218440890312195,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.298691526055336,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3946705758571625,
      "rewards/ReflectionBalance/mean": 0.798492819070816,
      "rewards/ReflectionBalance/std": 0.283019483089447,
      "step": 1103,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 484.90625,
      "completions/min_length": 282.0,
      "epoch": 5.964959568733153,
      "grad_norm": 19.140428039989505,
      "kl": 1.87109375,
      "learning_rate": 3.566288228037969e-07,
      "loss": 0.16350087523460388,
      "memory(GiB)": 75.26,
      "reward": 1.9932568073272705,
      "reward_std": 0.44814494252204895,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.41018790006637573,
      "rewards/ReflectionBalance/mean": 0.8370068371295929,
      "rewards/ReflectionBalance/std": 0.20989320427179337,
      "step": 1104,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 485.40625,
      "completions/min_length": 229.0,
      "epoch": 5.9703504043126685,
      "grad_norm": 3.1335968355899855,
      "kl": 1.91015625,
      "learning_rate": 3.5580716912329217e-07,
      "loss": 0.16028712689876556,
      "memory(GiB)": 75.26,
      "reward": 1.9689078330993652,
      "reward_std": 0.5148323774337769,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4416199326515198,
      "rewards/ReflectionBalance/mean": 0.78531414270401,
      "rewards/ReflectionBalance/std": 0.28984779119491577,
      "step": 1105,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 898.5,
      "completions/mean_length": 454.64453125,
      "completions/min_length": 264.0,
      "epoch": 5.975741239892184,
      "grad_norm": 3.2088016245232422,
      "kl": 1.10546875,
      "learning_rate": 3.5498593993196125e-07,
      "loss": 0.0963970273733139,
      "memory(GiB)": 75.26,
      "reward": 2.0936279296875,
      "reward_std": 0.37544962763786316,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2255912721157074,
      "rewards/MultiModalAccuracyORM/mean": 0.3046875,
      "rewards/MultiModalAccuracyORM/std": 0.458208367228508,
      "rewards/ReflectionBalance/mean": 0.8436278998851776,
      "rewards/ReflectionBalance/std": 0.1889043226838112,
      "step": 1106,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 927.0,
      "completions/mean_length": 478.78515625,
      "completions/min_length": 303.5,
      "epoch": 5.981132075471698,
      "grad_norm": 3.098191735489097,
      "kl": 2.37109375,
      "learning_rate": 3.5416513764742036e-07,
      "loss": 0.19734565913677216,
      "memory(GiB)": 75.26,
      "reward": 1.9183006882667542,
      "reward_std": 0.4274822324514389,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2676052302122116,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3792887181043625,
      "rewards/ReflectionBalance/mean": 0.8284570276737213,
      "rewards/ReflectionBalance/std": 0.22628262639045715,
      "step": 1107,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 504.23046875,
      "completions/min_length": 272.5,
      "epoch": 5.986522911051213,
      "grad_norm": 97.09702934779881,
      "kl": 2.6640625,
      "learning_rate": 3.533447646860284e-07,
      "loss": 0.21591980755329132,
      "memory(GiB)": 75.26,
      "reward": 1.8031330108642578,
      "reward_std": 0.5151907801628113,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2918590381741524,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3320184051990509,
      "rewards/ReflectionBalance/mean": 0.7757893800735474,
      "rewards/ReflectionBalance/std": 0.3180917203426361,
      "step": 1108,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 500.89453125,
      "completions/min_length": 272.0,
      "epoch": 5.991913746630727,
      "grad_norm": 2.8942661606054854,
      "kl": 2.34375,
      "learning_rate": 3.5252482346288096e-07,
      "loss": 0.19452378153800964,
      "memory(GiB)": 75.26,
      "reward": 1.880773901939392,
      "reward_std": 0.5515475124120712,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2813730239868164,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3707977831363678,
      "rewards/ReflectionBalance/mean": 0.7987426519393921,
      "rewards/ReflectionBalance/std": 0.2911064326763153,
      "step": 1109,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 460.6640625,
      "completions/min_length": 254.5,
      "epoch": 5.997304582210242,
      "grad_norm": 2.567738664575849,
      "kl": 1.5390625,
      "learning_rate": 3.517053163918026e-07,
      "loss": 0.13559642434120178,
      "memory(GiB)": 75.26,
      "reward": 1.9022737741470337,
      "reward_std": 0.3317233622074127,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24081215262413025,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.344347283244133,
      "rewards/ReflectionBalance/mean": 0.8241487741470337,
      "rewards/ReflectionBalance/std": 0.2476048395037651,
      "step": 1110,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1046.5,
      "completions/mean_length": 497.79296875,
      "completions/min_length": 242.5,
      "epoch": 6.005390835579515,
      "grad_norm": 2.974156223249173,
      "kl": 1.67578125,
      "learning_rate": 3.5088624588533935e-07,
      "loss": 0.16713520884513855,
      "memory(GiB)": 75.26,
      "reward": 1.9571826457977295,
      "reward_std": 0.4028323292732239,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3761351853609085,
      "rewards/ReflectionBalance/mean": 0.8399951159954071,
      "rewards/ReflectionBalance/std": 0.21074151247739792,
      "step": 1111,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 490.40234375,
      "completions/min_length": 210.5,
      "epoch": 6.010781671159029,
      "grad_norm": 3.1797778882787506,
      "kl": 1.796875,
      "learning_rate": 3.500676143547523e-07,
      "loss": 0.12716028094291687,
      "memory(GiB)": 75.26,
      "reward": 2.0760953426361084,
      "reward_std": 0.4393564313650131,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.3203125,
      "rewards/MultiModalAccuracyORM/std": 0.4484759718179703,
      "rewards/ReflectionBalance/mean": 0.8182828426361084,
      "rewards/ReflectionBalance/std": 0.22118276357650757,
      "step": 1112,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 475.46484375,
      "completions/min_length": 253.0,
      "epoch": 6.0161725067385445,
      "grad_norm": 3.2876575477148497,
      "kl": 1.9453125,
      "learning_rate": 3.4924942421001066e-07,
      "loss": 0.1814986765384674,
      "memory(GiB)": 75.26,
      "reward": 2.0468952655792236,
      "reward_std": 0.49238933622837067,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4483988285064697,
      "rewards/ReflectionBalance/mean": 0.8359578549861908,
      "rewards/ReflectionBalance/std": 0.20002985000610352,
      "step": 1113,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 485.01953125,
      "completions/min_length": 261.5,
      "epoch": 6.02156334231806,
      "grad_norm": 6.080285054418376,
      "kl": 3.3828125,
      "learning_rate": 3.4843167785978323e-07,
      "loss": 0.258439302444458,
      "memory(GiB)": 75.26,
      "reward": 1.8794286847114563,
      "reward_std": 0.5650513768196106,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.3526121824979782,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4264890104532242,
      "rewards/ReflectionBalance/mean": 0.7856787443161011,
      "rewards/ReflectionBalance/std": 0.28072744607925415,
      "step": 1114,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 512.81640625,
      "completions/min_length": 184.0,
      "epoch": 6.026954177897574,
      "grad_norm": 3.710837682808742,
      "kl": 3.8046875,
      "learning_rate": 3.476143777114332e-07,
      "loss": 0.2669246196746826,
      "memory(GiB)": 75.26,
      "reward": 1.8018896579742432,
      "reward_std": 0.6152731776237488,
      "rewards/Format/mean": 0.83203125,
      "rewards/Format/std": 0.37438808381557465,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.3937542736530304,
      "rewards/ReflectionBalance/mean": 0.7472021579742432,
      "rewards/ReflectionBalance/std": 0.31658002734184265,
      "step": 1115,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 463.59375,
      "completions/min_length": 254.5,
      "epoch": 6.032345013477089,
      "grad_norm": 4.558343736825836,
      "kl": 3.015625,
      "learning_rate": 3.4679752617101e-07,
      "loss": 0.22394905984401703,
      "memory(GiB)": 75.26,
      "reward": 1.9385433197021484,
      "reward_std": 0.49618276953697205,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4276812821626663,
      "rewards/ReflectionBalance/mean": 0.7940120697021484,
      "rewards/ReflectionBalance/std": 0.26732076704502106,
      "step": 1116,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.0,
      "completions/mean_length": 541.25,
      "completions/min_length": 151.5,
      "epoch": 6.037735849056604,
      "grad_norm": 4.6356535353003805,
      "kl": 5.1484375,
      "learning_rate": 3.459811256432421e-07,
      "loss": 0.39748814702033997,
      "memory(GiB)": 75.26,
      "reward": 1.7435400485992432,
      "reward_std": 0.68841153383255,
      "rewards/Format/mean": 0.82421875,
      "rewards/Format/std": 0.38181672990322113,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3774644732475281,
      "rewards/ReflectionBalance/mean": 0.7474462985992432,
      "rewards/ReflectionBalance/std": 0.3259670287370682,
      "step": 1117,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 527.32421875,
      "completions/min_length": 194.0,
      "epoch": 6.0431266846361185,
      "grad_norm": 3.8474658824952623,
      "kl": 4.890625,
      "learning_rate": 3.451651785315308e-07,
      "loss": 0.36490583419799805,
      "memory(GiB)": 75.26,
      "reward": 1.6943180561065674,
      "reward_std": 0.7171710729598999,
      "rewards/Format/mean": 0.81640625,
      "rewards/Format/std": 0.38837648928165436,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.32733818888664246,
      "rewards/ReflectionBalance/mean": 0.7450993061065674,
      "rewards/ReflectionBalance/std": 0.36185450851917267,
      "step": 1118,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 501.328125,
      "completions/min_length": 285.0,
      "epoch": 6.048517520215634,
      "grad_norm": 3.1467485290972372,
      "kl": 2.859375,
      "learning_rate": 3.443496872379417e-07,
      "loss": 0.2692997455596924,
      "memory(GiB)": 75.26,
      "reward": 1.8690641522407532,
      "reward_std": 0.4927234798669815,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29262590408325195,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.37157969176769257,
      "rewards/ReflectionBalance/mean": 0.7909391224384308,
      "rewards/ReflectionBalance/std": 0.2925659567117691,
      "step": 1119,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 495.5625,
      "completions/min_length": 205.5,
      "epoch": 6.053908355795148,
      "grad_norm": 4.590383669144696,
      "kl": 4.1328125,
      "learning_rate": 3.435346541631993e-07,
      "loss": 0.3137407898902893,
      "memory(GiB)": 75.26,
      "reward": 1.8119481801986694,
      "reward_std": 0.5742311179637909,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3460448384284973,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.4122922718524933,
      "rewards/ReflectionBalance/mean": 0.7338232398033142,
      "rewards/ReflectionBalance/std": 0.36688998341560364,
      "step": 1120,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 493.46484375,
      "completions/min_length": 230.5,
      "epoch": 6.059299191374663,
      "grad_norm": 2.752900796824547,
      "kl": 2.4765625,
      "learning_rate": 3.4272008170667923e-07,
      "loss": 0.18657681345939636,
      "memory(GiB)": 75.26,
      "reward": 1.8393830060958862,
      "reward_std": 0.48869548738002777,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35062892735004425,
      "rewards/ReflectionBalance/mean": 0.7925079762935638,
      "rewards/ReflectionBalance/std": 0.24509261548519135,
      "step": 1121,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 501.51171875,
      "completions/min_length": 208.5,
      "epoch": 6.064690026954178,
      "grad_norm": 2.937383052138618,
      "kl": 2.53125,
      "learning_rate": 3.4190597226640025e-07,
      "loss": 0.21402224898338318,
      "memory(GiB)": 75.26,
      "reward": 1.9080777764320374,
      "reward_std": 0.4930301308631897,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3897709399461746,
      "rewards/ReflectionBalance/mean": 0.8104215562343597,
      "rewards/ReflectionBalance/std": 0.2762753367424011,
      "step": 1122,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 472.34375,
      "completions/min_length": 236.5,
      "epoch": 6.070080862533692,
      "grad_norm": 3.599945202785192,
      "kl": 2.3359375,
      "learning_rate": 3.4109232823901874e-07,
      "loss": 0.1898176074028015,
      "memory(GiB)": 75.26,
      "reward": 1.952475905418396,
      "reward_std": 0.5075347721576691,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2875154986977577,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4243630915880203,
      "rewards/ReflectionBalance/mean": 0.811850905418396,
      "rewards/ReflectionBalance/std": 0.25288810580968857,
      "step": 1123,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 459.2890625,
      "completions/min_length": 235.5,
      "epoch": 6.0754716981132075,
      "grad_norm": 2.8360122203918667,
      "kl": 1.7734375,
      "learning_rate": 3.4027915201982096e-07,
      "loss": 0.11551278829574585,
      "memory(GiB)": 75.26,
      "reward": 1.9284716844558716,
      "reward_std": 0.46699705719947815,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3949224203824997,
      "rewards/ReflectionBalance/mean": 0.8112841844558716,
      "rewards/ReflectionBalance/std": 0.24094577133655548,
      "step": 1124,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 491.12890625,
      "completions/min_length": 231.5,
      "epoch": 6.080862533692723,
      "grad_norm": 2.659998775617247,
      "kl": 2.15234375,
      "learning_rate": 3.3946644600271557e-07,
      "loss": 0.1660965383052826,
      "memory(GiB)": 75.26,
      "reward": 1.9562028646469116,
      "reward_std": 0.41531454026699066,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4067413955926895,
      "rewards/ReflectionBalance/mean": 0.8233903050422668,
      "rewards/ReflectionBalance/std": 0.23792804032564163,
      "step": 1125,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 488.7578125,
      "completions/min_length": 278.0,
      "epoch": 6.086253369272237,
      "grad_norm": 2.8696653973596375,
      "kl": 1.69921875,
      "learning_rate": 3.386542125802274e-07,
      "loss": 0.11389429867267609,
      "memory(GiB)": 75.26,
      "reward": 1.9564160108566284,
      "reward_std": 0.5429192781448364,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2829600051045418,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.40902179479599,
      "rewards/ReflectionBalance/mean": 0.7962597608566284,
      "rewards/ReflectionBalance/std": 0.27940069139003754,
      "step": 1126,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 974.0,
      "completions/mean_length": 485.3203125,
      "completions/min_length": 254.5,
      "epoch": 6.091644204851752,
      "grad_norm": 2.9885988195228483,
      "kl": 1.21875,
      "learning_rate": 3.3784245414349e-07,
      "loss": 0.12067306041717529,
      "memory(GiB)": 75.26,
      "reward": 1.9743245244026184,
      "reward_std": 0.3039819821715355,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.19744773209095,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38489311933517456,
      "rewards/ReflectionBalance/mean": 0.8376057744026184,
      "rewards/ReflectionBalance/std": 0.17346946150064468,
      "step": 1127,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 496.3046875,
      "completions/min_length": 217.5,
      "epoch": 6.097035040431267,
      "grad_norm": 2.6869637053889397,
      "kl": 1.7265625,
      "learning_rate": 3.3703117308223807e-07,
      "loss": 0.14999008178710938,
      "memory(GiB)": 75.26,
      "reward": 1.9639225006103516,
      "reward_std": 0.4654502421617508,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.258706197142601,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4065109044313431,
      "rewards/ReflectionBalance/mean": 0.8350162506103516,
      "rewards/ReflectionBalance/std": 0.2433980107307434,
      "step": 1128,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 938.0,
      "completions/mean_length": 469.0546875,
      "completions/min_length": 237.5,
      "epoch": 6.1024258760107815,
      "grad_norm": 3.2346415348970115,
      "kl": 1.28515625,
      "learning_rate": 3.362203717848015e-07,
      "loss": 0.10233209282159805,
      "memory(GiB)": 75.26,
      "reward": 2.0733343362808228,
      "reward_std": 0.4286128282546997,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.31640625,
      "rewards/MultiModalAccuracyORM/std": 0.4599490314722061,
      "rewards/ReflectionBalance/mean": 0.8350530564785004,
      "rewards/ReflectionBalance/std": 0.20702797174453735,
      "step": 1129,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 496.75,
      "completions/min_length": 264.5,
      "epoch": 6.107816711590297,
      "grad_norm": 2.828023758083383,
      "kl": 1.50390625,
      "learning_rate": 3.3541005263809786e-07,
      "loss": 0.124593585729599,
      "memory(GiB)": 75.26,
      "reward": 1.9050554037094116,
      "reward_std": 0.4491468369960785,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3707977831363678,
      "rewards/ReflectionBalance/mean": 0.8113053143024445,
      "rewards/ReflectionBalance/std": 0.27769699692726135,
      "step": 1130,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 508.5390625,
      "completions/min_length": 251.5,
      "epoch": 6.113207547169812,
      "grad_norm": 3.008141369199531,
      "kl": 1.3984375,
      "learning_rate": 3.3460021802762496e-07,
      "loss": 0.11211448162794113,
      "memory(GiB)": 75.26,
      "reward": 1.9878449440002441,
      "reward_std": 0.39818595349788666,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2524137645959854,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.40963114798069,
      "rewards/ReflectionBalance/mean": 0.8394075334072113,
      "rewards/ReflectionBalance/std": 0.19118838012218475,
      "step": 1131,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 943.5,
      "completions/mean_length": 498.23046875,
      "completions/min_length": 278.5,
      "epoch": 6.118598382749326,
      "grad_norm": 890.2438784143251,
      "kl": 4.12109375,
      "learning_rate": 3.337908703374542e-07,
      "loss": 0.23878417909145355,
      "memory(GiB)": 75.26,
      "reward": 1.9476221799850464,
      "reward_std": 0.38261452317237854,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24678732454776764,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3230259120464325,
      "rewards/ReflectionBalance/mean": 0.8343408107757568,
      "rewards/ReflectionBalance/std": 0.2204810306429863,
      "step": 1132,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 517.140625,
      "completions/min_length": 258.0,
      "epoch": 6.123989218328841,
      "grad_norm": 3.0540748586471858,
      "kl": 2.48828125,
      "learning_rate": 3.329820119502242e-07,
      "loss": 0.22200945019721985,
      "memory(GiB)": 75.26,
      "reward": 1.9047566652297974,
      "reward_std": 0.5205066949129105,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3068038523197174,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3871416449546814,
      "rewards/ReflectionBalance/mean": 0.8149129152297974,
      "rewards/ReflectionBalance/std": 0.26435578614473343,
      "step": 1133,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1020.0,
      "completions/mean_length": 455.5859375,
      "completions/min_length": 249.0,
      "epoch": 6.129380053908355,
      "grad_norm": 2.8580180151476497,
      "kl": 1.41796875,
      "learning_rate": 3.32173645247132e-07,
      "loss": 0.11718269437551498,
      "memory(GiB)": 75.26,
      "reward": 2.0458340644836426,
      "reward_std": 0.5069065093994141,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.2890625,
      "rewards/MultiModalAccuracyORM/std": 0.4550260454416275,
      "rewards/ReflectionBalance/mean": 0.8192715048789978,
      "rewards/ReflectionBalance/std": 0.2578919678926468,
      "step": 1134,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 481.1484375,
      "completions/min_length": 205.0,
      "epoch": 6.1347708894878705,
      "grad_norm": 3.03063037672029,
      "kl": 2.53125,
      "learning_rate": 3.31365772607928e-07,
      "loss": 0.180486798286438,
      "memory(GiB)": 75.26,
      "reward": 1.9218652248382568,
      "reward_std": 0.560055136680603,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.424868568778038,
      "rewards/ReflectionBalance/mean": 0.7968652546405792,
      "rewards/ReflectionBalance/std": 0.2768699377775192,
      "step": 1135,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 503.45703125,
      "completions/min_length": 265.5,
      "epoch": 6.140161725067386,
      "grad_norm": 3.307234057944,
      "kl": 2.7890625,
      "learning_rate": 3.30558396410908e-07,
      "loss": 0.20339764654636383,
      "memory(GiB)": 75.26,
      "reward": 1.8365339040756226,
      "reward_std": 0.4924761801958084,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.322907418012619,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.356930136680603,
      "rewards/ReflectionBalance/mean": 0.8052840232849121,
      "rewards/ReflectionBalance/std": 0.2593526318669319,
      "step": 1136,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 495.5546875,
      "completions/min_length": 217.0,
      "epoch": 6.1455525606469,
      "grad_norm": 3.711198581681397,
      "kl": 3.203125,
      "learning_rate": 3.297515190329061e-07,
      "loss": 0.22743235528469086,
      "memory(GiB)": 75.26,
      "reward": 1.8972867727279663,
      "reward_std": 0.5023946166038513,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.310598224401474,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3978758305311203,
      "rewards/ReflectionBalance/mean": 0.8152555525302887,
      "rewards/ReflectionBalance/std": 0.2632276117801666,
      "step": 1137,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.0,
      "completions/mean_length": 499.46484375,
      "completions/min_length": 259.5,
      "epoch": 6.150943396226415,
      "grad_norm": 3.228841870553105,
      "kl": 3.0703125,
      "learning_rate": 3.2894514284928796e-07,
      "loss": 0.2555301785469055,
      "memory(GiB)": 75.26,
      "reward": 1.8876383900642395,
      "reward_std": 0.4862782061100006,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.298691526055336,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3968885838985443,
      "rewards/ReflectionBalance/mean": 0.7938883602619171,
      "rewards/ReflectionBalance/std": 0.28702107071876526,
      "step": 1138,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 514.0546875,
      "completions/min_length": 193.0,
      "epoch": 6.15633423180593,
      "grad_norm": 3.1543024644159834,
      "kl": 3.2890625,
      "learning_rate": 3.281392702339443e-07,
      "loss": 0.2743436098098755,
      "memory(GiB)": 75.26,
      "reward": 1.8258243799209595,
      "reward_std": 0.5607051700353622,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3191515803337097,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35872696340084076,
      "rewards/ReflectionBalance/mean": 0.7906681597232819,
      "rewards/ReflectionBalance/std": 0.3102886974811554,
      "step": 1139,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 502.11328125,
      "completions/min_length": 242.5,
      "epoch": 6.1617250673854445,
      "grad_norm": 14.288238860075568,
      "kl": 2.8359375,
      "learning_rate": 3.273339035592824e-07,
      "loss": 0.2299506962299347,
      "memory(GiB)": 75.26,
      "reward": 1.838061511516571,
      "reward_std": 0.6243001520633698,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.33569003641605377,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4020542651414871,
      "rewards/ReflectionBalance/mean": 0.756030261516571,
      "rewards/ReflectionBalance/std": 0.373797208070755,
      "step": 1140,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 506.3359375,
      "completions/min_length": 178.0,
      "epoch": 6.16711590296496,
      "grad_norm": 4.402151968969331,
      "kl": 2.765625,
      "learning_rate": 3.2652904519622093e-07,
      "loss": 0.22733637690544128,
      "memory(GiB)": 75.26,
      "reward": 1.8506142497062683,
      "reward_std": 0.5890680700540543,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.3390307128429413,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3980395793914795,
      "rewards/ReflectionBalance/mean": 0.7842080295085907,
      "rewards/ReflectionBalance/std": 0.28643715381622314,
      "step": 1141,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 510.17578125,
      "completions/min_length": 269.5,
      "epoch": 6.172506738544475,
      "grad_norm": 3.5727707531492676,
      "kl": 3.7421875,
      "learning_rate": 3.2572469751418174e-07,
      "loss": 0.320066899061203,
      "memory(GiB)": 75.26,
      "reward": 1.9116577506065369,
      "reward_std": 0.5620350539684296,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.3166656941175461,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.42517124116420746,
      "rewards/ReflectionBalance/mean": 0.7905639410018921,
      "rewards/ReflectionBalance/std": 0.29653826355934143,
      "step": 1142,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1037.0,
      "completions/mean_length": 500.34765625,
      "completions/min_length": 191.0,
      "epoch": 6.177897574123989,
      "grad_norm": 3.559653896684376,
      "kl": 2.8984375,
      "learning_rate": 3.2492086288108344e-07,
      "loss": 0.22898311913013458,
      "memory(GiB)": 75.26,
      "reward": 1.8371597528457642,
      "reward_std": 0.5973963141441345,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3300945907831192,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3752707988023758,
      "rewards/ReflectionBalance/mean": 0.7941911220550537,
      "rewards/ReflectionBalance/std": 0.25785670429468155,
      "step": 1143,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 487.2734375,
      "completions/min_length": 284.0,
      "epoch": 6.183288409703504,
      "grad_norm": 2.9980362109901457,
      "kl": 2.421875,
      "learning_rate": 3.2411754366333453e-07,
      "loss": 0.1871849149465561,
      "memory(GiB)": 75.26,
      "reward": 1.9501920342445374,
      "reward_std": 0.4010855555534363,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.4020131081342697,
      "rewards/ReflectionBalance/mean": 0.8212857842445374,
      "rewards/ReflectionBalance/std": 0.25485809892416,
      "step": 1144,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 483.16796875,
      "completions/min_length": 214.5,
      "epoch": 6.188679245283019,
      "grad_norm": 3.08250580060049,
      "kl": 1.87890625,
      "learning_rate": 3.2331474222582544e-07,
      "loss": 0.1523260772228241,
      "memory(GiB)": 75.26,
      "reward": 1.912662148475647,
      "reward_std": 0.46003296971321106,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.8110997676849365,
      "rewards/ReflectionBalance/std": 0.2327609360218048,
      "step": 1145,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 457.421875,
      "completions/min_length": 242.5,
      "epoch": 6.1940700808625335,
      "grad_norm": 2.7224217751371578,
      "kl": 1.7421875,
      "learning_rate": 3.22512460931923e-07,
      "loss": 0.15552666783332825,
      "memory(GiB)": 75.26,
      "reward": 2.0037923455238342,
      "reward_std": 0.4071824699640274,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21875878423452377,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.40383824706077576,
      "rewards/ReflectionBalance/mean": 0.8358235955238342,
      "rewards/ReflectionBalance/std": 0.1881236433982849,
      "step": 1146,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 477.74609375,
      "completions/min_length": 265.0,
      "epoch": 6.199460916442049,
      "grad_norm": 3.6123799887099013,
      "kl": 2.09765625,
      "learning_rate": 3.217107021434626e-07,
      "loss": 0.18787109851837158,
      "memory(GiB)": 75.26,
      "reward": 2.02035653591156,
      "reward_std": 0.5224110037088394,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4416199326515198,
      "rewards/ReflectionBalance/mean": 0.8211376965045929,
      "rewards/ReflectionBalance/std": 0.2347608059644699,
      "step": 1147,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 494.57421875,
      "completions/min_length": 296.0,
      "epoch": 6.204851752021563,
      "grad_norm": 3.14622592356396,
      "kl": 2.65625,
      "learning_rate": 3.209094682207412e-07,
      "loss": 0.24975401163101196,
      "memory(GiB)": 75.26,
      "reward": 1.9073599576950073,
      "reward_std": 0.4990188777446747,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.4002292603254318,
      "rewards/ReflectionBalance/mean": 0.7901725172996521,
      "rewards/ReflectionBalance/std": 0.29036231338977814,
      "step": 1148,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 491.55859375,
      "completions/min_length": 267.0,
      "epoch": 6.210242587601078,
      "grad_norm": 2.902852907159049,
      "kl": 1.859375,
      "learning_rate": 3.201087615225108e-07,
      "loss": 0.13792937994003296,
      "memory(GiB)": 75.26,
      "reward": 1.8995556235313416,
      "reward_std": 0.4265096038579941,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3475525379180908,
      "rewards/ReflectionBalance/mean": 0.8409619033336639,
      "rewards/ReflectionBalance/std": 0.21142766624689102,
      "step": 1149,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 500.86328125,
      "completions/min_length": 270.0,
      "epoch": 6.215633423180593,
      "grad_norm": 2.8140313454527677,
      "kl": 2.796875,
      "learning_rate": 3.193085844059715e-07,
      "loss": 0.24009230732917786,
      "memory(GiB)": 75.26,
      "reward": 1.920444369316101,
      "reward_std": 0.4829387366771698,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.28456588089466095,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.40284374356269836,
      "rewards/ReflectionBalance/mean": 0.8149756193161011,
      "rewards/ReflectionBalance/std": 0.23587217926979065,
      "step": 1150,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 482.25,
      "completions/min_length": 262.0,
      "epoch": 6.2210242587601075,
      "grad_norm": 2.9283416102060773,
      "kl": 1.95703125,
      "learning_rate": 3.1850893922676393e-07,
      "loss": 0.1619414985179901,
      "memory(GiB)": 75.26,
      "reward": 1.8544327020645142,
      "reward_std": 0.4357129633426666,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3318064510822296,
      "rewards/ReflectionBalance/mean": 0.8153702914714813,
      "rewards/ReflectionBalance/std": 0.24950602650642395,
      "step": 1151,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 516.5625,
      "completions/min_length": 263.5,
      "epoch": 6.226415094339623,
      "grad_norm": 3.047509162842933,
      "kl": 2.21875,
      "learning_rate": 3.1770982833896296e-07,
      "loss": 0.15938806533813477,
      "memory(GiB)": 75.26,
      "reward": 1.88712739944458,
      "reward_std": 0.4952205568552017,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.371787428855896,
      "rewards/ReflectionBalance/mean": 0.8207210302352905,
      "rewards/ReflectionBalance/std": 0.2245127409696579,
      "step": 1152,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 517.99609375,
      "completions/min_length": 304.0,
      "epoch": 6.231805929919138,
      "grad_norm": 3.42979433501648,
      "kl": 2.796875,
      "learning_rate": 3.16911254095071e-07,
      "loss": 0.22592535614967346,
      "memory(GiB)": 75.26,
      "reward": 1.8365380764007568,
      "reward_std": 0.48653078079223633,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3334042727947235,
      "rewards/ReflectionBalance/mean": 0.8013818264007568,
      "rewards/ReflectionBalance/std": 0.2565625086426735,
      "step": 1153,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 491.58203125,
      "completions/min_length": 273.0,
      "epoch": 6.237196765498652,
      "grad_norm": 3.6782300537126744,
      "kl": 3.0078125,
      "learning_rate": 3.161132188460096e-07,
      "loss": 0.2515614628791809,
      "memory(GiB)": 75.26,
      "reward": 1.8914957642555237,
      "reward_std": 0.5605341494083405,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32746291160583496,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.40759414434432983,
      "rewards/ReflectionBalance/mean": 0.7977457940578461,
      "rewards/ReflectionBalance/std": 0.26215992867946625,
      "step": 1154,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 496.7734375,
      "completions/min_length": 289.5,
      "epoch": 6.242587601078167,
      "grad_norm": 2.6384905605442874,
      "kl": 1.73828125,
      "learning_rate": 3.153157249411145e-07,
      "loss": 0.12736326456069946,
      "memory(GiB)": 75.26,
      "reward": 1.9135839939117432,
      "reward_std": 0.39909230172634125,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3643587827682495,
      "rewards/ReflectionBalance/mean": 0.8315527439117432,
      "rewards/ReflectionBalance/std": 0.22094029188156128,
      "step": 1155,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 488.7578125,
      "completions/min_length": 279.5,
      "epoch": 6.247978436657682,
      "grad_norm": 2.8895907907173766,
      "kl": 2.35546875,
      "learning_rate": 3.145187747281275e-07,
      "loss": 0.2086728811264038,
      "memory(GiB)": 75.26,
      "reward": 1.954250454902649,
      "reward_std": 0.4385036379098892,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4322105348110199,
      "rewards/ReflectionBalance/mean": 0.7901879847049713,
      "rewards/ReflectionBalance/std": 0.2930421680212021,
      "step": 1156,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 492.92578125,
      "completions/min_length": 274.0,
      "epoch": 6.2533692722371965,
      "grad_norm": 2.9026872438878373,
      "kl": 2.4609375,
      "learning_rate": 3.137223705531895e-07,
      "loss": 0.19571910798549652,
      "memory(GiB)": 75.26,
      "reward": 1.855050504207611,
      "reward_std": 0.48290328681468964,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36387722194194794,
      "rewards/ReflectionBalance/mean": 0.7964566946029663,
      "rewards/ReflectionBalance/std": 0.2669084966182709,
      "step": 1157,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 490.140625,
      "completions/min_length": 261.0,
      "epoch": 6.258760107816712,
      "grad_norm": 3.609780516132449,
      "kl": 3.703125,
      "learning_rate": 3.129265147608344e-07,
      "loss": 0.32898107171058655,
      "memory(GiB)": 75.26,
      "reward": 1.9300048351287842,
      "reward_std": 0.656242847442627,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.34485287964344025,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.44320113956928253,
      "rewards/ReflectionBalance/mean": 0.7893798649311066,
      "rewards/ReflectionBalance/std": 0.310108482837677,
      "step": 1158,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.5,
      "completions/mean_length": 505.62109375,
      "completions/min_length": 284.0,
      "epoch": 6.264150943396227,
      "grad_norm": 2.968924275223475,
      "kl": 3.75,
      "learning_rate": 3.1213120969398174e-07,
      "loss": 0.31783950328826904,
      "memory(GiB)": 75.26,
      "reward": 1.8309603333473206,
      "reward_std": 0.5710649490356445,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3247780203819275,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38597656786441803,
      "rewards/ReflectionBalance/mean": 0.7684603035449982,
      "rewards/ReflectionBalance/std": 0.3115979731082916,
      "step": 1159,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 499.171875,
      "completions/min_length": 306.0,
      "epoch": 6.269541778975741,
      "grad_norm": 3.35585332579424,
      "kl": 2.66796875,
      "learning_rate": 3.1133645769392903e-07,
      "loss": 0.2165214866399765,
      "memory(GiB)": 75.26,
      "reward": 1.9849196672439575,
      "reward_std": 0.5021049976348877,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2875154986977577,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.43386341631412506,
      "rewards/ReflectionBalance/mean": 0.8286697566509247,
      "rewards/ReflectionBalance/std": 0.19586969912052155,
      "step": 1160,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 468.0546875,
      "completions/min_length": 247.0,
      "epoch": 6.274932614555256,
      "grad_norm": 2.5711586507216766,
      "kl": 1.56640625,
      "learning_rate": 3.105422611003463e-07,
      "loss": 0.1435432732105255,
      "memory(GiB)": 75.26,
      "reward": 2.0566844940185547,
      "reward_std": 0.43733347952365875,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.4473883658647537,
      "rewards/ReflectionBalance/mean": 0.8418408632278442,
      "rewards/ReflectionBalance/std": 0.22964337468147278,
      "step": 1161,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 478.02734375,
      "completions/min_length": 280.0,
      "epoch": 6.280323450134771,
      "grad_norm": 9.266337813919817,
      "kl": 3.1875,
      "learning_rate": 3.097486222512686e-07,
      "loss": 0.2827274799346924,
      "memory(GiB)": 75.26,
      "reward": 2.004000663757324,
      "reward_std": 0.5365844517946243,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.28448406606912613,
      "rewards/MultiModalAccuracyORM/mean": 0.29296875,
      "rewards/MultiModalAccuracyORM/std": 0.4439031779766083,
      "rewards/ReflectionBalance/mean": 0.8047818541526794,
      "rewards/ReflectionBalance/std": 0.2703518867492676,
      "step": 1162,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 518.88671875,
      "completions/min_length": 187.5,
      "epoch": 6.285714285714286,
      "grad_norm": 3.1086471222738594,
      "kl": 3.5,
      "learning_rate": 3.0895554348308847e-07,
      "loss": 0.27277901768684387,
      "memory(GiB)": 75.26,
      "reward": 1.7629460096359253,
      "reward_std": 0.5286374092102051,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3375232368707657,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3375232368707657,
      "rewards/ReflectionBalance/mean": 0.7629459500312805,
      "rewards/ReflectionBalance/std": 0.3206920474767685,
      "step": 1163,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 479.26953125,
      "completions/min_length": 262.0,
      "epoch": 6.291105121293801,
      "grad_norm": 3.4453863124806556,
      "kl": 2.828125,
      "learning_rate": 3.081630271305502e-07,
      "loss": 0.21657884120941162,
      "memory(GiB)": 75.26,
      "reward": 1.9706494808197021,
      "reward_std": 0.5714651048183441,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2918590381741524,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4468510299921036,
      "rewards/ReflectionBalance/mean": 0.7909619212150574,
      "rewards/ReflectionBalance/std": 0.2661203145980835,
      "step": 1164,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 500.40234375,
      "completions/min_length": 147.0,
      "epoch": 6.296495956873315,
      "grad_norm": 4.228230805437538,
      "kl": 3.328125,
      "learning_rate": 3.0737107552674206e-07,
      "loss": 0.2616719901561737,
      "memory(GiB)": 75.26,
      "reward": 1.8238574266433716,
      "reward_std": 0.5991307497024536,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.34826864302158356,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39110109210014343,
      "rewards/ReflectionBalance/mean": 0.7730761766433716,
      "rewards/ReflectionBalance/std": 0.3286978155374527,
      "step": 1165,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 513.03125,
      "completions/min_length": 245.5,
      "epoch": 6.30188679245283,
      "grad_norm": 3.243081875142705,
      "kl": 2.71484375,
      "learning_rate": 3.065796910030897e-07,
      "loss": 0.21350130438804626,
      "memory(GiB)": 75.26,
      "reward": 1.7963980436325073,
      "reward_std": 0.5314040184020996,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.33116789162158966,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3460448384284973,
      "rewards/ReflectionBalance/mean": 0.7807731032371521,
      "rewards/ReflectionBalance/std": 0.26615381240844727,
      "step": 1166,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 508.82421875,
      "completions/min_length": 275.5,
      "epoch": 6.307277628032345,
      "grad_norm": 3.498625747647118,
      "kl": 3.171875,
      "learning_rate": 3.057888758893495e-07,
      "loss": 0.25470784306526184,
      "memory(GiB)": 75.26,
      "reward": 1.8149381279945374,
      "reward_std": 0.528798907995224,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3270204961299896,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3526121824979782,
      "rewards/ReflectionBalance/mean": 0.7915006577968597,
      "rewards/ReflectionBalance/std": 0.2533020153641701,
      "step": 1167,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 475.80078125,
      "completions/min_length": 131.5,
      "epoch": 6.3126684636118595,
      "grad_norm": 3.1197632023686235,
      "kl": 2.5546875,
      "learning_rate": 3.0499863251360145e-07,
      "loss": 0.1987750232219696,
      "memory(GiB)": 75.26,
      "reward": 1.9874170422554016,
      "reward_std": 0.44029806554317474,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.2960064336657524,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.44771209359169006,
      "rewards/ReflectionBalance/mean": 0.8116357326507568,
      "rewards/ReflectionBalance/std": 0.24679478257894516,
      "step": 1168,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 489.953125,
      "completions/min_length": 287.5,
      "epoch": 6.318059299191375,
      "grad_norm": 3.264977472639355,
      "kl": 2.2109375,
      "learning_rate": 3.0420896320224266e-07,
      "loss": 0.19518721103668213,
      "memory(GiB)": 75.26,
      "reward": 1.9354516863822937,
      "reward_std": 0.44609394669532776,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2566775679588318,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39238542318344116,
      "rewards/ReflectionBalance/mean": 0.8143579065799713,
      "rewards/ReflectionBalance/std": 0.24398387223482132,
      "step": 1169,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 473.04296875,
      "completions/min_length": 288.0,
      "epoch": 6.32345013477089,
      "grad_norm": 3.877005016666185,
      "kl": 1.609375,
      "learning_rate": 3.034198702799801e-07,
      "loss": 0.16192391514778137,
      "memory(GiB)": 75.26,
      "reward": 1.9636719226837158,
      "reward_std": 0.503593921661377,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4094643145799637,
      "rewards/ReflectionBalance/mean": 0.8269531428813934,
      "rewards/ReflectionBalance/std": 0.2639329209923744,
      "step": 1170,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 973.5,
      "completions/mean_length": 489.91796875,
      "completions/min_length": 260.0,
      "epoch": 6.328840970350404,
      "grad_norm": 2.824622331540634,
      "kl": 1.5,
      "learning_rate": 3.026313560698235e-07,
      "loss": 0.11346986144781113,
      "memory(GiB)": 75.26,
      "reward": 1.9217219948768616,
      "reward_std": 0.43444155156612396,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39184603095054626,
      "rewards/ReflectionBalance/mean": 0.800628274679184,
      "rewards/ReflectionBalance/std": 0.2442913055419922,
      "step": 1171,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 517.33203125,
      "completions/min_length": 295.5,
      "epoch": 6.334231805929919,
      "grad_norm": 3.36782519898268,
      "kl": 2.3359375,
      "learning_rate": 3.0184342289307965e-07,
      "loss": 0.1934920996427536,
      "memory(GiB)": 75.26,
      "reward": 1.806189775466919,
      "reward_std": 0.47400711476802826,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.28863146156072617,
      "rewards/ReflectionBalance/mean": 0.794471025466919,
      "rewards/ReflectionBalance/std": 0.3059573471546173,
      "step": 1172,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 501.46875,
      "completions/min_length": 249.0,
      "epoch": 6.339622641509434,
      "grad_norm": 2.7787543549743057,
      "kl": 2.1875,
      "learning_rate": 3.010560730693444e-07,
      "loss": 0.17024913430213928,
      "memory(GiB)": 75.26,
      "reward": 1.9446580410003662,
      "reward_std": 0.4946952760219574,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.4164551943540573,
      "rewards/ReflectionBalance/mean": 0.8001267910003662,
      "rewards/ReflectionBalance/std": 0.2867549881339073,
      "step": 1173,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 473.79296875,
      "completions/min_length": 247.5,
      "epoch": 6.345013477088949,
      "grad_norm": 3.2597600677212735,
      "kl": 2.65625,
      "learning_rate": 3.0026930891649617e-07,
      "loss": 0.23115716874599457,
      "memory(GiB)": 75.26,
      "reward": 1.7836002111434937,
      "reward_std": 0.5000216215848923,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3037657365202904,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.31232215464115143,
      "rewards/ReflectionBalance/mean": 0.783600240945816,
      "rewards/ReflectionBalance/std": 0.30689215660095215,
      "step": 1174,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 480.16015625,
      "completions/min_length": 173.5,
      "epoch": 6.350404312668464,
      "grad_norm": 3.3549455077370403,
      "kl": 1.765625,
      "learning_rate": 2.994831327506896e-07,
      "loss": 0.14525115489959717,
      "memory(GiB)": 75.26,
      "reward": 1.9223681688308716,
      "reward_std": 0.4597964882850647,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3978758305311203,
      "rewards/ReflectionBalance/mean": 0.8012744188308716,
      "rewards/ReflectionBalance/std": 0.25014297664165497,
      "step": 1175,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 484.7109375,
      "completions/min_length": 231.0,
      "epoch": 6.355795148247978,
      "grad_norm": 4.620352408935737,
      "kl": 2.140625,
      "learning_rate": 2.9869754688634823e-07,
      "loss": 0.204207181930542,
      "memory(GiB)": 75.26,
      "reward": 1.892117440700531,
      "reward_std": 0.46233758330345154,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3681531548500061,
      "rewards/ReflectionBalance/mean": 0.8178987503051758,
      "rewards/ReflectionBalance/std": 0.2274656817317009,
      "step": 1176,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 487.953125,
      "completions/min_length": 249.0,
      "epoch": 6.361185983827493,
      "grad_norm": 3.0334715221504505,
      "kl": 2.2109375,
      "learning_rate": 2.979125536361574e-07,
      "loss": 0.18104757368564606,
      "memory(GiB)": 75.26,
      "reward": 1.8087548613548279,
      "reward_std": 0.4882551282644272,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3191515803337097,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3437131494283676,
      "rewards/ReflectionBalance/mean": 0.7892236411571503,
      "rewards/ReflectionBalance/std": 0.2731597051024437,
      "step": 1177,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 457.25,
      "completions/min_length": 276.5,
      "epoch": 6.366576819407008,
      "grad_norm": 8.16022839962883,
      "kl": 2.7109375,
      "learning_rate": 2.971281553110583e-07,
      "loss": 0.21041959524154663,
      "memory(GiB)": 75.26,
      "reward": 1.9571533203125,
      "reward_std": 0.5841873288154602,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4468510299921036,
      "rewards/ReflectionBalance/mean": 0.7852783203125,
      "rewards/ReflectionBalance/std": 0.29574698209762573,
      "step": 1178,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 471.87890625,
      "completions/min_length": 252.5,
      "epoch": 6.3719676549865225,
      "grad_norm": 4.543334184062066,
      "kl": 3.015625,
      "learning_rate": 2.96344354220241e-07,
      "loss": 0.19899529218673706,
      "memory(GiB)": 75.26,
      "reward": 1.8825068473815918,
      "reward_std": 0.4391377568244934,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30283892154693604,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38864077627658844,
      "rewards/ReflectionBalance/mean": 0.8043818473815918,
      "rewards/ReflectionBalance/std": 0.22565043717622757,
      "step": 1179,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 505.83203125,
      "completions/min_length": 278.0,
      "epoch": 6.377358490566038,
      "grad_norm": 2.9700535505094523,
      "kl": 2.8125,
      "learning_rate": 2.955611526711365e-07,
      "loss": 0.2218213677406311,
      "memory(GiB)": 75.26,
      "reward": 1.964120090007782,
      "reward_std": 0.5581756830215454,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.4301380217075348,
      "rewards/ReflectionBalance/mean": 0.811776340007782,
      "rewards/ReflectionBalance/std": 0.24601581692695618,
      "step": 1180,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 491.30078125,
      "completions/min_length": 241.0,
      "epoch": 6.382749326145553,
      "grad_norm": 4.234815635734613,
      "kl": 2.96875,
      "learning_rate": 2.947785529694114e-07,
      "loss": 0.21456864476203918,
      "memory(GiB)": 75.26,
      "reward": 1.7878825068473816,
      "reward_std": 0.4829743355512619,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3407054841518402,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.32746291160583496,
      "rewards/ReflectionBalance/mean": 0.7996012270450592,
      "rewards/ReflectionBalance/std": 0.27327869832515717,
      "step": 1181,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 511.6171875,
      "completions/min_length": 268.5,
      "epoch": 6.388140161725067,
      "grad_norm": 2.945007146121384,
      "kl": 3.328125,
      "learning_rate": 2.939965574189606e-07,
      "loss": 0.2665247619152069,
      "memory(GiB)": 75.26,
      "reward": 1.8654223084449768,
      "reward_std": 0.5065838396549225,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3208130896091461,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3853980004787445,
      "rewards/ReflectionBalance/mean": 0.7951098382472992,
      "rewards/ReflectionBalance/std": 0.2476903349161148,
      "step": 1182,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 498.890625,
      "completions/min_length": 285.5,
      "epoch": 6.393530997304582,
      "grad_norm": 3.6301446953323895,
      "kl": 2.22265625,
      "learning_rate": 2.9321516832190007e-07,
      "loss": 0.20162253081798553,
      "memory(GiB)": 75.26,
      "reward": 2.000576138496399,
      "reward_std": 0.3787813186645508,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41699807345867157,
      "rewards/ReflectionBalance/mean": 0.8482324481010437,
      "rewards/ReflectionBalance/std": 0.1935228407382965,
      "step": 1183,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 501.32421875,
      "completions/min_length": 260.0,
      "epoch": 6.398921832884097,
      "grad_norm": 2.948925892906089,
      "kl": 2.21875,
      "learning_rate": 2.9243438797856066e-07,
      "loss": 0.18606147170066833,
      "memory(GiB)": 75.26,
      "reward": 1.832421898841858,
      "reward_std": 0.48435285687446594,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2875154986977577,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.3338889628648758,
      "rewards/ReflectionBalance/mean": 0.7972655892372131,
      "rewards/ReflectionBalance/std": 0.2765679210424423,
      "step": 1184,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 481.76171875,
      "completions/min_length": 279.0,
      "epoch": 6.404312668463612,
      "grad_norm": 2.96288166908848,
      "kl": 2.4765625,
      "learning_rate": 2.916542186874812e-07,
      "loss": 0.20609062910079956,
      "memory(GiB)": 75.26,
      "reward": 1.857882559299469,
      "reward_std": 0.4731747955083847,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.27175262570381165,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.33660952746868134,
      "rewards/ReflectionBalance/mean": 0.8110074698925018,
      "rewards/ReflectionBalance/std": 0.23454415053129196,
      "step": 1185,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 491.16015625,
      "completions/min_length": 199.0,
      "epoch": 6.409703504043127,
      "grad_norm": 3.6965027610906698,
      "kl": 2.03125,
      "learning_rate": 2.9087466274540147e-07,
      "loss": 0.16061896085739136,
      "memory(GiB)": 75.26,
      "reward": 1.916696012020111,
      "reward_std": 0.4929758906364441,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26276856660842896,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38047872483730316,
      "rewards/ReflectionBalance/mean": 0.8151334822177887,
      "rewards/ReflectionBalance/std": 0.2336738556623459,
      "step": 1186,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 480.5078125,
      "completions/min_length": 294.0,
      "epoch": 6.415094339622642,
      "grad_norm": 3.012659777997685,
      "kl": 1.703125,
      "learning_rate": 2.9009572244725534e-07,
      "loss": 0.14942288398742676,
      "memory(GiB)": 75.26,
      "reward": 2.0470606088638306,
      "reward_std": 0.41650938987731934,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.4418751001358032,
      "rewards/ReflectionBalance/mean": 0.8244042992591858,
      "rewards/ReflectionBalance/std": 0.2186703458428383,
      "step": 1187,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 493.47265625,
      "completions/min_length": 44.5,
      "epoch": 6.420485175202156,
      "grad_norm": 3.658067978942291,
      "kl": 2.203125,
      "learning_rate": 2.893174000861653e-07,
      "loss": 0.1237621009349823,
      "memory(GiB)": 75.26,
      "reward": 1.8956518769264221,
      "reward_std": 0.4553389549255371,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.37438808381557465,
      "rewards/ReflectionBalance/mean": 0.8214330673217773,
      "rewards/ReflectionBalance/std": 0.25564803183078766,
      "step": 1188,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 502.20703125,
      "completions/min_length": 265.5,
      "epoch": 6.425876010781671,
      "grad_norm": 2.818674635009765,
      "kl": 2.38671875,
      "learning_rate": 2.8853969795343323e-07,
      "loss": 0.20707815885543823,
      "memory(GiB)": 75.26,
      "reward": 1.8922005891799927,
      "reward_std": 0.5120502263307571,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3270204961299896,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.3893345892429352,
      "rewards/ReflectionBalance/mean": 0.8023567795753479,
      "rewards/ReflectionBalance/std": 0.25628994405269623,
      "step": 1189,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 484.77734375,
      "completions/min_length": 252.0,
      "epoch": 6.431266846361186,
      "grad_norm": 3.426426337465868,
      "kl": 2.59375,
      "learning_rate": 2.877626183385364e-07,
      "loss": 0.23088182508945465,
      "memory(GiB)": 75.26,
      "reward": 1.865821123123169,
      "reward_std": 0.4604959189891815,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3597298264503479,
      "rewards/ReflectionBalance/mean": 0.8150398433208466,
      "rewards/ReflectionBalance/std": 0.23798967897891998,
      "step": 1190,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 492.84375,
      "completions/min_length": 261.0,
      "epoch": 6.436657681940701,
      "grad_norm": 2.6912077466374047,
      "kl": 2.5,
      "learning_rate": 2.869861635291189e-07,
      "loss": 0.21274496614933014,
      "memory(GiB)": 75.26,
      "reward": 1.8970263600349426,
      "reward_std": 0.4757683277130127,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3778642416000366,
      "rewards/ReflectionBalance/mean": 0.807182639837265,
      "rewards/ReflectionBalance/std": 0.24832796305418015,
      "step": 1191,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 464.80859375,
      "completions/min_length": 140.5,
      "epoch": 6.442048517520216,
      "grad_norm": 2.7661345787273635,
      "kl": 2.09375,
      "learning_rate": 2.862103358109851e-07,
      "loss": 0.18948501348495483,
      "memory(GiB)": 75.26,
      "reward": 2.108344793319702,
      "reward_std": 0.4137064814567566,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.33203125,
      "rewards/MultiModalAccuracyORM/std": 0.4661055654287338,
      "rewards/ReflectionBalance/mean": 0.8310009837150574,
      "rewards/ReflectionBalance/std": 0.21484308689832687,
      "step": 1192,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 502.7265625,
      "completions/min_length": 281.0,
      "epoch": 6.44743935309973,
      "grad_norm": 2.9100340770515,
      "kl": 1.9765625,
      "learning_rate": 2.854351374680944e-07,
      "loss": 0.15882934629917145,
      "memory(GiB)": 75.26,
      "reward": 1.8487500548362732,
      "reward_std": 0.4457993805408478,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.33513282239437103,
      "rewards/ReflectionBalance/mean": 0.7979687750339508,
      "rewards/ReflectionBalance/std": 0.2507240101695061,
      "step": 1193,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 499.1171875,
      "completions/min_length": 294.0,
      "epoch": 6.452830188679245,
      "grad_norm": 7.896618020546631,
      "kl": 1.4453125,
      "learning_rate": 2.8466057078255184e-07,
      "loss": 0.122596874833107,
      "memory(GiB)": 75.26,
      "reward": 2.108574151992798,
      "reward_std": 0.3893255591392517,
      "rewards/Format/mean": 0.96875,
      "rewards/Format/std": 0.1746762990951538,
      "rewards/MultiModalAccuracyORM/mean": 0.29296875,
      "rewards/MultiModalAccuracyORM/std": 0.45640426874160767,
      "rewards/ReflectionBalance/mean": 0.8468554615974426,
      "rewards/ReflectionBalance/std": 0.19597209990024567,
      "step": 1194,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 462.671875,
      "completions/min_length": 240.5,
      "epoch": 6.45822102425876,
      "grad_norm": 2.884260027549306,
      "kl": 1.72265625,
      "learning_rate": 2.8388663803460433e-07,
      "loss": 0.1439625322818756,
      "memory(GiB)": 75.26,
      "reward": 2.0874284505844116,
      "reward_std": 0.40746621787548065,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21220162510871887,
      "rewards/MultiModalAccuracyORM/mean": 0.296875,
      "rewards/MultiModalAccuracyORM/std": 0.4468565881252289,
      "rewards/ReflectionBalance/mean": 0.8374283909797668,
      "rewards/ReflectionBalance/std": 0.1920241117477417,
      "step": 1195,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 491.4921875,
      "completions/min_length": 269.5,
      "epoch": 6.463611859838275,
      "grad_norm": 2.9821785843779947,
      "kl": 2.5,
      "learning_rate": 2.831133415026318e-07,
      "loss": 0.21613699197769165,
      "memory(GiB)": 75.26,
      "reward": 1.9709489345550537,
      "reward_std": 0.45543818175792694,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4152318239212036,
      "rewards/ReflectionBalance/mean": 0.8303238749504089,
      "rewards/ReflectionBalance/std": 0.23469685018062592,
      "step": 1196,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 506.02734375,
      "completions/min_length": 253.0,
      "epoch": 6.46900269541779,
      "grad_norm": 3.44238151229468,
      "kl": 3.0546875,
      "learning_rate": 2.82340683463141e-07,
      "loss": 0.26109012961387634,
      "memory(GiB)": 75.26,
      "reward": 1.9438720345497131,
      "reward_std": 0.509874239563942,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.4154975265264511,
      "rewards/ReflectionBalance/mean": 0.7993408143520355,
      "rewards/ReflectionBalance/std": 0.2755846902728081,
      "step": 1197,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 485.8203125,
      "completions/min_length": 270.5,
      "epoch": 6.474393530997305,
      "grad_norm": 3.173393970940472,
      "kl": 2.6640625,
      "learning_rate": 2.8156866619075995e-07,
      "loss": 0.2337820678949356,
      "memory(GiB)": 75.26,
      "reward": 1.848853349685669,
      "reward_std": 0.4172687977552414,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.332206666469574,
      "rewards/ReflectionBalance/mean": 0.8176033794879913,
      "rewards/ReflectionBalance/std": 0.2350112721323967,
      "step": 1198,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 483.4453125,
      "completions/min_length": 287.5,
      "epoch": 6.479784366576819,
      "grad_norm": 2.883177459733998,
      "kl": 2.8671875,
      "learning_rate": 2.807972919582295e-07,
      "loss": 0.24111194908618927,
      "memory(GiB)": 75.26,
      "reward": 1.909072995185852,
      "reward_std": 0.5429404377937317,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.41230008006095886,
      "rewards/ReflectionBalance/mean": 0.7957918345928192,
      "rewards/ReflectionBalance/std": 0.2618979290127754,
      "step": 1199,
      "train_speed(iter/s)": 0.007118
    },
    {
      "epoch": 6.485175202156334,
      "grad_norm": 4.444670472593089,
      "learning_rate": 2.800265630363978e-07,
      "loss": 0.19788981974124908,
      "memory(GiB)": 75.26,
      "step": 1200,
      "train_speed(iter/s)": 0.007119
    },
    {
      "epoch": 6.485175202156334,
      "eval_clip_ratio": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 1026.75,
      "eval_completions/mean_length": 526.1171875,
      "eval_completions/min_length": 271.75,
      "eval_kl": 2.66796875,
      "eval_loss": 0.21480923891067505,
      "eval_reward": 1.8610375821590424,
      "eval_reward_std": 0.45071814954280853,
      "eval_rewards/Format/mean": 0.900390625,
      "eval_rewards/Format/std": 0.29971204698085785,
      "eval_rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "eval_rewards/MultiModalAccuracyORM/std": 0.34485070034861565,
      "eval_rewards/ReflectionBalance/mean": 0.8122094869613647,
      "eval_rewards/ReflectionBalance/std": 0.27069971337914467,
      "eval_runtime": 248.5669,
      "eval_samples_per_second": 0.237,
      "eval_steps_per_second": 0.004,
      "step": 1200
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.25,
      "completions/mean_length": 495.61328125,
      "completions/min_length": 259.75,
      "epoch": 6.490566037735849,
      "grad_norm": 2.9933593060362256,
      "kl": 2.6328125,
      "learning_rate": 2.7925648169421287e-07,
      "loss": 0.2405872344970703,
      "memory(GiB)": 75.26,
      "reward": 1.9153401553630829,
      "reward_std": 0.43768205493688583,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.27940889447927475,
      "rewards/MultiModalAccuracyORM/mean": 0.181640625,
      "rewards/MultiModalAccuracyORM/std": 0.36445289850234985,
      "rewards/ReflectionBalance/mean": 0.8196370452642441,
      "rewards/ReflectionBalance/std": 0.22936009615659714,
      "step": 1201,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 515.72265625,
      "completions/min_length": 193.0,
      "epoch": 6.495956873315364,
      "grad_norm": 3.37054565243638,
      "kl": 3.7734375,
      "learning_rate": 2.784870501987171e-07,
      "loss": 0.31367555260658264,
      "memory(GiB)": 75.26,
      "reward": 1.8323885202407837,
      "reward_std": 0.5083427131175995,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.344347283244133,
      "rewards/ReflectionBalance/mean": 0.8050447404384613,
      "rewards/ReflectionBalance/std": 0.26867368817329407,
      "step": 1202,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 506.3984375,
      "completions/min_length": 259.0,
      "epoch": 6.501347708894879,
      "grad_norm": 3.7762767033864963,
      "kl": 2.4140625,
      "learning_rate": 2.777182708150391e-07,
      "loss": 0.19688180088996887,
      "memory(GiB)": 75.26,
      "reward": 2.04755300283432,
      "reward_std": 0.5435801148414612,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.3359375,
      "rewards/MultiModalAccuracyORM/std": 0.46682190895080566,
      "rewards/ReflectionBalance/mean": 0.8131778836250305,
      "rewards/ReflectionBalance/std": 0.28200705349445343,
      "step": 1203,
      "train_speed(iter/s)": 0.007105
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 491.2578125,
      "completions/min_length": 258.0,
      "epoch": 6.506738544474393,
      "grad_norm": 4.721176563642066,
      "kl": 3.953125,
      "learning_rate": 2.7695014580638775e-07,
      "loss": 0.3044852912425995,
      "memory(GiB)": 75.26,
      "reward": 1.8158349394798279,
      "reward_std": 0.5900527238845825,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.330063596367836,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36417925357818604,
      "rewards/ReflectionBalance/mean": 0.7806787192821503,
      "rewards/ReflectionBalance/std": 0.26063496619462967,
      "step": 1204,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 514.0546875,
      "completions/min_length": 246.5,
      "epoch": 6.512129380053908,
      "grad_norm": 3.5817974757313324,
      "kl": 3.28125,
      "learning_rate": 2.761826774340463e-07,
      "loss": 0.24681103229522705,
      "memory(GiB)": 75.26,
      "reward": 1.9128206968307495,
      "reward_std": 0.5573684424161911,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.33569003641605377,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.43674568831920624,
      "rewards/ReflectionBalance/mean": 0.7917268872261047,
      "rewards/ReflectionBalance/std": 0.29262496531009674,
      "step": 1205,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 475.50390625,
      "completions/min_length": 274.0,
      "epoch": 6.517520215633423,
      "grad_norm": 4.950148378257762,
      "kl": 2.4140625,
      "learning_rate": 2.7541586795736365e-07,
      "loss": 0.20972663164138794,
      "memory(GiB)": 75.26,
      "reward": 1.93660306930542,
      "reward_std": 0.42017558217048645,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.382988765835762,
      "rewards/ReflectionBalance/mean": 0.8272281587123871,
      "rewards/ReflectionBalance/std": 0.2153693363070488,
      "step": 1206,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 488.2421875,
      "completions/min_length": 223.5,
      "epoch": 6.5229110512129385,
      "grad_norm": 3.0081526752307237,
      "kl": 2.953125,
      "learning_rate": 2.7464971963375024e-07,
      "loss": 0.24432432651519775,
      "memory(GiB)": 75.26,
      "reward": 1.8927929997444153,
      "reward_std": 0.5884075462818146,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4067341536283493,
      "rewards/ReflectionBalance/mean": 0.7834179401397705,
      "rewards/ReflectionBalance/std": 0.2950349897146225,
      "step": 1207,
      "train_speed(iter/s)": 0.007106
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 516.31640625,
      "completions/min_length": 277.0,
      "epoch": 6.528301886792453,
      "grad_norm": 2.873227250103773,
      "kl": 2.16796875,
      "learning_rate": 2.738842347186693e-07,
      "loss": 0.1718202531337738,
      "memory(GiB)": 75.26,
      "reward": 1.8685709238052368,
      "reward_std": 0.4525923728942871,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.26755452156066895,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3508307933807373,
      "rewards/ReflectionBalance/mean": 0.8060709238052368,
      "rewards/ReflectionBalance/std": 0.26726557314395905,
      "step": 1208,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 480.19921875,
      "completions/min_length": 275.5,
      "epoch": 6.533692722371968,
      "grad_norm": 3.0216763792794676,
      "kl": 2.6328125,
      "learning_rate": 2.731194154656312e-07,
      "loss": 0.2271796464920044,
      "memory(GiB)": 75.26,
      "reward": 1.82848060131073,
      "reward_std": 0.4914552569389343,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.3219822645187378,
      "rewards/ReflectionBalance/mean": 0.8050431609153748,
      "rewards/ReflectionBalance/std": 0.24531108140945435,
      "step": 1209,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 487.80078125,
      "completions/min_length": 166.0,
      "epoch": 6.539083557951482,
      "grad_norm": 4.5012981604466145,
      "kl": 2.1953125,
      "learning_rate": 2.723552641261873e-07,
      "loss": 0.1769721657037735,
      "memory(GiB)": 75.26,
      "reward": 1.884534478187561,
      "reward_std": 0.5361996293067932,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3586205691099167,
      "rewards/ReflectionBalance/mean": 0.8142220079898834,
      "rewards/ReflectionBalance/std": 0.2796531468629837,
      "step": 1210,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 497.12109375,
      "completions/min_length": 259.0,
      "epoch": 6.544474393530997,
      "grad_norm": 3.04903537362066,
      "kl": 2.9140625,
      "learning_rate": 2.7159178294992185e-07,
      "loss": 0.25411859154701233,
      "memory(GiB)": 75.26,
      "reward": 1.9240837693214417,
      "reward_std": 0.5182321071624756,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.39185720682144165,
      "rewards/ReflectionBalance/mean": 0.8068962097167969,
      "rewards/ReflectionBalance/std": 0.2648835554718971,
      "step": 1211,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 474.6171875,
      "completions/min_length": 239.0,
      "epoch": 6.549865229110512,
      "grad_norm": 3.5948626852809666,
      "kl": 3.0546875,
      "learning_rate": 2.708289741844464e-07,
      "loss": 0.23139631748199463,
      "memory(GiB)": 75.26,
      "reward": 1.8317660093307495,
      "reward_std": 0.5714785754680634,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33595384657382965,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3823556900024414,
      "rewards/ReflectionBalance/mean": 0.7653596699237823,
      "rewards/ReflectionBalance/std": 0.33145031332969666,
      "step": 1212,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 506.2421875,
      "completions/min_length": 245.0,
      "epoch": 6.555256064690027,
      "grad_norm": 3.878036160729327,
      "kl": 3.984375,
      "learning_rate": 2.700668400753936e-07,
      "loss": 0.3007262647151947,
      "memory(GiB)": 75.26,
      "reward": 1.8963883519172668,
      "reward_std": 0.5886881500482559,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.30921147763729095,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.4149145632982254,
      "rewards/ReflectionBalance/mean": 0.7870133519172668,
      "rewards/ReflectionBalance/std": 0.25606391578912735,
      "step": 1213,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 495.71875,
      "completions/min_length": 192.5,
      "epoch": 6.560646900269542,
      "grad_norm": 5.202111453838227,
      "kl": 3.921875,
      "learning_rate": 2.6930538286640914e-07,
      "loss": 0.2752750515937805,
      "memory(GiB)": 75.26,
      "reward": 1.7978577017784119,
      "reward_std": 0.5423589646816254,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.33968591690063477,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3538244515657425,
      "rewards/ReflectionBalance/mean": 0.7822327315807343,
      "rewards/ReflectionBalance/std": 0.26241758465766907,
      "step": 1214,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 500.34375,
      "completions/min_length": 281.0,
      "epoch": 6.566037735849057,
      "grad_norm": 3.1590510943760037,
      "kl": 2.40234375,
      "learning_rate": 2.685446047991466e-07,
      "loss": 0.20948931574821472,
      "memory(GiB)": 75.26,
      "reward": 1.9471744298934937,
      "reward_std": 0.5259094089269638,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41502299904823303,
      "rewards/ReflectionBalance/mean": 0.829986959695816,
      "rewards/ReflectionBalance/std": 0.22386865317821503,
      "step": 1215,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 496.32421875,
      "completions/min_length": 288.5,
      "epoch": 6.571428571428571,
      "grad_norm": 4.103675925335907,
      "kl": 3.0703125,
      "learning_rate": 2.677845081132596e-07,
      "loss": 0.23201264441013336,
      "memory(GiB)": 75.26,
      "reward": 1.9331478476524353,
      "reward_std": 0.47603365778923035,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3169707804918289,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42852601408958435,
      "rewards/ReflectionBalance/mean": 0.8081478178501129,
      "rewards/ReflectionBalance/std": 0.25424348562955856,
      "step": 1216,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 459.578125,
      "completions/min_length": 267.5,
      "epoch": 6.576819407008086,
      "grad_norm": 2.743231981348441,
      "kl": 1.78125,
      "learning_rate": 2.670250950463968e-07,
      "loss": 0.1301954835653305,
      "memory(GiB)": 75.26,
      "reward": 1.9925683736801147,
      "reward_std": 0.36687037348747253,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23196382820606232,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.38770951330661774,
      "rewards/ReflectionBalance/mean": 0.84413081407547,
      "rewards/ReflectionBalance/std": 0.18747256696224213,
      "step": 1217,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 483.15625,
      "completions/min_length": 241.0,
      "epoch": 6.5822102425876015,
      "grad_norm": 3.2176071532549892,
      "kl": 2.09375,
      "learning_rate": 2.662663678341935e-07,
      "loss": 0.16290514171123505,
      "memory(GiB)": 75.26,
      "reward": 1.9170231819152832,
      "reward_std": 0.45351310074329376,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.39205317199230194,
      "rewards/ReflectionBalance/mean": 0.8076481223106384,
      "rewards/ReflectionBalance/std": 0.25089050084352493,
      "step": 1218,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 984.0,
      "completions/mean_length": 480.92578125,
      "completions/min_length": 255.5,
      "epoch": 6.587601078167116,
      "grad_norm": 2.9501257863396577,
      "kl": 2.08203125,
      "learning_rate": 2.6550832871026616e-07,
      "loss": 0.17347575724124908,
      "memory(GiB)": 75.26,
      "reward": 1.9322379231452942,
      "reward_std": 0.4906030297279358,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39110109210014343,
      "rewards/ReflectionBalance/mean": 0.8228629529476166,
      "rewards/ReflectionBalance/std": 0.22330810874700546,
      "step": 1219,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 487.8203125,
      "completions/min_length": 299.0,
      "epoch": 6.592991913746631,
      "grad_norm": 3.358002567419166,
      "kl": 2.8671875,
      "learning_rate": 2.6475097990620576e-07,
      "loss": 0.2697169780731201,
      "memory(GiB)": 75.26,
      "reward": 1.8854060769081116,
      "reward_std": 0.5674784481525421,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.3131764084100723,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4067341536283493,
      "rewards/ReflectionBalance/mean": 0.787749856710434,
      "rewards/ReflectionBalance/std": 0.2890332341194153,
      "step": 1220,
      "train_speed(iter/s)": 0.007107
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 494.51171875,
      "completions/min_length": 238.0,
      "epoch": 6.598382749326145,
      "grad_norm": 2.960081141091214,
      "kl": 1.8046875,
      "learning_rate": 2.6399432365157065e-07,
      "loss": 0.12883850932121277,
      "memory(GiB)": 75.26,
      "reward": 1.96866774559021,
      "reward_std": 0.4466911256313324,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4270860105752945,
      "rewards/ReflectionBalance/mean": 0.8202303051948547,
      "rewards/ReflectionBalance/std": 0.20113708078861237,
      "step": 1221,
      "train_speed(iter/s)": 0.007108
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 496.515625,
      "completions/min_length": 180.0,
      "epoch": 6.60377358490566,
      "grad_norm": 2.7990422243179074,
      "kl": 3.046875,
      "learning_rate": 2.6323836217388105e-07,
      "loss": 0.2374088019132614,
      "memory(GiB)": 75.26,
      "reward": 1.83637535572052,
      "reward_std": 0.49744945764541626,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.32164282351732254,
      "rewards/ReflectionBalance/mean": 0.8090315759181976,
      "rewards/ReflectionBalance/std": 0.27309055626392365,
      "step": 1222,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 507.10546875,
      "completions/min_length": 264.0,
      "epoch": 6.609164420485175,
      "grad_norm": 3.243794556435187,
      "kl": 2.3359375,
      "learning_rate": 2.624830976986112e-07,
      "loss": 0.19238466024398804,
      "memory(GiB)": 75.26,
      "reward": 1.9325676560401917,
      "reward_std": 0.4419337660074234,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.4007115960121155,
      "rewards/ReflectionBalance/mean": 0.8231925368309021,
      "rewards/ReflectionBalance/std": 0.22345446795225143,
      "step": 1223,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 512.1015625,
      "completions/min_length": 284.0,
      "epoch": 6.6145552560646905,
      "grad_norm": 3.1322484043551055,
      "kl": 2.7265625,
      "learning_rate": 2.6172853244918345e-07,
      "loss": 0.2229379266500473,
      "memory(GiB)": 75.26,
      "reward": 1.9017366170883179,
      "reward_std": 0.5402640700340271,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3219822645187378,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.3979068696498871,
      "rewards/ReflectionBalance/mean": 0.8079866766929626,
      "rewards/ReflectionBalance/std": 0.2392750009894371,
      "step": 1224,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 485.53515625,
      "completions/min_length": 261.0,
      "epoch": 6.619946091644205,
      "grad_norm": 7.037276895610793,
      "kl": 2.5703125,
      "learning_rate": 2.609746686469626e-07,
      "loss": 0.2299671173095703,
      "memory(GiB)": 75.26,
      "reward": 1.9609586596488953,
      "reward_std": 0.49395349621772766,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4041530042886734,
      "rewards/ReflectionBalance/mean": 0.8242399096488953,
      "rewards/ReflectionBalance/std": 0.2387576550245285,
      "step": 1225,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 491.62890625,
      "completions/min_length": 285.5,
      "epoch": 6.62533692722372,
      "grad_norm": 3.0118019550461703,
      "kl": 2.0234375,
      "learning_rate": 2.602215085112469e-07,
      "loss": 0.16244468092918396,
      "memory(GiB)": 75.26,
      "reward": 1.9196256399154663,
      "reward_std": 0.40187668800354004,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3556671589612961,
      "rewards/ReflectionBalance/mean": 0.8454069197177887,
      "rewards/ReflectionBalance/std": 0.18701984733343124,
      "step": 1226,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 474.63671875,
      "completions/min_length": 247.0,
      "epoch": 6.630727762803234,
      "grad_norm": 2.8796597291585537,
      "kl": 1.9921875,
      "learning_rate": 2.594690542592646e-07,
      "loss": 0.1847163587808609,
      "memory(GiB)": 75.26,
      "reward": 1.9335750341415405,
      "reward_std": 0.3625767230987549,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3613582104444504,
      "rewards/ReflectionBalance/mean": 0.8359187841415405,
      "rewards/ReflectionBalance/std": 0.2117074653506279,
      "step": 1227,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 501.15234375,
      "completions/min_length": 285.0,
      "epoch": 6.636118598382749,
      "grad_norm": 2.73426203991772,
      "kl": 2.59375,
      "learning_rate": 2.58717308106165e-07,
      "loss": 0.22580870985984802,
      "memory(GiB)": 75.26,
      "reward": 2.009031593799591,
      "reward_std": 0.4291585683822632,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4416199326515198,
      "rewards/ReflectionBalance/mean": 0.8215316236019135,
      "rewards/ReflectionBalance/std": 0.24175578355789185,
      "step": 1228,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 500.96875,
      "completions/min_length": 276.5,
      "epoch": 6.6415094339622645,
      "grad_norm": 4.044959369327926,
      "kl": 2.9921875,
      "learning_rate": 2.579662722650128e-07,
      "loss": 0.22720777988433838,
      "memory(GiB)": 75.26,
      "reward": 1.8889778852462769,
      "reward_std": 0.5323039591312408,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39110109210014343,
      "rewards/ReflectionBalance/mean": 0.7952278554439545,
      "rewards/ReflectionBalance/std": 0.27343258261680603,
      "step": 1229,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 486.34375,
      "completions/min_length": 274.0,
      "epoch": 6.646900269541779,
      "grad_norm": 3.4828145231682472,
      "kl": 2.859375,
      "learning_rate": 2.572159489467824e-07,
      "loss": 0.23835158348083496,
      "memory(GiB)": 75.26,
      "reward": 1.8259081840515137,
      "reward_std": 0.54798424243927,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31333550810813904,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.34734736382961273,
      "rewards/ReflectionBalance/mean": 0.7790332138538361,
      "rewards/ReflectionBalance/std": 0.3007567226886749,
      "step": 1230,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 502.19140625,
      "completions/min_length": 263.0,
      "epoch": 6.652291105121294,
      "grad_norm": 2.9075677655745342,
      "kl": 2.8359375,
      "learning_rate": 2.5646634036034987e-07,
      "loss": 0.23972319066524506,
      "memory(GiB)": 75.26,
      "reward": 1.9692594408988953,
      "reward_std": 0.4295571744441986,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4202718883752823,
      "rewards/ReflectionBalance/mean": 0.8325406908988953,
      "rewards/ReflectionBalance/std": 0.2262963056564331,
      "step": 1231,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 498.75390625,
      "completions/min_length": 256.5,
      "epoch": 6.657681940700809,
      "grad_norm": 3.119650191104673,
      "kl": 2.9296875,
      "learning_rate": 2.5571744871248735e-07,
      "loss": 0.258363276720047,
      "memory(GiB)": 75.26,
      "reward": 1.9339436888694763,
      "reward_std": 0.524832010269165,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.4299930930137634,
      "rewards/ReflectionBalance/mean": 0.7933186888694763,
      "rewards/ReflectionBalance/std": 0.25189872831106186,
      "step": 1232,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 485.09375,
      "completions/min_length": 165.5,
      "epoch": 6.663072776280323,
      "grad_norm": 3.1188745921117254,
      "kl": 1.76953125,
      "learning_rate": 2.5496927620785625e-07,
      "loss": 0.1455971598625183,
      "memory(GiB)": 75.26,
      "reward": 1.975333571434021,
      "reward_std": 0.43105967342853546,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.40284374356269836,
      "rewards/ReflectionBalance/mean": 0.8386149108409882,
      "rewards/ReflectionBalance/std": 0.20902227610349655,
      "step": 1233,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 485.8828125,
      "completions/min_length": 302.0,
      "epoch": 6.668463611859838,
      "grad_norm": 4.126984786693167,
      "kl": 2.9296875,
      "learning_rate": 2.5422182504900156e-07,
      "loss": 0.24757137894630432,
      "memory(GiB)": 75.26,
      "reward": 1.9456217288970947,
      "reward_std": 0.4835495352745056,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4187151938676834,
      "rewards/ReflectionBalance/mean": 0.8128092586994171,
      "rewards/ReflectionBalance/std": 0.23786961287260056,
      "step": 1234,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 464.49609375,
      "completions/min_length": 263.5,
      "epoch": 6.6738544474393535,
      "grad_norm": 3.0694021675364502,
      "kl": 2.7109375,
      "learning_rate": 2.53475097436344e-07,
      "loss": 0.2450944483280182,
      "memory(GiB)": 75.26,
      "reward": 1.936650276184082,
      "reward_std": 0.5149484872817993,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.40794737637043,
      "rewards/ReflectionBalance/mean": 0.8233691453933716,
      "rewards/ReflectionBalance/std": 0.22952697426080704,
      "step": 1235,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 495.01171875,
      "completions/min_length": 275.0,
      "epoch": 6.679245283018868,
      "grad_norm": 3.4096742292708466,
      "kl": 2.18359375,
      "learning_rate": 2.527290955681742e-07,
      "loss": 0.19682788848876953,
      "memory(GiB)": 75.26,
      "reward": 1.9099723100662231,
      "reward_std": 0.5155666768550873,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.27558503299951553,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.37514132261276245,
      "rewards/ReflectionBalance/mean": 0.8240348696708679,
      "rewards/ReflectionBalance/std": 0.24878494441509247,
      "step": 1236,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 477.578125,
      "completions/min_length": 245.0,
      "epoch": 6.684636118598383,
      "grad_norm": 4.281152606559403,
      "kl": 2.3203125,
      "learning_rate": 2.519838216406471e-07,
      "loss": 0.19583973288536072,
      "memory(GiB)": 75.26,
      "reward": 1.8992341756820679,
      "reward_std": 0.5195949375629425,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38325364887714386,
      "rewards/ReflectionBalance/mean": 0.8093904554843903,
      "rewards/ReflectionBalance/std": 0.24803907424211502,
      "step": 1237,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 477.08203125,
      "completions/min_length": 242.5,
      "epoch": 6.690026954177897,
      "grad_norm": 3.724966954764858,
      "kl": 3.9453125,
      "learning_rate": 2.512392778477735e-07,
      "loss": 0.33171242475509644,
      "memory(GiB)": 75.26,
      "reward": 1.8375406861305237,
      "reward_std": 0.5560036301612854,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3219822645187378,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38201163709163666,
      "rewards/ReflectionBalance/mean": 0.7750407159328461,
      "rewards/ReflectionBalance/std": 0.2699516713619232,
      "step": 1238,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 519.1328125,
      "completions/min_length": 270.5,
      "epoch": 6.695417789757412,
      "grad_norm": 2.7947931402204604,
      "kl": 3.90625,
      "learning_rate": 2.504954663814155e-07,
      "loss": 0.32316333055496216,
      "memory(GiB)": 75.26,
      "reward": 1.8888159394264221,
      "reward_std": 0.5967477262020111,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3490002751350403,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.42468734085559845,
      "rewards/ReflectionBalance/mean": 0.7911597192287445,
      "rewards/ReflectionBalance/std": 0.2863287776708603,
      "step": 1239,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 514.77734375,
      "completions/min_length": 188.5,
      "epoch": 6.7008086253369274,
      "grad_norm": 4.360246026410489,
      "kl": 3.9609375,
      "learning_rate": 2.4975238943127975e-07,
      "loss": 0.30287033319473267,
      "memory(GiB)": 75.26,
      "reward": 1.8458072543144226,
      "reward_std": 0.6128253042697906,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.34826864302158356,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3978758305311203,
      "rewards/ReflectionBalance/mean": 0.791119784116745,
      "rewards/ReflectionBalance/std": 0.3009806424379349,
      "step": 1240,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.0,
      "completions/mean_length": 479.578125,
      "completions/min_length": 264.0,
      "epoch": 6.706199460916442,
      "grad_norm": 4.587134242095675,
      "kl": 4.1640625,
      "learning_rate": 2.490100491849091e-07,
      "loss": 0.3290892243385315,
      "memory(GiB)": 75.26,
      "reward": 1.8857324123382568,
      "reward_std": 0.5769242346286774,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.33116789162158966,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4312489330768585,
      "rewards/ReflectionBalance/mean": 0.7646386623382568,
      "rewards/ReflectionBalance/std": 0.2908641844987869,
      "step": 1241,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 492.390625,
      "completions/min_length": 268.5,
      "epoch": 6.711590296495957,
      "grad_norm": 3.777017966757803,
      "kl": 3.84375,
      "learning_rate": 2.4826844782767895e-07,
      "loss": 0.28272438049316406,
      "memory(GiB)": 75.26,
      "reward": 1.9289119243621826,
      "reward_std": 0.5192840099334717,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31518664956092834,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4322105348110199,
      "rewards/ReflectionBalance/mean": 0.7960994839668274,
      "rewards/ReflectionBalance/std": 0.2707923352718353,
      "step": 1242,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 517.58984375,
      "completions/min_length": 276.5,
      "epoch": 6.716981132075472,
      "grad_norm": 3.2405784442624497,
      "kl": 2.671875,
      "learning_rate": 2.47527587542789e-07,
      "loss": 0.2089138627052307,
      "memory(GiB)": 75.26,
      "reward": 1.81201171875,
      "reward_std": 0.5004003345966339,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.31812146306037903,
      "rewards/ReflectionBalance/mean": 0.80810546875,
      "rewards/ReflectionBalance/std": 0.25204140692949295,
      "step": 1243,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 466.19921875,
      "completions/min_length": 242.0,
      "epoch": 6.722371967654986,
      "grad_norm": 2.925396975955346,
      "kl": 2.8203125,
      "learning_rate": 2.467874705112571e-07,
      "loss": 0.2558298707008362,
      "memory(GiB)": 75.26,
      "reward": 1.9029703736305237,
      "reward_std": 0.5115288496017456,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29262590408325195,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3820948451757431,
      "rewards/ReflectionBalance/mean": 0.8209391236305237,
      "rewards/ReflectionBalance/std": 0.23442183434963226,
      "step": 1244,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 502.96875,
      "completions/min_length": 260.0,
      "epoch": 6.727762803234501,
      "grad_norm": 3.054968825993957,
      "kl": 2.6640625,
      "learning_rate": 2.4604809891191376e-07,
      "loss": 0.23806415498256683,
      "memory(GiB)": 75.26,
      "reward": 1.924903929233551,
      "reward_std": 0.48390740156173706,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2813730239868164,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39184603095054626,
      "rewards/ReflectionBalance/mean": 0.8233414590358734,
      "rewards/ReflectionBalance/std": 0.21626875549554825,
      "step": 1245,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 502.25390625,
      "completions/min_length": 298.0,
      "epoch": 6.7331536388140165,
      "grad_norm": 8.538588042888763,
      "kl": 2.1796875,
      "learning_rate": 2.453094749213938e-07,
      "loss": 0.16587501764297485,
      "memory(GiB)": 75.26,
      "reward": 1.790138304233551,
      "reward_std": 0.5237132608890533,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.29881857335567474,
      "rewards/ReflectionBalance/mean": 0.7862321138381958,
      "rewards/ReflectionBalance/std": 0.26684846729040146,
      "step": 1246,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 450.078125,
      "completions/min_length": 256.5,
      "epoch": 6.738544474393531,
      "grad_norm": 3.700216094672748,
      "kl": 2.078125,
      "learning_rate": 2.445716007141324e-07,
      "loss": 0.18529066443443298,
      "memory(GiB)": 75.26,
      "reward": 1.9683008193969727,
      "reward_std": 0.4565891772508621,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.39705096185207367,
      "rewards/ReflectionBalance/mean": 0.8276757597923279,
      "rewards/ReflectionBalance/std": 0.21206512302160263,
      "step": 1247,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 496.13671875,
      "completions/min_length": 266.5,
      "epoch": 6.743935309973046,
      "grad_norm": 3.0737023828068817,
      "kl": 1.90625,
      "learning_rate": 2.438344784623568e-07,
      "loss": 0.1508854329586029,
      "memory(GiB)": 75.26,
      "reward": 1.95719313621521,
      "reward_std": 0.44968266785144806,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.38600586354732513,
      "rewards/ReflectionBalance/mean": 0.8282869458198547,
      "rewards/ReflectionBalance/std": 0.22786111384630203,
      "step": 1248,
      "train_speed(iter/s)": 0.007109
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 921.5,
      "completions/mean_length": 482.2421875,
      "completions/min_length": 257.5,
      "epoch": 6.74932614555256,
      "grad_norm": 2.8935146098397664,
      "kl": 1.57421875,
      "learning_rate": 2.4309811033608046e-07,
      "loss": 0.11622978746891022,
      "memory(GiB)": 75.26,
      "reward": 1.8796029090881348,
      "reward_std": 0.31592628359794617,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21220162510871887,
      "rewards/MultiModalAccuracyORM/mean": 0.0859375,
      "rewards/MultiModalAccuracyORM/std": 0.2810242921113968,
      "rewards/ReflectionBalance/mean": 0.84054034948349,
      "rewards/ReflectionBalance/std": 0.19718850404024124,
      "step": 1249,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 961.5,
      "completions/mean_length": 447.48046875,
      "completions/min_length": 250.5,
      "epoch": 6.754716981132075,
      "grad_norm": 4.424242122538845,
      "kl": 1.26953125,
      "learning_rate": 2.423624985030973e-07,
      "loss": 0.09698843955993652,
      "memory(GiB)": 75.26,
      "reward": 2.042320966720581,
      "reward_std": 0.46829453110694885,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4322105348110199,
      "rewards/ReflectionBalance/mean": 0.847008466720581,
      "rewards/ReflectionBalance/std": 0.19688314199447632,
      "step": 1250,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 487.77734375,
      "completions/min_length": 255.5,
      "epoch": 6.7601078167115904,
      "grad_norm": 2.8479589671082715,
      "kl": 2.40625,
      "learning_rate": 2.4162764512897443e-07,
      "loss": 0.19856590032577515,
      "memory(GiB)": 75.26,
      "reward": 1.8854883313179016,
      "reward_std": 0.522531658411026,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3707977831363678,
      "rewards/ReflectionBalance/mean": 0.8151758015155792,
      "rewards/ReflectionBalance/std": 0.26527902483940125,
      "step": 1251,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1020.5,
      "completions/mean_length": 490.57421875,
      "completions/min_length": 247.0,
      "epoch": 6.765498652291106,
      "grad_norm": 3.2250859191142833,
      "kl": 2.58203125,
      "learning_rate": 2.408935523770461e-07,
      "loss": 0.23245039582252502,
      "memory(GiB)": 75.26,
      "reward": 1.7914544343948364,
      "reward_std": 0.4909729063510895,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.0859375,
      "rewards/MultiModalAccuracyORM/std": 0.27558503299951553,
      "rewards/ReflectionBalance/mean": 0.8070794343948364,
      "rewards/ReflectionBalance/std": 0.25559062510728836,
      "step": 1252,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 466.44921875,
      "completions/min_length": 257.0,
      "epoch": 6.77088948787062,
      "grad_norm": 3.535121438016926,
      "kl": 2.46875,
      "learning_rate": 2.4016022240840717e-07,
      "loss": 0.1800910383462906,
      "memory(GiB)": 75.26,
      "reward": 1.8540478348731995,
      "reward_std": 0.549062967300415,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.29259639233350754,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35872696340084076,
      "rewards/ReflectionBalance/mean": 0.8032666444778442,
      "rewards/ReflectionBalance/std": 0.2505806088447571,
      "step": 1253,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 472.78515625,
      "completions/min_length": 247.5,
      "epoch": 6.776280323450135,
      "grad_norm": 3.073636961586124,
      "kl": 1.875,
      "learning_rate": 2.3942765738190777e-07,
      "loss": 0.14750264585018158,
      "memory(GiB)": 75.26,
      "reward": 1.8890169858932495,
      "reward_std": 0.4434949457645416,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.32345911115407944,
      "rewards/ReflectionBalance/mean": 0.8226106762886047,
      "rewards/ReflectionBalance/std": 0.23670221865177155,
      "step": 1254,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 476.1953125,
      "completions/min_length": 148.5,
      "epoch": 6.781671159029649,
      "grad_norm": 2.8352367272621177,
      "kl": 1.796875,
      "learning_rate": 2.386958594541453e-07,
      "loss": 0.1387372463941574,
      "memory(GiB)": 75.26,
      "reward": 1.9711441397666931,
      "reward_std": 0.45315590500831604,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.4020131081342697,
      "rewards/ReflectionBalance/mean": 0.8227067291736603,
      "rewards/ReflectionBalance/std": 0.23549865931272507,
      "step": 1255,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 476.94921875,
      "completions/min_length": 198.0,
      "epoch": 6.787061994609164,
      "grad_norm": 3.969447272247183,
      "kl": 2.375,
      "learning_rate": 2.3796483077945895e-07,
      "loss": 0.16586267948150635,
      "memory(GiB)": 75.26,
      "reward": 1.8263338804244995,
      "reward_std": 0.501958966255188,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.3166656941175461,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.33312736451625824,
      "rewards/ReflectionBalance/mean": 0.7989901006221771,
      "rewards/ReflectionBalance/std": 0.2797229364514351,
      "step": 1256,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 460.87109375,
      "completions/min_length": 245.5,
      "epoch": 6.7924528301886795,
      "grad_norm": 3.7224340104369658,
      "kl": 2.23828125,
      "learning_rate": 2.3723457350992416e-07,
      "loss": 0.19010847806930542,
      "memory(GiB)": 75.26,
      "reward": 1.9063215851783752,
      "reward_std": 0.5166977941989899,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3846261203289032,
      "rewards/ReflectionBalance/mean": 0.81257164478302,
      "rewards/ReflectionBalance/std": 0.28038744628429413,
      "step": 1257,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 496.4609375,
      "completions/min_length": 262.5,
      "epoch": 6.797843665768194,
      "grad_norm": 3.7040906436704844,
      "kl": 2.4765625,
      "learning_rate": 2.3650508979534394e-07,
      "loss": 0.22079655528068542,
      "memory(GiB)": 75.26,
      "reward": 1.871961236000061,
      "reward_std": 0.4356488734483719,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.32050570100545883,
      "rewards/ReflectionBalance/mean": 0.8133675158023834,
      "rewards/ReflectionBalance/std": 0.2289532795548439,
      "step": 1258,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 466.08984375,
      "completions/min_length": 232.0,
      "epoch": 6.803234501347709,
      "grad_norm": 3.135277003486892,
      "kl": 2.32421875,
      "learning_rate": 2.3577638178324555e-07,
      "loss": 0.17899779975414276,
      "memory(GiB)": 75.26,
      "reward": 2.0090593099594116,
      "reward_std": 0.48992858827114105,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.27558503299951553,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.42646360397338867,
      "rewards/ReflectionBalance/mean": 0.8410905003547668,
      "rewards/ReflectionBalance/std": 0.22598478198051453,
      "step": 1259,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 472.41015625,
      "completions/min_length": 271.5,
      "epoch": 6.808625336927224,
      "grad_norm": 2.955604367548489,
      "kl": 2.8203125,
      "learning_rate": 2.3504845161887183e-07,
      "loss": 0.23219989240169525,
      "memory(GiB)": 75.26,
      "reward": 1.9348355531692505,
      "reward_std": 0.4754379093647003,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.41492627561092377,
      "rewards/ReflectionBalance/mean": 0.8059293627738953,
      "rewards/ReflectionBalance/std": 0.22867532074451447,
      "step": 1260,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 475.68359375,
      "completions/min_length": 224.0,
      "epoch": 6.814016172506738,
      "grad_norm": 2.96512719467971,
      "kl": 2.75,
      "learning_rate": 2.3432130144517553e-07,
      "loss": 0.2209193855524063,
      "memory(GiB)": 75.26,
      "reward": 1.9160847067832947,
      "reward_std": 0.4933543801307678,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.4007115960121155,
      "rewards/ReflectionBalance/mean": 0.8106160461902618,
      "rewards/ReflectionBalance/std": 0.24262230843305588,
      "step": 1261,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 495.85546875,
      "completions/min_length": 280.0,
      "epoch": 6.819407008086253,
      "grad_norm": 3.1363569223638277,
      "kl": 3.390625,
      "learning_rate": 2.335949334028141e-07,
      "loss": 0.2992185354232788,
      "memory(GiB)": 75.26,
      "reward": 1.8374038934707642,
      "reward_std": 0.47421014308929443,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.3219822645187378,
      "rewards/ReflectionBalance/mean": 0.8139664828777313,
      "rewards/ReflectionBalance/std": 0.23774641007184982,
      "step": 1262,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 511.53515625,
      "completions/min_length": 243.0,
      "epoch": 6.824797843665769,
      "grad_norm": 4.55279311294729,
      "kl": 4.859375,
      "learning_rate": 2.328693496301416e-07,
      "loss": 0.3847385048866272,
      "memory(GiB)": 75.26,
      "reward": 1.8767611384391785,
      "reward_std": 0.6366117298603058,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.34485287964344025,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.43581391870975494,
      "rewards/ReflectionBalance/mean": 0.7595735788345337,
      "rewards/ReflectionBalance/std": 0.30831991136074066,
      "step": 1263,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 500.71484375,
      "completions/min_length": 267.0,
      "epoch": 6.830188679245283,
      "grad_norm": 3.3852113953658654,
      "kl": 3.4609375,
      "learning_rate": 2.321445522632036e-07,
      "loss": 0.2562131881713867,
      "memory(GiB)": 75.26,
      "reward": 1.8297606706619263,
      "reward_std": 0.5886930525302887,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33636194467544556,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3689865916967392,
      "rewards/ReflectionBalance/mean": 0.7828857600688934,
      "rewards/ReflectionBalance/std": 0.30756694078445435,
      "step": 1264,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 508.6796875,
      "completions/min_length": 247.0,
      "epoch": 6.835579514824798,
      "grad_norm": 4.94220925126604,
      "kl": 4.0390625,
      "learning_rate": 2.3142054343573048e-07,
      "loss": 0.29185429215431213,
      "memory(GiB)": 75.26,
      "reward": 1.929248034954071,
      "reward_std": 0.5286378413438797,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42764075100421906,
      "rewards/ReflectionBalance/mean": 0.800341784954071,
      "rewards/ReflectionBalance/std": 0.26648783683776855,
      "step": 1265,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 487.3203125,
      "completions/min_length": 277.0,
      "epoch": 6.840970350404312,
      "grad_norm": 3.9269311489972822,
      "kl": 2.734375,
      "learning_rate": 2.3069732527913156e-07,
      "loss": 0.19722016155719757,
      "memory(GiB)": 75.26,
      "reward": 1.8731982111930847,
      "reward_std": 0.33341100811958313,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.30669571459293365,
      "rewards/ReflectionBalance/mean": 0.8458545207977295,
      "rewards/ReflectionBalance/std": 0.2013665810227394,
      "step": 1266,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 500.4140625,
      "completions/min_length": 249.0,
      "epoch": 6.846361185983827,
      "grad_norm": 3.025220009111975,
      "kl": 4.015625,
      "learning_rate": 2.2997489992248815e-07,
      "loss": 0.34525078535079956,
      "memory(GiB)": 75.26,
      "reward": 1.8575406670570374,
      "reward_std": 0.5931659042835236,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.339918777346611,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.38200071454048157,
      "rewards/ReflectionBalance/mean": 0.7794156968593597,
      "rewards/ReflectionBalance/std": 0.2906758040189743,
      "step": 1267,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 522.3671875,
      "completions/min_length": 290.5,
      "epoch": 6.8517520215633425,
      "grad_norm": 3.4315674973483103,
      "kl": 3.6015625,
      "learning_rate": 2.2925326949254759e-07,
      "loss": 0.27033689618110657,
      "memory(GiB)": 75.26,
      "reward": 1.8121386170387268,
      "reward_std": 0.5302986204624176,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32612966001033783,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3473474234342575,
      "rewards/ReflectionBalance/mean": 0.7926074266433716,
      "rewards/ReflectionBalance/std": 0.29188530147075653,
      "step": 1268,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 514.109375,
      "completions/min_length": 170.0,
      "epoch": 6.857142857142857,
      "grad_norm": 6.056678889818387,
      "kl": 2.7109375,
      "learning_rate": 2.2853243611371763e-07,
      "loss": 0.19260618090629578,
      "memory(GiB)": 75.26,
      "reward": 1.838955044746399,
      "reward_std": 0.42586249113082886,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.34485287964344025,
      "rewards/ReflectionBalance/mean": 0.8037987947463989,
      "rewards/ReflectionBalance/std": 0.2562400996685028,
      "step": 1269,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 460.078125,
      "completions/min_length": 230.5,
      "epoch": 6.862533692722372,
      "grad_norm": 3.9026349771232476,
      "kl": 1.63671875,
      "learning_rate": 2.2781240190805855e-07,
      "loss": 0.14712205529212952,
      "memory(GiB)": 75.26,
      "reward": 1.957076907157898,
      "reward_std": 0.3656056672334671,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23443959653377533,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38047872483730316,
      "rewards/ReflectionBalance/mean": 0.8359830379486084,
      "rewards/ReflectionBalance/std": 0.21635105460882187,
      "step": 1270,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 496.36328125,
      "completions/min_length": 256.0,
      "epoch": 6.867924528301887,
      "grad_norm": 3.302457625743519,
      "kl": 3.0,
      "learning_rate": 2.2709316899527907e-07,
      "loss": 0.2649056911468506,
      "memory(GiB)": 75.26,
      "reward": 1.8555980324745178,
      "reward_std": 0.47068752348423004,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3422359675168991,
      "rewards/ReflectionBalance/mean": 0.8126291930675507,
      "rewards/ReflectionBalance/std": 0.23247423768043518,
      "step": 1271,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1019.5,
      "completions/mean_length": 480.24609375,
      "completions/min_length": 282.5,
      "epoch": 6.873315363881401,
      "grad_norm": 3.1464370739832384,
      "kl": 2.71875,
      "learning_rate": 2.2637473949272822e-07,
      "loss": 0.2139233946800232,
      "memory(GiB)": 75.26,
      "reward": 1.9756510257720947,
      "reward_std": 0.434722363948822,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4184056967496872,
      "rewards/ReflectionBalance/mean": 0.8311197757720947,
      "rewards/ReflectionBalance/std": 0.23337073624134064,
      "step": 1272,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1036.0,
      "completions/mean_length": 479.453125,
      "completions/min_length": 184.0,
      "epoch": 6.878706199460916,
      "grad_norm": 3.955973079099826,
      "kl": 2.703125,
      "learning_rate": 2.2565711551538997e-07,
      "loss": 0.20561225712299347,
      "memory(GiB)": 75.26,
      "reward": 1.90101557970047,
      "reward_std": 0.5340216606855392,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4060489982366562,
      "rewards/ReflectionBalance/mean": 0.7877343595027924,
      "rewards/ReflectionBalance/std": 0.29466331005096436,
      "step": 1273,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 488.44921875,
      "completions/min_length": 280.5,
      "epoch": 6.884097035040432,
      "grad_norm": 2.940040459481491,
      "kl": 2.01171875,
      "learning_rate": 2.249402991758773e-07,
      "loss": 0.160515695810318,
      "memory(GiB)": 75.26,
      "reward": 1.931892991065979,
      "reward_std": 0.4465995728969574,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.37402482330799103,
      "rewards/ReflectionBalance/mean": 0.8264241218566895,
      "rewards/ReflectionBalance/std": 0.23858823627233505,
      "step": 1274,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 497.3203125,
      "completions/min_length": 144.0,
      "epoch": 6.889487870619946,
      "grad_norm": 3.2592113134818033,
      "kl": 1.4921875,
      "learning_rate": 2.2422429258442532e-07,
      "loss": 0.11138056218624115,
      "memory(GiB)": 75.26,
      "reward": 1.941792070865631,
      "reward_std": 0.480577677488327,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.35081249475479126,
      "rewards/ReflectionBalance/mean": 0.8519482314586639,
      "rewards/ReflectionBalance/std": 0.18449600785970688,
      "step": 1275,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 477.15625,
      "completions/min_length": 241.0,
      "epoch": 6.894878706199461,
      "grad_norm": 3.061565842866254,
      "kl": 1.515625,
      "learning_rate": 2.2350909784888512e-07,
      "loss": 0.11329472810029984,
      "memory(GiB)": 75.26,
      "reward": 1.9720028042793274,
      "reward_std": 0.4509774446487427,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24081215262413025,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3980395793914795,
      "rewards/ReflectionBalance/mean": 0.8313777446746826,
      "rewards/ReflectionBalance/std": 0.20477106422185898,
      "step": 1276,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 912.5,
      "completions/mean_length": 492.43359375,
      "completions/min_length": 283.0,
      "epoch": 6.900269541778976,
      "grad_norm": 3.845174836316949,
      "kl": 1.6796875,
      "learning_rate": 2.2279471707471837e-07,
      "loss": 0.15400198101997375,
      "memory(GiB)": 75.26,
      "reward": 2.0179067850112915,
      "reward_std": 0.42243510484695435,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4229802340269089,
      "rewards/ReflectionBalance/mean": 0.8382193744182587,
      "rewards/ReflectionBalance/std": 0.19385895133018494,
      "step": 1277,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 493.09765625,
      "completions/min_length": 259.0,
      "epoch": 6.90566037735849,
      "grad_norm": 3.1974225262894196,
      "kl": 3.0078125,
      "learning_rate": 2.2208115236498958e-07,
      "loss": 0.20955853164196014,
      "memory(GiB)": 75.26,
      "reward": 1.9066650867462158,
      "reward_std": 0.5520674884319305,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32746291160583496,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.43946409225463867,
      "rewards/ReflectionBalance/mean": 0.766040027141571,
      "rewards/ReflectionBalance/std": 0.3158201277256012,
      "step": 1278,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 493.359375,
      "completions/min_length": 197.0,
      "epoch": 6.9110512129380055,
      "grad_norm": 2.834789225760471,
      "kl": 1.875,
      "learning_rate": 2.2136840582036188e-07,
      "loss": 0.14016801118850708,
      "memory(GiB)": 75.26,
      "reward": 1.9050260186195374,
      "reward_std": 0.486402228474617,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3634065240621567,
      "rewards/ReflectionBalance/mean": 0.8229947686195374,
      "rewards/ReflectionBalance/std": 0.25595883280038834,
      "step": 1279,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1038.0,
      "completions/mean_length": 496.90234375,
      "completions/min_length": 268.0,
      "epoch": 6.916442048517521,
      "grad_norm": 3.402543980343285,
      "kl": 2.6640625,
      "learning_rate": 2.2065647953908907e-07,
      "loss": 0.23487582802772522,
      "memory(GiB)": 75.26,
      "reward": 1.9893360137939453,
      "reward_std": 0.5383568108081818,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.3125,
      "rewards/MultiModalAccuracyORM/std": 0.44985245168209076,
      "rewards/ReflectionBalance/mean": 0.7862109541893005,
      "rewards/ReflectionBalance/std": 0.2972100079059601,
      "step": 1280,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 521.73046875,
      "completions/min_length": 285.5,
      "epoch": 6.921832884097035,
      "grad_norm": 3.34393934526314,
      "kl": 2.6328125,
      "learning_rate": 2.1994537561701036e-07,
      "loss": 0.22180961072444916,
      "memory(GiB)": 75.26,
      "reward": 1.8487744331359863,
      "reward_std": 0.5445437431335449,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3764989674091339,
      "rewards/ReflectionBalance/mean": 0.7784619033336639,
      "rewards/ReflectionBalance/std": 0.2766066938638687,
      "step": 1281,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 488.2109375,
      "completions/min_length": 276.0,
      "epoch": 6.92722371967655,
      "grad_norm": 3.4149416657011087,
      "kl": 2.1328125,
      "learning_rate": 2.1923509614754443e-07,
      "loss": 0.17509880661964417,
      "memory(GiB)": 75.26,
      "reward": 1.9280778169631958,
      "reward_std": 0.5157838761806488,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41328008472919464,
      "rewards/ReflectionBalance/mean": 0.7952652871608734,
      "rewards/ReflectionBalance/std": 0.27606847137212753,
      "step": 1282,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 487.48828125,
      "completions/min_length": 258.5,
      "epoch": 6.932614555256064,
      "grad_norm": 3.924030006680645,
      "kl": 2.2578125,
      "learning_rate": 2.1852564322168227e-07,
      "loss": 0.19975249469280243,
      "memory(GiB)": 75.26,
      "reward": 2.0824657678604126,
      "reward_std": 0.5121864676475525,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.33984375,
      "rewards/MultiModalAccuracyORM/std": 0.47549907863140106,
      "rewards/ReflectionBalance/mean": 0.8246532678604126,
      "rewards/ReflectionBalance/std": 0.2581566274166107,
      "step": 1283,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 974.0,
      "completions/mean_length": 491.89453125,
      "completions/min_length": 129.5,
      "epoch": 6.938005390835579,
      "grad_norm": 3.2959567238437137,
      "kl": 2.8984375,
      "learning_rate": 2.1781701892798188e-07,
      "loss": 0.20860061049461365,
      "memory(GiB)": 75.26,
      "reward": 1.82475745677948,
      "reward_std": 0.5348005145788193,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3142324984073639,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36935214698314667,
      "rewards/ReflectionBalance/mean": 0.7778824865818024,
      "rewards/ReflectionBalance/std": 0.26025065034627914,
      "step": 1284,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 498.77734375,
      "completions/min_length": 249.0,
      "epoch": 6.943396226415095,
      "grad_norm": 3.4560718942035877,
      "kl": 2.6171875,
      "learning_rate": 2.171092253525617e-07,
      "loss": 0.18538638949394226,
      "memory(GiB)": 75.26,
      "reward": 1.9057177305221558,
      "reward_std": 0.484185591340065,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39184603095054626,
      "rewards/ReflectionBalance/mean": 0.8080615401268005,
      "rewards/ReflectionBalance/std": 0.23737163841724396,
      "step": 1285,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 497.84375,
      "completions/min_length": 111.0,
      "epoch": 6.948787061994609,
      "grad_norm": 4.091996214781672,
      "kl": 3.265625,
      "learning_rate": 2.16402264579095e-07,
      "loss": 0.2377641201019287,
      "memory(GiB)": 75.26,
      "reward": 1.8352522253990173,
      "reward_std": 0.5833252519369125,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.3274291008710861,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3787541687488556,
      "rewards/ReflectionBalance/mean": 0.7922835350036621,
      "rewards/ReflectionBalance/std": 0.2960692346096039,
      "step": 1286,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 476.953125,
      "completions/min_length": 266.0,
      "epoch": 6.954177897574124,
      "grad_norm": 3.23841698383203,
      "kl": 2.703125,
      "learning_rate": 2.1569613868880304e-07,
      "loss": 0.2108107954263687,
      "memory(GiB)": 75.26,
      "reward": 1.9161051511764526,
      "reward_std": 0.5414489507675171,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.27175262570381165,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41762344539165497,
      "rewards/ReflectionBalance/mean": 0.7793864011764526,
      "rewards/ReflectionBalance/std": 0.2818537652492523,
      "step": 1287,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 501.17578125,
      "completions/min_length": 187.5,
      "epoch": 6.959568733153639,
      "grad_norm": 5.090799108546534,
      "kl": 3.546875,
      "learning_rate": 2.1499084976044906e-07,
      "loss": 0.22851580381393433,
      "memory(GiB)": 75.26,
      "reward": 1.9157161712646484,
      "reward_std": 0.5831074416637421,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.3425614684820175,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.44543786346912384,
      "rewards/ReflectionBalance/mean": 0.7829036712646484,
      "rewards/ReflectionBalance/std": 0.27705787867307663,
      "step": 1288,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 486.6484375,
      "completions/min_length": 260.0,
      "epoch": 6.964959568733153,
      "grad_norm": 3.53467540043184,
      "kl": 2.94140625,
      "learning_rate": 2.142863998703332e-07,
      "loss": 0.24177616834640503,
      "memory(GiB)": 75.26,
      "reward": 1.9442521333694458,
      "reward_std": 0.5380814224481583,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.27645355463027954,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.4363734573125839,
      "rewards/ReflectionBalance/mean": 0.7801896333694458,
      "rewards/ReflectionBalance/std": 0.2886490300297737,
      "step": 1289,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.0,
      "completions/mean_length": 517.9140625,
      "completions/min_length": 252.5,
      "epoch": 6.9703504043126685,
      "grad_norm": 3.433860025731393,
      "kl": 2.9921875,
      "learning_rate": 2.1358279109228434e-07,
      "loss": 0.229686439037323,
      "memory(GiB)": 75.26,
      "reward": 1.8511182069778442,
      "reward_std": 0.5079220235347748,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.29639076441526413,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.37157969176769257,
      "rewards/ReflectionBalance/mean": 0.7847118973731995,
      "rewards/ReflectionBalance/std": 0.29783471673727036,
      "step": 1290,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 506.2734375,
      "completions/min_length": 285.0,
      "epoch": 6.975741239892184,
      "grad_norm": 3.070492964322453,
      "kl": 2.08984375,
      "learning_rate": 2.1288002549765634e-07,
      "loss": 0.15219458937644958,
      "memory(GiB)": 75.26,
      "reward": 1.9527400135993958,
      "reward_std": 0.40466445684432983,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24301259219646454,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3778642416000366,
      "rewards/ReflectionBalance/mean": 0.8394587934017181,
      "rewards/ReflectionBalance/std": 0.18898390978574753,
      "step": 1291,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 481.71484375,
      "completions/min_length": 252.5,
      "epoch": 6.981132075471698,
      "grad_norm": 4.13332297718687,
      "kl": 1.984375,
      "learning_rate": 2.1217810515532013e-07,
      "loss": 0.19783467054367065,
      "memory(GiB)": 75.26,
      "reward": 1.8994848728179932,
      "reward_std": 0.4884076416492462,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3752707988023758,
      "rewards/ReflectionBalance/mean": 0.8096410930156708,
      "rewards/ReflectionBalance/std": 0.26577533781528473,
      "step": 1292,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 501.6875,
      "completions/min_length": 270.0,
      "epoch": 6.986522911051213,
      "grad_norm": 2.9235569113903916,
      "kl": 1.84375,
      "learning_rate": 2.1147703213165845e-07,
      "loss": 0.15414732694625854,
      "memory(GiB)": 75.26,
      "reward": 1.9370312094688416,
      "reward_std": 0.3978228271007538,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3949224203824997,
      "rewards/ReflectionBalance/mean": 0.8120312690734863,
      "rewards/ReflectionBalance/std": 0.2522634416818619,
      "step": 1293,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 457.69140625,
      "completions/min_length": 282.5,
      "epoch": 6.991913746630727,
      "grad_norm": 2.629667394527606,
      "kl": 1.21484375,
      "learning_rate": 2.1077680849055995e-07,
      "loss": 0.09749595820903778,
      "memory(GiB)": 75.26,
      "reward": 2.011744797229767,
      "reward_std": 0.3922479450702667,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.194504976272583,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4059968739748001,
      "rewards/ReflectionBalance/mean": 0.8359635472297668,
      "rewards/ReflectionBalance/std": 0.23278042674064636,
      "step": 1294,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 498.05078125,
      "completions/min_length": 257.0,
      "epoch": 6.997304582210242,
      "grad_norm": 3.138248660025777,
      "kl": 1.884765625,
      "learning_rate": 2.100774362934124e-07,
      "loss": 0.15978515148162842,
      "memory(GiB)": 75.26,
      "reward": 1.88078773021698,
      "reward_std": 0.4438387304544449,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.253920242190361,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3177158683538437,
      "rewards/ReflectionBalance/mean": 0.8339127600193024,
      "rewards/ReflectionBalance/std": 0.21289414912462234,
      "step": 1295,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 456.22265625,
      "completions/min_length": 266.5,
      "epoch": 7.005390835579515,
      "grad_norm": 8.695526212113602,
      "kl": 1.59765625,
      "learning_rate": 2.0937891759909703e-07,
      "loss": 0.173575758934021,
      "memory(GiB)": 75.26,
      "reward": 2.0818586349487305,
      "reward_std": 0.44610893726348877,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.21138381212949753,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.45224134624004364,
      "rewards/ReflectionBalance/mean": 0.84357750415802,
      "rewards/ReflectionBalance/std": 0.20686130970716476,
      "step": 1296,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 472.18359375,
      "completions/min_length": 230.5,
      "epoch": 7.010781671159029,
      "grad_norm": 16.169497468873548,
      "kl": 1.12109375,
      "learning_rate": 2.0868125446398266e-07,
      "loss": 0.09272237867116928,
      "memory(GiB)": 75.26,
      "reward": 1.9353271126747131,
      "reward_std": 0.33102478086948395,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.19007275998592377,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3334042727947235,
      "rewards/ReflectionBalance/mean": 0.8415771722793579,
      "rewards/ReflectionBalance/std": 0.1826067715883255,
      "step": 1297,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 484.66015625,
      "completions/min_length": 268.5,
      "epoch": 7.0161725067385445,
      "grad_norm": 3.5799124179756,
      "kl": 1.58984375,
      "learning_rate": 2.07984448941919e-07,
      "loss": 0.14095164835453033,
      "memory(GiB)": 75.26,
      "reward": 1.9398340582847595,
      "reward_std": 0.4492327868938446,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.37408973276615143,
      "rewards/ReflectionBalance/mean": 0.8343652188777924,
      "rewards/ReflectionBalance/std": 0.22451671212911606,
      "step": 1298,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 470.5390625,
      "completions/min_length": 250.0,
      "epoch": 7.02156334231806,
      "grad_norm": 2.965928742946467,
      "kl": 1.77734375,
      "learning_rate": 2.0728850308423184e-07,
      "loss": 0.1601022332906723,
      "memory(GiB)": 75.26,
      "reward": 2.025271773338318,
      "reward_std": 0.4396960139274597,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4243630915880203,
      "rewards/ReflectionBalance/mean": 0.8455843031406403,
      "rewards/ReflectionBalance/std": 0.20838221162557602,
      "step": 1299,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 478.53515625,
      "completions/min_length": 275.0,
      "epoch": 7.026954177897574,
      "grad_norm": 2.6476162445420264,
      "kl": 1.55078125,
      "learning_rate": 2.0659341893971538e-07,
      "loss": 0.12359219789505005,
      "memory(GiB)": 75.26,
      "reward": 1.9738354682922363,
      "reward_std": 0.3400861769914627,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.4007115960121155,
      "rewards/ReflectionBalance/mean": 0.8332104682922363,
      "rewards/ReflectionBalance/std": 0.1971072033047676,
      "step": 1300,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.0,
      "completions/mean_length": 490.32421875,
      "completions/min_length": 270.5,
      "epoch": 7.032345013477089,
      "grad_norm": 3.1052746578026245,
      "kl": 2.1953125,
      "learning_rate": 2.0589919855462745e-07,
      "loss": 0.1706361174583435,
      "memory(GiB)": 75.26,
      "reward": 1.9550211429595947,
      "reward_std": 0.40097571909427643,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4041530042886734,
      "rewards/ReflectionBalance/mean": 0.8143961727619171,
      "rewards/ReflectionBalance/std": 0.2288384959101677,
      "step": 1301,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 495.12890625,
      "completions/min_length": 283.5,
      "epoch": 7.037735849056604,
      "grad_norm": 3.207913725954092,
      "kl": 2.30078125,
      "learning_rate": 2.052058439726827e-07,
      "loss": 0.19356068968772888,
      "memory(GiB)": 75.26,
      "reward": 1.9581835865974426,
      "reward_std": 0.44829893112182617,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3985297381877899,
      "rewards/ReflectionBalance/mean": 0.829277366399765,
      "rewards/ReflectionBalance/std": 0.22960717976093292,
      "step": 1302,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 511.91796875,
      "completions/min_length": 268.5,
      "epoch": 7.0431266846361185,
      "grad_norm": 3.5252776045176097,
      "kl": 2.18359375,
      "learning_rate": 2.045133572350476e-07,
      "loss": 0.2022431492805481,
      "memory(GiB)": 75.26,
      "reward": 1.8632943630218506,
      "reward_std": 0.4690646380186081,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3410066068172455,
      "rewards/ReflectionBalance/mean": 0.8281380534172058,
      "rewards/ReflectionBalance/std": 0.18638260662555695,
      "step": 1303,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.0,
      "completions/mean_length": 494.078125,
      "completions/min_length": 253.0,
      "epoch": 7.048517520215634,
      "grad_norm": 2.9285885464648307,
      "kl": 2.0234375,
      "learning_rate": 2.0382174038033317e-07,
      "loss": 0.16220054030418396,
      "memory(GiB)": 75.26,
      "reward": 2.0064233541488647,
      "reward_std": 0.5109542608261108,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2813730239868164,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.4460870176553726,
      "rewards/ReflectionBalance/mean": 0.8189233243465424,
      "rewards/ReflectionBalance/std": 0.23787689208984375,
      "step": 1304,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 491.078125,
      "completions/min_length": 226.0,
      "epoch": 7.053908355795148,
      "grad_norm": 3.4383014354210952,
      "kl": 3.3046875,
      "learning_rate": 2.0313099544458962e-07,
      "loss": 0.2430202066898346,
      "memory(GiB)": 75.26,
      "reward": 1.7935628294944763,
      "reward_std": 0.48203064501285553,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.3021402209997177,
      "rewards/ReflectionBalance/mean": 0.7974690496921539,
      "rewards/ReflectionBalance/std": 0.2606668025255203,
      "step": 1305,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 504.61328125,
      "completions/min_length": 228.5,
      "epoch": 7.059299191374663,
      "grad_norm": 2.752608674315312,
      "kl": 2.453125,
      "learning_rate": 2.0244112446130085e-07,
      "loss": 0.21125906705856323,
      "memory(GiB)": 75.26,
      "reward": 1.8968831896781921,
      "reward_std": 0.49387456476688385,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29262590408325195,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.368965208530426,
      "rewards/ReflectionBalance/mean": 0.8226643800735474,
      "rewards/ReflectionBalance/std": 0.2292303964495659,
      "step": 1306,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 492.28515625,
      "completions/min_length": 245.0,
      "epoch": 7.064690026954178,
      "grad_norm": 3.135191978572639,
      "kl": 2.765625,
      "learning_rate": 2.0175212946137732e-07,
      "loss": 0.22117243707180023,
      "memory(GiB)": 75.26,
      "reward": 1.889855146408081,
      "reward_std": 0.48869891464710236,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.803917646408081,
      "rewards/ReflectionBalance/std": 0.26881228387355804,
      "step": 1307,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 501.8828125,
      "completions/min_length": 261.5,
      "epoch": 7.070080862533692,
      "grad_norm": 4.219713185456338,
      "kl": 3.671875,
      "learning_rate": 2.0106401247315085e-07,
      "loss": 0.29359129071235657,
      "memory(GiB)": 75.26,
      "reward": 1.8135303258895874,
      "reward_std": 0.5379239618778229,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.34734736382961273,
      "rewards/ReflectionBalance/mean": 0.762749046087265,
      "rewards/ReflectionBalance/std": 0.29993192851543427,
      "step": 1308,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 474.515625,
      "completions/min_length": 244.5,
      "epoch": 7.0754716981132075,
      "grad_norm": 3.687637215188661,
      "kl": 3.953125,
      "learning_rate": 2.0037677552236903e-07,
      "loss": 0.30548569560050964,
      "memory(GiB)": 75.26,
      "reward": 1.8714045882225037,
      "reward_std": 0.5459803938865662,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3300945907831192,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4077630192041397,
      "rewards/ReflectionBalance/mean": 0.7854671478271484,
      "rewards/ReflectionBalance/std": 0.288265660405159,
      "step": 1309,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 494.12890625,
      "completions/min_length": 266.0,
      "epoch": 7.080862533692723,
      "grad_norm": 3.627453914730806,
      "kl": 3.3984375,
      "learning_rate": 1.9969042063218762e-07,
      "loss": 0.25763773918151855,
      "memory(GiB)": 75.26,
      "reward": 1.977576494216919,
      "reward_std": 0.5430286675691605,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3285723924636841,
      "rewards/MultiModalAccuracyORM/mean": 0.3125,
      "rewards/MultiModalAccuracyORM/std": 0.4634045511484146,
      "rewards/ReflectionBalance/mean": 0.790076494216919,
      "rewards/ReflectionBalance/std": 0.26970648020505905,
      "step": 1310,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 501.67578125,
      "completions/min_length": 296.5,
      "epoch": 7.086253369272237,
      "grad_norm": 5.203947476568152,
      "kl": 3.4609375,
      "learning_rate": 1.9900494982316668e-07,
      "loss": 0.23602178692817688,
      "memory(GiB)": 75.26,
      "reward": 1.8492041230201721,
      "reward_std": 0.4944663345813751,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3037657365202904,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3607245087623596,
      "rewards/ReflectionBalance/mean": 0.8062353432178497,
      "rewards/ReflectionBalance/std": 0.2591498866677284,
      "step": 1311,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 491.109375,
      "completions/min_length": 262.0,
      "epoch": 7.091644204851752,
      "grad_norm": 3.2492632337381546,
      "kl": 3.40625,
      "learning_rate": 1.9832036511326317e-07,
      "loss": 0.2879559397697449,
      "memory(GiB)": 75.26,
      "reward": 1.8875749111175537,
      "reward_std": 0.5980162918567657,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3210565447807312,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4141528308391571,
      "rewards/ReflectionBalance/mean": 0.7742936313152313,
      "rewards/ReflectionBalance/std": 0.2946169003844261,
      "step": 1312,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 517.82421875,
      "completions/min_length": 264.5,
      "epoch": 7.097035040431267,
      "grad_norm": 3.0256678543820166,
      "kl": 3.421875,
      "learning_rate": 1.9763666851782528e-07,
      "loss": 0.25942981243133545,
      "memory(GiB)": 75.26,
      "reward": 1.8584537506103516,
      "reward_std": 0.6137642860412598,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.32267695665359497,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3940901756286621,
      "rewards/ReflectionBalance/mean": 0.772516280412674,
      "rewards/ReflectionBalance/std": 0.28733035922050476,
      "step": 1313,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 485.05859375,
      "completions/min_length": 291.0,
      "epoch": 7.1024258760107815,
      "grad_norm": 3.121530364941258,
      "kl": 2.625,
      "learning_rate": 1.969538620495874e-07,
      "loss": 0.24410608410835266,
      "memory(GiB)": 75.26,
      "reward": 1.9956811666488647,
      "reward_std": 0.5414870083332062,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.44543786346912384,
      "rewards/ReflectionBalance/mean": 0.8081811666488647,
      "rewards/ReflectionBalance/std": 0.27147530764341354,
      "step": 1314,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 501.4375,
      "completions/min_length": 240.0,
      "epoch": 7.107816711590297,
      "grad_norm": 2.773731205680078,
      "kl": 2.1015625,
      "learning_rate": 1.962719477186628e-07,
      "loss": 0.166191965341568,
      "memory(GiB)": 75.26,
      "reward": 1.9280436038970947,
      "reward_std": 0.4849397987127304,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3804051876068115,
      "rewards/ReflectionBalance/mean": 0.8108561336994171,
      "rewards/ReflectionBalance/std": 0.2509533390402794,
      "step": 1315,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 489.06640625,
      "completions/min_length": 215.0,
      "epoch": 7.113207547169812,
      "grad_norm": 3.1935969391990224,
      "kl": 2.390625,
      "learning_rate": 1.9559092753253858e-07,
      "loss": 0.20214278995990753,
      "memory(GiB)": 75.26,
      "reward": 1.9827131032943726,
      "reward_std": 0.5798712968826294,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4378891587257385,
      "rewards/ReflectionBalance/mean": 0.8030257225036621,
      "rewards/ReflectionBalance/std": 0.28320519626140594,
      "step": 1316,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 502.19140625,
      "completions/min_length": 247.0,
      "epoch": 7.118598382749326,
      "grad_norm": 2.7184911880594256,
      "kl": 1.92578125,
      "learning_rate": 1.9491080349606936e-07,
      "loss": 0.14374931156635284,
      "memory(GiB)": 75.26,
      "reward": 1.8924511671066284,
      "reward_std": 0.4381546378135681,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.37352070212364197,
      "rewards/ReflectionBalance/mean": 0.8104199469089508,
      "rewards/ReflectionBalance/std": 0.257904089987278,
      "step": 1317,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 486.58203125,
      "completions/min_length": 246.0,
      "epoch": 7.123989218328841,
      "grad_norm": 3.48264325714106,
      "kl": 2.12109375,
      "learning_rate": 1.942315776114723e-07,
      "loss": 0.18085089325904846,
      "memory(GiB)": 75.26,
      "reward": 2.0128588676452637,
      "reward_std": 0.45990481972694397,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.265625,
      "rewards/MultiModalAccuracyORM/std": 0.4343355745077133,
      "rewards/ReflectionBalance/mean": 0.8214526176452637,
      "rewards/ReflectionBalance/std": 0.22264500707387924,
      "step": 1318,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 519.1796875,
      "completions/min_length": 216.5,
      "epoch": 7.129380053908355,
      "grad_norm": 3.6802817193794994,
      "kl": 2.1640625,
      "learning_rate": 1.9355325187831972e-07,
      "loss": 0.1798938810825348,
      "memory(GiB)": 75.26,
      "reward": 1.9400358200073242,
      "reward_std": 0.5423725843429565,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3270204961299896,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.4407331943511963,
      "rewards/ReflectionBalance/mean": 0.7994107902050018,
      "rewards/ReflectionBalance/std": 0.26154208928346634,
      "step": 1319,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.0,
      "completions/mean_length": 497.37890625,
      "completions/min_length": 267.5,
      "epoch": 7.1347708894878705,
      "grad_norm": 3.5065028893534693,
      "kl": 2.09375,
      "learning_rate": 1.928758282935341e-07,
      "loss": 0.14993375539779663,
      "memory(GiB)": 75.26,
      "reward": 1.9742839336395264,
      "reward_std": 0.5508432537317276,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.42950694262981415,
      "rewards/ReflectionBalance/mean": 0.8102213740348816,
      "rewards/ReflectionBalance/std": 0.25718359649181366,
      "step": 1320,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 497.9921875,
      "completions/min_length": 282.0,
      "epoch": 7.140161725067386,
      "grad_norm": 3.409210534352189,
      "kl": 2.046875,
      "learning_rate": 1.9219930885138285e-07,
      "loss": 0.1910914182662964,
      "memory(GiB)": 75.26,
      "reward": 1.93704754114151,
      "reward_std": 0.46121643483638763,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3752707988023758,
      "rewards/ReflectionBalance/mean": 0.83157879114151,
      "rewards/ReflectionBalance/std": 0.247752383351326,
      "step": 1321,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 479.0859375,
      "completions/min_length": 196.0,
      "epoch": 7.1455525606469,
      "grad_norm": 3.226485854182279,
      "kl": 2.5,
      "learning_rate": 1.9152369554347036e-07,
      "loss": 0.19113823771476746,
      "memory(GiB)": 75.26,
      "reward": 2.021682858467102,
      "reward_std": 0.5137245953083038,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.29296875,
      "rewards/MultiModalAccuracyORM/std": 0.4559154361486435,
      "rewards/ReflectionBalance/mean": 0.8146516978740692,
      "rewards/ReflectionBalance/std": 0.2499179244041443,
      "step": 1322,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1020.5,
      "completions/mean_length": 476.43359375,
      "completions/min_length": 267.0,
      "epoch": 7.150943396226415,
      "grad_norm": 4.21079532492817,
      "kl": 1.796875,
      "learning_rate": 1.9084899035873463e-07,
      "loss": 0.14934101700782776,
      "memory(GiB)": 75.26,
      "reward": 1.9764404296875,
      "reward_std": 0.4688674807548523,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.4150650203227997,
      "rewards/ReflectionBalance/mean": 0.8240966796875,
      "rewards/ReflectionBalance/std": 0.23079269379377365,
      "step": 1323,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 880.0,
      "completions/mean_length": 459.875,
      "completions/min_length": 151.5,
      "epoch": 7.15633423180593,
      "grad_norm": 3.8712164386007766,
      "kl": 2.5,
      "learning_rate": 1.9017519528343974e-07,
      "loss": 0.17402322590351105,
      "memory(GiB)": 75.26,
      "reward": 1.8981640934944153,
      "reward_std": 0.41539812088012695,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2487918585538864,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35358233749866486,
      "rewards/ReflectionBalance/mean": 0.8200390338897705,
      "rewards/ReflectionBalance/std": 0.224089115858078,
      "step": 1324,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 481.95703125,
      "completions/min_length": 138.5,
      "epoch": 7.1617250673854445,
      "grad_norm": 3.38099997999619,
      "kl": 2.4375,
      "learning_rate": 1.895023123011702e-07,
      "loss": 0.169133722782135,
      "memory(GiB)": 75.26,
      "reward": 1.9037134051322937,
      "reward_std": 0.4613294005393982,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.38315413892269135,
      "rewards/ReflectionBalance/mean": 0.7982446253299713,
      "rewards/ReflectionBalance/std": 0.23968086391687393,
      "step": 1325,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 527.37890625,
      "completions/min_length": 121.0,
      "epoch": 7.16711590296496,
      "grad_norm": 4.121624696502965,
      "kl": 2.89453125,
      "learning_rate": 1.8883034339282644e-07,
      "loss": 0.1968962550163269,
      "memory(GiB)": 75.26,
      "reward": 1.8815876841545105,
      "reward_std": 0.5074574500322342,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2829600051045418,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.38125884532928467,
      "rewards/ReflectionBalance/mean": 0.7956502139568329,
      "rewards/ReflectionBalance/std": 0.2698790431022644,
      "step": 1326,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 487.48046875,
      "completions/min_length": 248.5,
      "epoch": 7.172506738544475,
      "grad_norm": 13.955059351982644,
      "kl": 2.65625,
      "learning_rate": 1.881592905366164e-07,
      "loss": 0.20280900597572327,
      "memory(GiB)": 75.26,
      "reward": 1.8977913856506348,
      "reward_std": 0.4464391618967056,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35538628697395325,
      "rewards/ReflectionBalance/mean": 0.8196663558483124,
      "rewards/ReflectionBalance/std": 0.22171606123447418,
      "step": 1327,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 497.2578125,
      "completions/min_length": 270.5,
      "epoch": 7.177897574123989,
      "grad_norm": 10286.485489211242,
      "kl": 29.78125,
      "learning_rate": 1.8748915570805273e-07,
      "loss": 1.5836148262023926,
      "memory(GiB)": 75.26,
      "reward": 1.8755159378051758,
      "reward_std": 0.5336340814828873,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3050640821456909,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38784652948379517,
      "rewards/ReflectionBalance/mean": 0.7973909378051758,
      "rewards/ReflectionBalance/std": 0.2859741896390915,
      "step": 1328,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 487.51171875,
      "completions/min_length": 286.0,
      "epoch": 7.183288409703504,
      "grad_norm": 3.2954060999844565,
      "kl": 3.03125,
      "learning_rate": 1.8681994087994473e-07,
      "loss": 0.24931402504444122,
      "memory(GiB)": 75.26,
      "reward": 1.9974658489227295,
      "reward_std": 0.5873472541570663,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.29317550361156464,
      "rewards/MultiModalAccuracyORM/mean": 0.29296875,
      "rewards/MultiModalAccuracyORM/std": 0.43074773252010345,
      "rewards/ReflectionBalance/mean": 0.8177783191204071,
      "rewards/ReflectionBalance/std": 0.22928770631551743,
      "step": 1329,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 482.13671875,
      "completions/min_length": 261.5,
      "epoch": 7.188679245283019,
      "grad_norm": 3.4273604436089995,
      "kl": 2.5546875,
      "learning_rate": 1.8615164802239325e-07,
      "loss": 0.17392319440841675,
      "memory(GiB)": 75.26,
      "reward": 1.8903189897537231,
      "reward_std": 0.49433064460754395,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.378810778260231,
      "rewards/ReflectionBalance/mean": 0.8082877397537231,
      "rewards/ReflectionBalance/std": 0.23959801346063614,
      "step": 1330,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 513.1484375,
      "completions/min_length": 275.0,
      "epoch": 7.1940700808625335,
      "grad_norm": 4.773938237901458,
      "kl": 3.65625,
      "learning_rate": 1.8548427910278552e-07,
      "loss": 0.2575967609882355,
      "memory(GiB)": 75.26,
      "reward": 1.793664574623108,
      "reward_std": 0.5521959960460663,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3318064510822296,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3290177956223488,
      "rewards/ReflectionBalance/mean": 0.7780395448207855,
      "rewards/ReflectionBalance/std": 0.28269582986831665,
      "step": 1331,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 505.2109375,
      "completions/min_length": 263.0,
      "epoch": 7.199460916442049,
      "grad_norm": 5.604008072988432,
      "kl": 2.53125,
      "learning_rate": 1.8481783608578826e-07,
      "loss": 0.2001911997795105,
      "memory(GiB)": 75.26,
      "reward": 1.9545669555664062,
      "reward_std": 0.4647527039051056,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.41096827387809753,
      "rewards/ReflectionBalance/mean": 0.813942015171051,
      "rewards/ReflectionBalance/std": 0.22326766699552536,
      "step": 1332,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 489.94140625,
      "completions/min_length": 136.0,
      "epoch": 7.204851752021563,
      "grad_norm": 3.084456666811237,
      "kl": 2.640625,
      "learning_rate": 1.8415232093334236e-07,
      "loss": 0.2171257734298706,
      "memory(GiB)": 75.26,
      "reward": 1.9289274215698242,
      "reward_std": 0.464589923620224,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41699807345867157,
      "rewards/ReflectionBalance/mean": 0.8078336715698242,
      "rewards/ReflectionBalance/std": 0.2668912410736084,
      "step": 1333,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 488.14453125,
      "completions/min_length": 278.5,
      "epoch": 7.210242587601078,
      "grad_norm": 6.007946243343818,
      "kl": 1.96875,
      "learning_rate": 1.834877356046579e-07,
      "loss": 0.14878061413764954,
      "memory(GiB)": 75.26,
      "reward": 1.974118709564209,
      "reward_std": 0.4412483274936676,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42526112496852875,
      "rewards/ReflectionBalance/mean": 0.8100561499595642,
      "rewards/ReflectionBalance/std": 0.2586629241704941,
      "step": 1334,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 498.1640625,
      "completions/min_length": 298.5,
      "epoch": 7.215633423180593,
      "grad_norm": 3.5990798132887556,
      "kl": 1.94921875,
      "learning_rate": 1.828240820562069e-07,
      "loss": 0.14051152765750885,
      "memory(GiB)": 75.26,
      "reward": 1.9430192112922668,
      "reward_std": 0.37924134731292725,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.2156769335269928,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.37402482330799103,
      "rewards/ReflectionBalance/mean": 0.8141129016876221,
      "rewards/ReflectionBalance/std": 0.257848396897316,
      "step": 1335,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 509.68359375,
      "completions/min_length": 294.0,
      "epoch": 7.2210242587601075,
      "grad_norm": 3.1178545727558324,
      "kl": 2.3359375,
      "learning_rate": 1.8216136224171856e-07,
      "loss": 0.18781669437885284,
      "memory(GiB)": 75.26,
      "reward": 1.9639111757278442,
      "reward_std": 0.4780984967947006,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4120787978172302,
      "rewards/ReflectionBalance/mean": 0.8271923661231995,
      "rewards/ReflectionBalance/std": 0.23980707675218582,
      "step": 1336,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 490.33203125,
      "completions/min_length": 215.5,
      "epoch": 7.226415094339623,
      "grad_norm": 2.870483027555212,
      "kl": 2.6328125,
      "learning_rate": 1.814995781121732e-07,
      "loss": 0.18939515948295593,
      "memory(GiB)": 75.26,
      "reward": 2.02675461769104,
      "reward_std": 0.5204677283763885,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.3046875,
      "rewards/MultiModalAccuracyORM/std": 0.4620049446821213,
      "rewards/ReflectionBalance/mean": 0.8158170282840729,
      "rewards/ReflectionBalance/std": 0.21274830400943756,
      "step": 1337,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 494.19140625,
      "completions/min_length": 233.5,
      "epoch": 7.231805929919138,
      "grad_norm": 3.4502692789877347,
      "kl": 2.6328125,
      "learning_rate": 1.8083873161579687e-07,
      "loss": 0.19545620679855347,
      "memory(GiB)": 75.26,
      "reward": 1.8913509249687195,
      "reward_std": 0.4861401319503784,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3050640821456909,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3854355216026306,
      "rewards/ReflectionBalance/mean": 0.8171321749687195,
      "rewards/ReflectionBalance/std": 0.24106699228286743,
      "step": 1338,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 517.234375,
      "completions/min_length": 299.0,
      "epoch": 7.237196765498652,
      "grad_norm": 3.3131357496458564,
      "kl": 2.359375,
      "learning_rate": 1.80178824698055e-07,
      "loss": 0.19474934041500092,
      "memory(GiB)": 75.26,
      "reward": 1.8596175909042358,
      "reward_std": 0.5019479393959045,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.34826864302158356,
      "rewards/ReflectionBalance/mean": 0.8010237514972687,
      "rewards/ReflectionBalance/std": 0.2995784878730774,
      "step": 1339,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1016.0,
      "completions/mean_length": 486.734375,
      "completions/min_length": 244.0,
      "epoch": 7.242587601078167,
      "grad_norm": 3.354328134606025,
      "kl": 1.8359375,
      "learning_rate": 1.79519859301647e-07,
      "loss": 0.1760859489440918,
      "memory(GiB)": 75.26,
      "reward": 2.0187093019485474,
      "reward_std": 0.42117419838905334,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.42950694262981415,
      "rewards/ReflectionBalance/mean": 0.8390218019485474,
      "rewards/ReflectionBalance/std": 0.18253551423549652,
      "step": 1340,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 475.51953125,
      "completions/min_length": 258.0,
      "epoch": 7.247978436657682,
      "grad_norm": 3.001291094479151,
      "kl": 2.15234375,
      "learning_rate": 1.78861837366501e-07,
      "loss": 0.16373322904109955,
      "memory(GiB)": 75.26,
      "reward": 1.9935896396636963,
      "reward_std": 0.3596459776163101,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2280246615409851,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.39356333017349243,
      "rewards/ReflectionBalance/mean": 0.8451521396636963,
      "rewards/ReflectionBalance/std": 0.1874227374792099,
      "step": 1341,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 494.30859375,
      "completions/min_length": 231.5,
      "epoch": 7.2533692722371965,
      "grad_norm": 4.5596982196414935,
      "kl": 3.4609375,
      "learning_rate": 1.7820476082976677e-07,
      "loss": 0.2607671320438385,
      "memory(GiB)": 75.26,
      "reward": 1.85623699426651,
      "reward_std": 0.5554766654968262,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3300945907831192,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.39601969718933105,
      "rewards/ReflectionBalance/mean": 0.7859244644641876,
      "rewards/ReflectionBalance/std": 0.30151042342185974,
      "step": 1342,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 502.87109375,
      "completions/min_length": 235.0,
      "epoch": 7.258760107816712,
      "grad_norm": 3.442116701271029,
      "kl": 1.7578125,
      "learning_rate": 1.775486316258119e-07,
      "loss": 0.15025408565998077,
      "memory(GiB)": 75.26,
      "reward": 1.9057551622390747,
      "reward_std": 0.4619295597076416,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35872696340084076,
      "rewards/ReflectionBalance/mean": 0.8276302218437195,
      "rewards/ReflectionBalance/std": 0.21891947090625763,
      "step": 1343,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 501.234375,
      "completions/min_length": 268.0,
      "epoch": 7.264150943396227,
      "grad_norm": 3.0451031721291972,
      "kl": 2.4296875,
      "learning_rate": 1.768934516862146e-07,
      "loss": 0.20220626890659332,
      "memory(GiB)": 75.26,
      "reward": 1.8987063765525818,
      "reward_std": 0.534910261631012,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3940746486186981,
      "rewards/ReflectionBalance/mean": 0.8049563765525818,
      "rewards/ReflectionBalance/std": 0.26540276408195496,
      "step": 1344,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 516.59375,
      "completions/min_length": 247.5,
      "epoch": 7.269541778975741,
      "grad_norm": 3.1867093801330038,
      "kl": 2.3828125,
      "learning_rate": 1.7623922293975834e-07,
      "loss": 0.21824471652507782,
      "memory(GiB)": 75.26,
      "reward": 1.8579297065734863,
      "reward_std": 0.45368942618370056,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.33063435554504395,
      "rewards/ReflectionBalance/mean": 0.8032421767711639,
      "rewards/ReflectionBalance/std": 0.23049746453762054,
      "step": 1345,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 498.55859375,
      "completions/min_length": 270.0,
      "epoch": 7.274932614555256,
      "grad_norm": 4.015013728601335,
      "kl": 1.71875,
      "learning_rate": 1.7558594731242716e-07,
      "loss": 0.14325016736984253,
      "memory(GiB)": 75.26,
      "reward": 1.93954598903656,
      "reward_std": 0.44277726113796234,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.4002292603254318,
      "rewards/ReflectionBalance/mean": 0.8106396496295929,
      "rewards/ReflectionBalance/std": 0.24514570087194443,
      "step": 1346,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 470.5,
      "completions/min_length": 230.5,
      "epoch": 7.280323450134771,
      "grad_norm": 3.0114737756196006,
      "kl": 2.1328125,
      "learning_rate": 1.7493362672739843e-07,
      "loss": 0.18231362104415894,
      "memory(GiB)": 75.26,
      "reward": 1.9074430465698242,
      "reward_std": 0.4764765053987503,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.38125884532928467,
      "rewards/ReflectionBalance/mean": 0.8097867667675018,
      "rewards/ReflectionBalance/std": 0.252509705722332,
      "step": 1347,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 882.0,
      "completions/mean_length": 460.87109375,
      "completions/min_length": 231.0,
      "epoch": 7.285714285714286,
      "grad_norm": 2.8795503958676716,
      "kl": 1.62109375,
      "learning_rate": 1.742822631050383e-07,
      "loss": 0.11984112858772278,
      "memory(GiB)": 75.26,
      "reward": 2.0747753381729126,
      "reward_std": 0.3802599757909775,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.2379390001296997,
      "rewards/MultiModalAccuracyORM/mean": 0.3046875,
      "rewards/MultiModalAccuracyORM/std": 0.460112988948822,
      "rewards/ReflectionBalance/mean": 0.8325878977775574,
      "rewards/ReflectionBalance/std": 0.21259647607803345,
      "step": 1348,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 485.76953125,
      "completions/min_length": 283.0,
      "epoch": 7.291105121293801,
      "grad_norm": 2.7529880599650864,
      "kl": 2.59375,
      "learning_rate": 1.7363185836289567e-07,
      "loss": 0.20428860187530518,
      "memory(GiB)": 75.26,
      "reward": 1.9288947582244873,
      "reward_std": 0.44255292415618896,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27079907804727554,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.42152732610702515,
      "rewards/ReflectionBalance/mean": 0.7804573476314545,
      "rewards/ReflectionBalance/std": 0.29230090230703354,
      "step": 1349,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 484.375,
      "completions/min_length": 254.5,
      "epoch": 7.296495956873315,
      "grad_norm": 2.9522411004433495,
      "kl": 1.875,
      "learning_rate": 1.729824144156965e-07,
      "loss": 0.13796406984329224,
      "memory(GiB)": 75.26,
      "reward": 1.9888280630111694,
      "reward_std": 0.43408074975013733,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.253920242190361,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42764075100421906,
      "rewards/ReflectionBalance/mean": 0.8208593726158142,
      "rewards/ReflectionBalance/std": 0.2269277200102806,
      "step": 1350,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 477.70703125,
      "completions/min_length": 286.5,
      "epoch": 7.30188679245283,
      "grad_norm": 3.320663843980118,
      "kl": 2.0859375,
      "learning_rate": 1.723339331753386e-07,
      "loss": 0.18342512845993042,
      "memory(GiB)": 75.26,
      "reward": 1.8644953966140747,
      "reward_std": 0.4048251807689667,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3389271944761276,
      "rewards/ReflectionBalance/mean": 0.8176203966140747,
      "rewards/ReflectionBalance/std": 0.23148707300424576,
      "step": 1351,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 488.8671875,
      "completions/min_length": 258.5,
      "epoch": 7.307277628032345,
      "grad_norm": 2.926565957083692,
      "kl": 2.6640625,
      "learning_rate": 1.716864165508854e-07,
      "loss": 0.21983736753463745,
      "memory(GiB)": 75.26,
      "reward": 1.997262418270111,
      "reward_std": 0.5707214176654816,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2875154986977577,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.44712863862514496,
      "rewards/ReflectionBalance/mean": 0.8175748586654663,
      "rewards/ReflectionBalance/std": 0.24265997111797333,
      "step": 1352,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 476.8515625,
      "completions/min_length": 210.5,
      "epoch": 7.3126684636118595,
      "grad_norm": 3.715761513570307,
      "kl": 2.10546875,
      "learning_rate": 1.7103986644856056e-07,
      "loss": 0.16023652255535126,
      "memory(GiB)": 75.26,
      "reward": 1.8778434991836548,
      "reward_std": 0.4909994602203369,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2801405265927315,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3679620176553726,
      "rewards/ReflectionBalance/mean": 0.80362468957901,
      "rewards/ReflectionBalance/std": 0.2524215430021286,
      "step": 1353,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 493.95703125,
      "completions/min_length": 281.5,
      "epoch": 7.318059299191375,
      "grad_norm": 2.9926952503052,
      "kl": 2.484375,
      "learning_rate": 1.703942847717423e-07,
      "loss": 0.21978990733623505,
      "memory(GiB)": 75.26,
      "reward": 1.8007519245147705,
      "reward_std": 0.4741414785385132,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.08203125,
      "rewards/MultiModalAccuracyORM/std": 0.275397852063179,
      "rewards/ReflectionBalance/mean": 0.8202832341194153,
      "rewards/ReflectionBalance/std": 0.2542329430580139,
      "step": 1354,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 471.015625,
      "completions/min_length": 237.5,
      "epoch": 7.32345013477089,
      "grad_norm": 3.1017504806666176,
      "kl": 2.859375,
      "learning_rate": 1.697496734209584e-07,
      "loss": 0.258367657661438,
      "memory(GiB)": 75.26,
      "reward": 2.0225536227226257,
      "reward_std": 0.5631612539291382,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.328125,
      "rewards/MultiModalAccuracyORM/std": 0.47107939422130585,
      "rewards/ReflectionBalance/mean": 0.7998974621295929,
      "rewards/ReflectionBalance/std": 0.2994464635848999,
      "step": 1355,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 467.21875,
      "completions/min_length": 267.0,
      "epoch": 7.328840970350404,
      "grad_norm": 2.959782309871927,
      "kl": 1.7890625,
      "learning_rate": 1.6910603429387964e-07,
      "loss": 0.14798520505428314,
      "memory(GiB)": 75.26,
      "reward": 2.0742350816726685,
      "reward_std": 0.42825818061828613,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22826264798641205,
      "rewards/MultiModalAccuracyORM/mean": 0.29296875,
      "rewards/MultiModalAccuracyORM/std": 0.4567296952009201,
      "rewards/ReflectionBalance/mean": 0.8359537720680237,
      "rewards/ReflectionBalance/std": 0.20117606222629547,
      "step": 1356,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 487.30859375,
      "completions/min_length": 216.0,
      "epoch": 7.334231805929919,
      "grad_norm": 2.7997164649132986,
      "kl": 1.859375,
      "learning_rate": 1.6846336928531464e-07,
      "loss": 0.1493268609046936,
      "memory(GiB)": 75.26,
      "reward": 2.0056836009025574,
      "reward_std": 0.45542867481708527,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4283375144004822,
      "rewards/ReflectionBalance/mean": 0.833808571100235,
      "rewards/ReflectionBalance/std": 0.21696452051401138,
      "step": 1357,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 488.51171875,
      "completions/min_length": 264.5,
      "epoch": 7.339622641509434,
      "grad_norm": 3.058039323372885,
      "kl": 3.375,
      "learning_rate": 1.678216802872049e-07,
      "loss": 0.2577354907989502,
      "memory(GiB)": 75.26,
      "reward": 1.9461009502410889,
      "reward_std": 0.5392521470785141,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.43047651648521423,
      "rewards/ReflectionBalance/mean": 0.8054759204387665,
      "rewards/ReflectionBalance/std": 0.2715574651956558,
      "step": 1358,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 506.9921875,
      "completions/min_length": 277.5,
      "epoch": 7.345013477088949,
      "grad_norm": 83.69670640712626,
      "kl": 3.3359375,
      "learning_rate": 1.6718096918861752e-07,
      "loss": 0.24547728896141052,
      "memory(GiB)": 75.26,
      "reward": 1.920762538909912,
      "reward_std": 0.5074734538793564,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.300025038421154,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.39705096185207367,
      "rewards/ReflectionBalance/mean": 0.8152937889099121,
      "rewards/ReflectionBalance/std": 0.2643677592277527,
      "step": 1359,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 484.48828125,
      "completions/min_length": 257.5,
      "epoch": 7.350404312668464,
      "grad_norm": 3.44689747839169,
      "kl": 3.765625,
      "learning_rate": 1.665412378757421e-07,
      "loss": 0.2940472960472107,
      "memory(GiB)": 75.26,
      "reward": 1.8567562699317932,
      "reward_std": 0.5020028054714203,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35062892735004425,
      "rewards/ReflectionBalance/mean": 0.8137874603271484,
      "rewards/ReflectionBalance/std": 0.2448279932141304,
      "step": 1360,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 508.73046875,
      "completions/min_length": 221.0,
      "epoch": 7.355795148247978,
      "grad_norm": 4.438436223362302,
      "kl": 3.6640625,
      "learning_rate": 1.6590248823188291e-07,
      "loss": 0.2769192159175873,
      "memory(GiB)": 75.26,
      "reward": 1.856540024280548,
      "reward_std": 0.5655320882797241,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3219822645187378,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38597656786441803,
      "rewards/ReflectionBalance/mean": 0.7901338040828705,
      "rewards/ReflectionBalance/std": 0.2894497364759445,
      "step": 1361,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 457.484375,
      "completions/min_length": 248.5,
      "epoch": 7.361185983827493,
      "grad_norm": 3.689141919605127,
      "kl": 2.58203125,
      "learning_rate": 1.6526472213745446e-07,
      "loss": 0.20566609501838684,
      "memory(GiB)": 75.26,
      "reward": 2.0115413665771484,
      "reward_std": 0.49030835926532745,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.265625,
      "rewards/MultiModalAccuracyORM/std": 0.4401783347129822,
      "rewards/ReflectionBalance/mean": 0.8201351165771484,
      "rewards/ReflectionBalance/std": 0.25768591463565826,
      "step": 1362,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 489.734375,
      "completions/min_length": 227.5,
      "epoch": 7.366576819407008,
      "grad_norm": 5.181360825639693,
      "kl": 3.3828125,
      "learning_rate": 1.646279414699762e-07,
      "loss": 0.25882384181022644,
      "memory(GiB)": 75.26,
      "reward": 1.8198713064193726,
      "reward_std": 0.5868701636791229,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33513282239437103,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3679620176553726,
      "rewards/ReflectionBalance/mean": 0.7847151756286621,
      "rewards/ReflectionBalance/std": 0.25670410692691803,
      "step": 1363,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 480.74609375,
      "completions/min_length": 240.5,
      "epoch": 7.3719676549865225,
      "grad_norm": 4.077675918816781,
      "kl": 3.65625,
      "learning_rate": 1.639921481040662e-07,
      "loss": 0.2943146824836731,
      "memory(GiB)": 75.26,
      "reward": 2.0001497864723206,
      "reward_std": 0.5210727453231812,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.28125,
      "rewards/MultiModalAccuracyORM/std": 0.4506162703037262,
      "rewards/ReflectionBalance/mean": 0.8087434768676758,
      "rewards/ReflectionBalance/std": 0.28371575474739075,
      "step": 1364,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 482.140625,
      "completions/min_length": 259.5,
      "epoch": 7.377358490566038,
      "grad_norm": 2.8849529101264033,
      "kl": 2.62890625,
      "learning_rate": 1.6335734391143585e-07,
      "loss": 0.21936358511447906,
      "memory(GiB)": 75.26,
      "reward": 1.8835140466690063,
      "reward_std": 0.5140093117952347,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2524137645959854,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3704231530427933,
      "rewards/ReflectionBalance/mean": 0.7897639870643616,
      "rewards/ReflectionBalance/std": 0.3005535453557968,
      "step": 1365,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 513.7109375,
      "completions/min_length": 250.5,
      "epoch": 7.382749326145553,
      "grad_norm": 4.692342214956903,
      "kl": 4.390625,
      "learning_rate": 1.6272353076088518e-07,
      "loss": 0.34377700090408325,
      "memory(GiB)": 75.26,
      "reward": 1.7472738027572632,
      "reward_std": 0.6550470888614655,
      "rewards/Format/mean": 0.81640625,
      "rewards/Format/std": 0.3846261203289032,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3774644732475281,
      "rewards/ReflectionBalance/mean": 0.7589924931526184,
      "rewards/ReflectionBalance/std": 0.3183806836605072,
      "step": 1366,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1033.0,
      "completions/mean_length": 510.5234375,
      "completions/min_length": 270.5,
      "epoch": 7.388140161725067,
      "grad_norm": 3.5269599952045,
      "kl": 4.609375,
      "learning_rate": 1.620907105182961e-07,
      "loss": 0.3693876564502716,
      "memory(GiB)": 75.26,
      "reward": 1.892475664615631,
      "reward_std": 0.5990841686725616,
      "rewards/Format/mean": 0.85546875,
      "rewards/Format/std": 0.35190291702747345,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.45073302090168,
      "rewards/ReflectionBalance/mean": 0.7518506050109863,
      "rewards/ReflectionBalance/std": 0.3540215939283371,
      "step": 1367,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 502.23828125,
      "completions/min_length": 274.5,
      "epoch": 7.393530997304582,
      "grad_norm": 3.2728430538728666,
      "kl": 2.8046875,
      "learning_rate": 1.6145888504662762e-07,
      "loss": 0.21543407440185547,
      "memory(GiB)": 75.26,
      "reward": 1.8477832078933716,
      "reward_std": 0.45910757780075073,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3645188808441162,
      "rewards/ReflectionBalance/mean": 0.7891894578933716,
      "rewards/ReflectionBalance/std": 0.2813098430633545,
      "step": 1368,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 503.14453125,
      "completions/min_length": 147.5,
      "epoch": 7.398921832884097,
      "grad_norm": 2.9343269343125638,
      "kl": 2.8046875,
      "learning_rate": 1.6082805620591022e-07,
      "loss": 0.2276272475719452,
      "memory(GiB)": 75.26,
      "reward": 1.90012127161026,
      "reward_std": 0.47646452486515045,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2829600051045418,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3846261203289032,
      "rewards/ReflectionBalance/mean": 0.8063712418079376,
      "rewards/ReflectionBalance/std": 0.2420836240053177,
      "step": 1369,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 505.4765625,
      "completions/min_length": 307.0,
      "epoch": 7.404312668463612,
      "grad_norm": 3.421997721974365,
      "kl": 2.1015625,
      "learning_rate": 1.6019822585324078e-07,
      "loss": 0.18648386001586914,
      "memory(GiB)": 75.26,
      "reward": 1.8492724895477295,
      "reward_std": 0.42862360179424286,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2566775679588318,
      "rewards/MultiModalAccuracyORM/mean": 0.09375,
      "rewards/MultiModalAccuracyORM/std": 0.29137909412384033,
      "rewards/ReflectionBalance/mean": 0.8258349597454071,
      "rewards/ReflectionBalance/std": 0.2531224936246872,
      "step": 1370,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 485.4140625,
      "completions/min_length": 249.0,
      "epoch": 7.409703504043127,
      "grad_norm": 3.551042122991471,
      "kl": 1.50390625,
      "learning_rate": 1.5956939584277618e-07,
      "loss": 0.15948602557182312,
      "memory(GiB)": 75.26,
      "reward": 1.9813119173049927,
      "reward_std": 0.388389527797699,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22826264798641205,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39110109210014343,
      "rewards/ReflectionBalance/mean": 0.8445931077003479,
      "rewards/ReflectionBalance/std": 0.19302216172218323,
      "step": 1371,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 515.85546875,
      "completions/min_length": 327.0,
      "epoch": 7.415094339622642,
      "grad_norm": 3.541070389895742,
      "kl": 2.11328125,
      "learning_rate": 1.5894156802572862e-07,
      "loss": 0.15591786801815033,
      "memory(GiB)": 75.26,
      "reward": 1.955314040184021,
      "reward_std": 0.4634208381175995,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.41966044902801514,
      "rewards/ReflectionBalance/mean": 0.8264078795909882,
      "rewards/ReflectionBalance/std": 0.264285609126091,
      "step": 1372,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 522.75390625,
      "completions/min_length": 271.0,
      "epoch": 7.420485175202156,
      "grad_norm": 3.037010741267506,
      "kl": 1.421875,
      "learning_rate": 1.583147442503604e-07,
      "loss": 0.10613328218460083,
      "memory(GiB)": 75.26,
      "reward": 1.9022184610366821,
      "reward_std": 0.42191918194293976,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.35358233749866486,
      "rewards/ReflectionBalance/mean": 0.8397184312343597,
      "rewards/ReflectionBalance/std": 0.21882152557373047,
      "step": 1373,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 506.4140625,
      "completions/min_length": 270.0,
      "epoch": 7.425876010781671,
      "grad_norm": 3.5013790748476676,
      "kl": 1.8046875,
      "learning_rate": 1.5768892636197694e-07,
      "loss": 0.12759917974472046,
      "memory(GiB)": 75.26,
      "reward": 1.86343914270401,
      "reward_std": 0.47541898488998413,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3098885118961334,
      "rewards/ReflectionBalance/mean": 0.81265789270401,
      "rewards/ReflectionBalance/std": 0.23130282014608383,
      "step": 1374,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1020.5,
      "completions/mean_length": 463.73046875,
      "completions/min_length": 266.5,
      "epoch": 7.431266846361186,
      "grad_norm": 3.3212979502560733,
      "kl": 1.61328125,
      "learning_rate": 1.5706411620292358e-07,
      "loss": 0.13876676559448242,
      "memory(GiB)": 75.26,
      "reward": 2.0290609002113342,
      "reward_std": 0.49636903405189514,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.22023213654756546,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.43684960901737213,
      "rewards/ReflectionBalance/mean": 0.8220295906066895,
      "rewards/ReflectionBalance/std": 0.23762677609920502,
      "step": 1375,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 463.6640625,
      "completions/min_length": 243.5,
      "epoch": 7.436657681940701,
      "grad_norm": 2.957724374482299,
      "kl": 2.015625,
      "learning_rate": 1.5644031561257838e-07,
      "loss": 0.17108914256095886,
      "memory(GiB)": 75.26,
      "reward": 2.036606550216675,
      "reward_std": 0.49487242102622986,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26276856660842896,
      "rewards/MultiModalAccuracyORM/mean": 0.3046875,
      "rewards/MultiModalAccuracyORM/std": 0.4541175067424774,
      "rewards/ReflectionBalance/mean": 0.8100439608097076,
      "rewards/ReflectionBalance/std": 0.24943982064723969,
      "step": 1376,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 466.33984375,
      "completions/min_length": 266.0,
      "epoch": 7.442048517520216,
      "grad_norm": 3.693287188234897,
      "kl": 2.08984375,
      "learning_rate": 1.5581752642734718e-07,
      "loss": 0.16424405574798584,
      "memory(GiB)": 75.26,
      "reward": 1.8281662464141846,
      "reward_std": 0.36505086719989777,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24678732454776764,
      "rewards/MultiModalAccuracyORM/mean": 0.07421875,
      "rewards/MultiModalAccuracyORM/std": 0.232606191188097,
      "rewards/ReflectionBalance/mean": 0.8203536570072174,
      "rewards/ReflectionBalance/std": 0.23367993533611298,
      "step": 1377,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 495.609375,
      "completions/min_length": 149.5,
      "epoch": 7.44743935309973,
      "grad_norm": 3.2586003514839694,
      "kl": 2.03515625,
      "learning_rate": 1.5519575048065925e-07,
      "loss": 0.17785632610321045,
      "memory(GiB)": 75.26,
      "reward": 1.9530858993530273,
      "reward_std": 0.4356889873743057,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38784652948379517,
      "rewards/ReflectionBalance/mean": 0.8358984291553497,
      "rewards/ReflectionBalance/std": 0.22353292256593704,
      "step": 1378,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 475.37890625,
      "completions/min_length": 261.0,
      "epoch": 7.452830188679245,
      "grad_norm": 4.409803524634953,
      "kl": 1.796875,
      "learning_rate": 1.5457498960295967e-07,
      "loss": 0.18957778811454773,
      "memory(GiB)": 75.26,
      "reward": 1.9852782487869263,
      "reward_std": 0.496682345867157,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4285898506641388,
      "rewards/ReflectionBalance/mean": 0.813403308391571,
      "rewards/ReflectionBalance/std": 0.25813572108745575,
      "step": 1379,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 499.1875,
      "completions/min_length": 266.0,
      "epoch": 7.45822102425876,
      "grad_norm": 3.2669003120314883,
      "kl": 2.4296875,
      "learning_rate": 1.5395524562170643e-07,
      "loss": 0.19747522473335266,
      "memory(GiB)": 75.26,
      "reward": 1.9254231452941895,
      "reward_std": 0.552087739109993,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2829600051045418,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4234771430492401,
      "rewards/ReflectionBalance/mean": 0.7769856452941895,
      "rewards/ReflectionBalance/std": 0.3244166225194931,
      "step": 1380,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 493.05078125,
      "completions/min_length": 264.5,
      "epoch": 7.463611859838275,
      "grad_norm": 3.1916731597097296,
      "kl": 2.6015625,
      "learning_rate": 1.5333652036136318e-07,
      "loss": 0.21884402632713318,
      "memory(GiB)": 75.26,
      "reward": 1.8577978610992432,
      "reward_std": 0.4881688952445984,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2801405265927315,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35062892735004425,
      "rewards/ReflectionBalance/mean": 0.7992041110992432,
      "rewards/ReflectionBalance/std": 0.28769490867853165,
      "step": 1381,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 912.5,
      "completions/mean_length": 480.4296875,
      "completions/min_length": 283.5,
      "epoch": 7.46900269541779,
      "grad_norm": 2.820987448591115,
      "kl": 1.65234375,
      "learning_rate": 1.5271881564339474e-07,
      "loss": 0.13417428731918335,
      "memory(GiB)": 75.26,
      "reward": 1.9131901264190674,
      "reward_std": 0.38617420196533203,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.370050385594368,
      "rewards/ReflectionBalance/mean": 0.803815096616745,
      "rewards/ReflectionBalance/std": 0.2370864525437355,
      "step": 1382,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 465.25390625,
      "completions/min_length": 260.5,
      "epoch": 7.474393530997305,
      "grad_norm": 3.6010381889631,
      "kl": 1.671875,
      "learning_rate": 1.5210213328626188e-07,
      "loss": 0.15215229988098145,
      "memory(GiB)": 75.26,
      "reward": 2.0767756700515747,
      "reward_std": 0.45450419187545776,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.3125,
      "rewards/MultiModalAccuracyORM/std": 0.46154065430164337,
      "rewards/ReflectionBalance/mean": 0.8345882296562195,
      "rewards/ReflectionBalance/std": 0.2181272655725479,
      "step": 1383,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 493.4609375,
      "completions/min_length": 253.5,
      "epoch": 7.479784366576819,
      "grad_norm": 3.13720295965897,
      "kl": 2.515625,
      "learning_rate": 1.5148647510541512e-07,
      "loss": 0.19474901258945465,
      "memory(GiB)": 75.26,
      "reward": 1.846254825592041,
      "reward_std": 0.4543720632791519,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2676052302122116,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3437131494283676,
      "rewards/ReflectionBalance/mean": 0.7954736053943634,
      "rewards/ReflectionBalance/std": 0.29184678941965103,
      "step": 1384,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 490.234375,
      "completions/min_length": 238.5,
      "epoch": 7.485175202156334,
      "grad_norm": 3.156959106925666,
      "kl": 2.390625,
      "learning_rate": 1.5087184291329026e-07,
      "loss": 0.18864746391773224,
      "memory(GiB)": 75.26,
      "reward": 1.9091349244117737,
      "reward_std": 0.5379095673561096,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3877098113298416,
      "rewards/ReflectionBalance/mean": 0.8036661744117737,
      "rewards/ReflectionBalance/std": 0.2434564307332039,
      "step": 1385,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 475.96484375,
      "completions/min_length": 261.0,
      "epoch": 7.490566037735849,
      "grad_norm": 2.960235609015125,
      "kl": 1.88671875,
      "learning_rate": 1.502582385193023e-07,
      "loss": 0.16335634887218475,
      "memory(GiB)": 75.26,
      "reward": 1.9500471949577332,
      "reward_std": 0.3791225999593735,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.21875878423452377,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3602610230445862,
      "rewards/ReflectionBalance/mean": 0.8289534449577332,
      "rewards/ReflectionBalance/std": 0.18562430888414383,
      "step": 1386,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 536.18359375,
      "completions/min_length": 165.0,
      "epoch": 7.495956873315364,
      "grad_norm": 3.1580072125420604,
      "kl": 2.84375,
      "learning_rate": 1.496456637298412e-07,
      "loss": 0.23388206958770752,
      "memory(GiB)": 75.26,
      "reward": 1.8057901859283447,
      "reward_std": 0.5083139538764954,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3270204961299896,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3240884244441986,
      "rewards/ReflectionBalance/mean": 0.8018839359283447,
      "rewards/ReflectionBalance/std": 0.2597740888595581,
      "step": 1387,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 485.33984375,
      "completions/min_length": 275.0,
      "epoch": 7.501347708894879,
      "grad_norm": 3.832836772273543,
      "kl": 2.859375,
      "learning_rate": 1.490341203482653e-07,
      "loss": 0.24435222148895264,
      "memory(GiB)": 75.26,
      "reward": 1.94558584690094,
      "reward_std": 0.5840531885623932,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4168776273727417,
      "rewards/ReflectionBalance/mean": 0.7971484065055847,
      "rewards/ReflectionBalance/std": 0.27098116278648376,
      "step": 1388,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 493.6640625,
      "completions/min_length": 293.5,
      "epoch": 7.506738544474393,
      "grad_norm": 2.9727962872051354,
      "kl": 2.8984375,
      "learning_rate": 1.4842361017489647e-07,
      "loss": 0.22690945863723755,
      "memory(GiB)": 75.26,
      "reward": 1.8722217082977295,
      "reward_std": 0.506012573838234,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38489311933517456,
      "rewards/ReflectionBalance/mean": 0.8019091784954071,
      "rewards/ReflectionBalance/std": 0.2749498188495636,
      "step": 1389,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.5,
      "completions/mean_length": 499.05859375,
      "completions/min_length": 253.5,
      "epoch": 7.512129380053908,
      "grad_norm": 4.9037082860076735,
      "kl": 3.3671875,
      "learning_rate": 1.4781413500701578e-07,
      "loss": 0.2584259808063507,
      "memory(GiB)": 75.26,
      "reward": 1.7990186214447021,
      "reward_std": 0.4499986320734024,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2918590381741524,
      "rewards/MultiModalAccuracyORM/mean": 0.08984375,
      "rewards/MultiModalAccuracyORM/std": 0.2869994640350342,
      "rewards/ReflectionBalance/mean": 0.8068310618400574,
      "rewards/ReflectionBalance/std": 0.24413029849529266,
      "step": 1390,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 496.65234375,
      "completions/min_length": 260.0,
      "epoch": 7.517520215633423,
      "grad_norm": 3.322925733465678,
      "kl": 2.90625,
      "learning_rate": 1.47205696638856e-07,
      "loss": 0.23816975951194763,
      "memory(GiB)": 75.26,
      "reward": 1.8529777526855469,
      "reward_std": 0.4702102243900299,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.298691526055336,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3645188808441162,
      "rewards/ReflectionBalance/mean": 0.7982902228832245,
      "rewards/ReflectionBalance/std": 0.2600891664624214,
      "step": 1391,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 487.7109375,
      "completions/min_length": 224.5,
      "epoch": 7.5229110512129385,
      "grad_norm": 3.513750326272262,
      "kl": 2.96875,
      "learning_rate": 1.4659829686159886e-07,
      "loss": 0.22261899709701538,
      "memory(GiB)": 75.26,
      "reward": 1.8545637726783752,
      "reward_std": 0.5845808982849121,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3320184051990509,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.37864522635936737,
      "rewards/ReflectionBalance/mean": 0.79987633228302,
      "rewards/ReflectionBalance/std": 0.2633695900440216,
      "step": 1392,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 528.77734375,
      "completions/min_length": 218.0,
      "epoch": 7.528301886792453,
      "grad_norm": 2.897358850421987,
      "kl": 2.5234375,
      "learning_rate": 1.4599193746336835e-07,
      "loss": 0.18368938565254211,
      "memory(GiB)": 75.26,
      "reward": 1.8560107350349426,
      "reward_std": 0.5274397879838943,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3734995424747467,
      "rewards/ReflectionBalance/mean": 0.8013232350349426,
      "rewards/ReflectionBalance/std": 0.281659297645092,
      "step": 1393,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 511.31640625,
      "completions/min_length": 264.5,
      "epoch": 7.533692722371968,
      "grad_norm": 2.985318019584598,
      "kl": 2.2109375,
      "learning_rate": 1.4538662022922493e-07,
      "loss": 0.17769181728363037,
      "memory(GiB)": 75.26,
      "reward": 1.9322136640548706,
      "reward_std": 0.4551359564065933,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.382988765835762,
      "rewards/ReflectionBalance/mean": 0.8345572650432587,
      "rewards/ReflectionBalance/std": 0.20929696410894394,
      "step": 1394,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 477.56640625,
      "completions/min_length": 183.0,
      "epoch": 7.539083557951482,
      "grad_norm": 2.9087470704872684,
      "kl": 2.10546875,
      "learning_rate": 1.44782346941162e-07,
      "loss": 0.16245561838150024,
      "memory(GiB)": 75.26,
      "reward": 1.9976985454559326,
      "reward_std": 0.5367000102996826,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.28125,
      "rewards/MultiModalAccuracyORM/std": 0.45103853940963745,
      "rewards/ReflectionBalance/mean": 0.8141048550605774,
      "rewards/ReflectionBalance/std": 0.23479348421096802,
      "step": 1395,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 485.546875,
      "completions/min_length": 242.5,
      "epoch": 7.544474393530997,
      "grad_norm": 3.0658206377279464,
      "kl": 2.578125,
      "learning_rate": 1.4417911937809906e-07,
      "loss": 0.22386866807937622,
      "memory(GiB)": 75.26,
      "reward": 1.849973976612091,
      "reward_std": 0.5261476635932922,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.3131764084100723,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3681531548500061,
      "rewards/ReflectionBalance/mean": 0.8030989766120911,
      "rewards/ReflectionBalance/std": 0.25954442471265793,
      "step": 1396,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 481.640625,
      "completions/min_length": 252.0,
      "epoch": 7.549865229110512,
      "grad_norm": 3.30669611712174,
      "kl": 1.94140625,
      "learning_rate": 1.4357693931587728e-07,
      "loss": 0.1769806444644928,
      "memory(GiB)": 75.26,
      "reward": 1.9361311793327332,
      "reward_std": 0.38609282672405243,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24678732454776764,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3752707988023758,
      "rewards/ReflectionBalance/mean": 0.8345687091350555,
      "rewards/ReflectionBalance/std": 0.23080898076295853,
      "step": 1397,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 508.21484375,
      "completions/min_length": 300.5,
      "epoch": 7.555256064690027,
      "grad_norm": 3.088909735894888,
      "kl": 3.140625,
      "learning_rate": 1.4297580852725433e-07,
      "loss": 0.24914203584194183,
      "memory(GiB)": 75.26,
      "reward": 1.9071191549301147,
      "reward_std": 0.5123686194419861,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.40095221996307373,
      "rewards/ReflectionBalance/mean": 0.8055566251277924,
      "rewards/ReflectionBalance/std": 0.2831765189766884,
      "step": 1398,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 482.4296875,
      "completions/min_length": 291.5,
      "epoch": 7.560646900269542,
      "grad_norm": 3.2748555235980716,
      "kl": 2.40625,
      "learning_rate": 1.4237572878189862e-07,
      "loss": 0.1850053369998932,
      "memory(GiB)": 75.26,
      "reward": 1.8824186325073242,
      "reward_std": 0.4033597558736801,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.33660952746868134,
      "rewards/ReflectionBalance/mean": 0.8238248825073242,
      "rewards/ReflectionBalance/std": 0.22513510286808014,
      "step": 1399,
      "train_speed(iter/s)": 0.007121
    },
    {
      "epoch": 7.566037735849057,
      "grad_norm": 3.5084681783649283,
      "learning_rate": 1.417767018463845e-07,
      "loss": 0.2281503975391388,
      "memory(GiB)": 75.26,
      "step": 1400,
      "train_speed(iter/s)": 0.007121
    },
    {
      "epoch": 7.566037735849057,
      "eval_clip_ratio": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 1026.5,
      "eval_completions/mean_length": 510.095703125,
      "eval_completions/min_length": 231.25,
      "eval_kl": 2.71875,
      "eval_loss": 0.22227352857589722,
      "eval_reward": 1.8531038463115692,
      "eval_reward_std": 0.4759775772690773,
      "eval_rewards/Format/mean": 0.912109375,
      "eval_rewards/Format/std": 0.2803555205464363,
      "eval_rewards/MultiModalAccuracyORM/mean": 0.134765625,
      "eval_rewards/MultiModalAccuracyORM/std": 0.3358204811811447,
      "eval_rewards/ReflectionBalance/mean": 0.806228831410408,
      "eval_rewards/ReflectionBalance/std": 0.2612329535186291,
      "eval_runtime": 246.4566,
      "eval_samples_per_second": 0.239,
      "eval_steps_per_second": 0.004,
      "step": 1400
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.75,
      "completions/mean_length": 505.990234375,
      "completions/min_length": 215.75,
      "epoch": 7.571428571428571,
      "grad_norm": 3.577757365142448,
      "kl": 2.94921875,
      "learning_rate": 1.4117872948418692e-07,
      "loss": 0.20149224996566772,
      "memory(GiB)": 75.26,
      "reward": 1.8330138325691223,
      "reward_std": 0.5580212324857712,
      "rewards/Format/mean": 0.873046875,
      "rewards/Format/std": 0.3309350237250328,
      "rewards/MultiModalAccuracyORM/mean": 0.173828125,
      "rewards/MultiModalAccuracyORM/std": 0.3716045543551445,
      "rewards/ReflectionBalance/mean": 0.7861388176679611,
      "rewards/ReflectionBalance/std": 0.2961832620203495,
      "step": 1401,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 467.89453125,
      "completions/min_length": 257.0,
      "epoch": 7.576819407008086,
      "grad_norm": 3.3937659413527665,
      "kl": 2.859375,
      "learning_rate": 1.405818134556761e-07,
      "loss": 0.26143789291381836,
      "memory(GiB)": 75.26,
      "reward": 1.7718847393989563,
      "reward_std": 0.5196592807769775,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.32746291160583496,
      "rewards/ReflectionBalance/mean": 0.7601659893989563,
      "rewards/ReflectionBalance/std": 0.3220980316400528,
      "step": 1402,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 480.390625,
      "completions/min_length": 274.5,
      "epoch": 7.5822102425876015,
      "grad_norm": 4.107479663178576,
      "kl": 2.078125,
      "learning_rate": 1.399859555181131e-07,
      "loss": 0.17321541905403137,
      "memory(GiB)": 75.26,
      "reward": 2.0662508606910706,
      "reward_std": 0.553556889295578,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29262590408325195,
      "rewards/MultiModalAccuracyORM/mean": 0.34765625,
      "rewards/MultiModalAccuracyORM/std": 0.4740851819515228,
      "rewards/ReflectionBalance/mean": 0.8123445808887482,
      "rewards/ReflectionBalance/std": 0.22121787816286087,
      "step": 1403,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 499.4296875,
      "completions/min_length": 250.5,
      "epoch": 7.587601078167116,
      "grad_norm": 5.171112645978245,
      "kl": 4.3125,
      "learning_rate": 1.393911574256435e-07,
      "loss": 0.3275955617427826,
      "memory(GiB)": 75.26,
      "reward": 1.874449908733368,
      "reward_std": 0.5608006864786148,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.34447427093982697,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4331994354724884,
      "rewards/ReflectionBalance/mean": 0.7611686289310455,
      "rewards/ReflectionBalance/std": 0.32608403265476227,
      "step": 1404,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 482.03125,
      "completions/min_length": 257.5,
      "epoch": 7.592991913746631,
      "grad_norm": 3.406923327251076,
      "kl": 3.5078125,
      "learning_rate": 1.3879742092929299e-07,
      "loss": 0.2828136086463928,
      "memory(GiB)": 75.26,
      "reward": 1.9412646293640137,
      "reward_std": 0.605676531791687,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.44543786346912384,
      "rewards/ReflectionBalance/mean": 0.7850146889686584,
      "rewards/ReflectionBalance/std": 0.2744786888360977,
      "step": 1405,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 485.296875,
      "completions/min_length": 261.5,
      "epoch": 7.598382749326145,
      "grad_norm": 4.357954976971423,
      "kl": 2.7734375,
      "learning_rate": 1.382047477769619e-07,
      "loss": 0.21837946772575378,
      "memory(GiB)": 75.26,
      "reward": 1.8996549248695374,
      "reward_std": 0.5678423941135406,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4060405045747757,
      "rewards/ReflectionBalance/mean": 0.7824674546718597,
      "rewards/ReflectionBalance/std": 0.28921931982040405,
      "step": 1406,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 501.296875,
      "completions/min_length": 275.0,
      "epoch": 7.60377358490566,
      "grad_norm": 3.581255143693394,
      "kl": 3.390625,
      "learning_rate": 1.3761313971342054e-07,
      "loss": 0.25323769450187683,
      "memory(GiB)": 75.26,
      "reward": 1.8661669492721558,
      "reward_std": 0.5512914806604385,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.7880420088768005,
      "rewards/ReflectionBalance/std": 0.29023976624011993,
      "step": 1407,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 513.125,
      "completions/min_length": 254.0,
      "epoch": 7.609164420485175,
      "grad_norm": 4.13718732014766,
      "kl": 4.234375,
      "learning_rate": 1.370225984803034e-07,
      "loss": 0.32147347927093506,
      "memory(GiB)": 75.26,
      "reward": 1.80027836561203,
      "reward_std": 0.5591346919536591,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3389271944761276,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36045311391353607,
      "rewards/ReflectionBalance/mean": 0.7768408358097076,
      "rewards/ReflectionBalance/std": 0.27416402101516724,
      "step": 1408,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 515.52734375,
      "completions/min_length": 288.5,
      "epoch": 7.6145552560646905,
      "grad_norm": 3.2617874558008437,
      "kl": 2.8984375,
      "learning_rate": 1.3643312581610422e-07,
      "loss": 0.23722442984580994,
      "memory(GiB)": 75.26,
      "reward": 1.89980947971344,
      "reward_std": 0.4905068278312683,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3050640821456909,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3525480553507805,
      "rewards/ReflectionBalance/mean": 0.8216845691204071,
      "rewards/ReflectionBalance/std": 0.25591709464788437,
      "step": 1409,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.5,
      "completions/mean_length": 486.76953125,
      "completions/min_length": 266.0,
      "epoch": 7.619946091644205,
      "grad_norm": 3.096377670094973,
      "kl": 2.34375,
      "learning_rate": 1.358447234561716e-07,
      "loss": 0.17412596940994263,
      "memory(GiB)": 75.26,
      "reward": 1.9376513957977295,
      "reward_std": 0.42977987229824066,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3871416449546814,
      "rewards/ReflectionBalance/mean": 0.8165575861930847,
      "rewards/ReflectionBalance/std": 0.23010771721601486,
      "step": 1410,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 504.00390625,
      "completions/min_length": 190.5,
      "epoch": 7.62533692722372,
      "grad_norm": 3.3389477273550674,
      "kl": 3.0703125,
      "learning_rate": 1.3525739313270224e-07,
      "loss": 0.2496025413274765,
      "memory(GiB)": 75.26,
      "reward": 1.9095377326011658,
      "reward_std": 0.5506840199232101,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42852601408958435,
      "rewards/ReflectionBalance/mean": 0.7728189826011658,
      "rewards/ReflectionBalance/std": 0.2957913428544998,
      "step": 1411,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 499.22265625,
      "completions/min_length": 242.0,
      "epoch": 7.630727762803234,
      "grad_norm": 3.759038768479262,
      "kl": 3.0859375,
      "learning_rate": 1.3467113657473795e-07,
      "loss": 0.2514306604862213,
      "memory(GiB)": 75.26,
      "reward": 1.839367687702179,
      "reward_std": 0.5846083760261536,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.38125884532928467,
      "rewards/ReflectionBalance/mean": 0.772961437702179,
      "rewards/ReflectionBalance/std": 0.32300569117069244,
      "step": 1412,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 523.09765625,
      "completions/min_length": 304.0,
      "epoch": 7.636118598382749,
      "grad_norm": 3.792797674903501,
      "kl": 3.6171875,
      "learning_rate": 1.3408595550815883e-07,
      "loss": 0.2923920154571533,
      "memory(GiB)": 75.26,
      "reward": 1.8226221203804016,
      "reward_std": 0.5474547445774078,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3208130896091461,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.339918777346611,
      "rewards/ReflectionBalance/mean": 0.8069970607757568,
      "rewards/ReflectionBalance/std": 0.3035087436437607,
      "step": 1413,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 497.8046875,
      "completions/min_length": 211.5,
      "epoch": 7.6415094339622645,
      "grad_norm": 3.360304618059019,
      "kl": 2.890625,
      "learning_rate": 1.3350185165567885e-07,
      "loss": 0.21588662266731262,
      "memory(GiB)": 75.26,
      "reward": 1.9204792976379395,
      "reward_std": 0.49871641397476196,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.40855903923511505,
      "rewards/ReflectionBalance/mean": 0.8071980774402618,
      "rewards/ReflectionBalance/std": 0.2486906349658966,
      "step": 1414,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 517.828125,
      "completions/min_length": 168.0,
      "epoch": 7.646900269541779,
      "grad_norm": 3.1912094885488167,
      "kl": 3.13671875,
      "learning_rate": 1.3291882673684124e-07,
      "loss": 0.25409916043281555,
      "memory(GiB)": 75.26,
      "reward": 1.838390290737152,
      "reward_std": 0.5007858574390411,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.28456588089466095,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.3562464565038681,
      "rewards/ReflectionBalance/mean": 0.7876090407371521,
      "rewards/ReflectionBalance/std": 0.28651033341884613,
      "step": 1415,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 492.68359375,
      "completions/min_length": 264.0,
      "epoch": 7.652291105121294,
      "grad_norm": 3.1076046425855597,
      "kl": 1.77734375,
      "learning_rate": 1.323368824680125e-07,
      "loss": 0.13484913110733032,
      "memory(GiB)": 75.26,
      "reward": 1.9794645309448242,
      "reward_std": 0.47368161380290985,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.40554189682006836,
      "rewards/ReflectionBalance/mean": 0.8310270011425018,
      "rewards/ReflectionBalance/std": 0.2112032100558281,
      "step": 1416,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 485.44921875,
      "completions/min_length": 280.5,
      "epoch": 7.657681940700809,
      "grad_norm": 2.6778854411107758,
      "kl": 2.4921875,
      "learning_rate": 1.31756020562378e-07,
      "loss": 0.19737011194229126,
      "memory(GiB)": 75.26,
      "reward": 1.8398014307022095,
      "reward_std": 0.42056770622730255,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.310598224401474,
      "rewards/ReflectionBalance/mean": 0.8241764307022095,
      "rewards/ReflectionBalance/std": 0.23554463684558868,
      "step": 1417,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 507.7734375,
      "completions/min_length": 299.5,
      "epoch": 7.663072776280323,
      "grad_norm": 3.4985803919972622,
      "kl": 2.2421875,
      "learning_rate": 1.3117624272993643e-07,
      "loss": 0.19302481412887573,
      "memory(GiB)": 75.26,
      "reward": 1.8934556245803833,
      "reward_std": 0.4104413241147995,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3098885118961334,
      "rewards/ReflectionBalance/mean": 0.8270494639873505,
      "rewards/ReflectionBalance/std": 0.22676482796669006,
      "step": 1418,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 499.421875,
      "completions/min_length": 226.5,
      "epoch": 7.668463611859838,
      "grad_norm": 7.601273414043862,
      "kl": 2.5390625,
      "learning_rate": 1.3059755067749578e-07,
      "loss": 0.21919330954551697,
      "memory(GiB)": 75.26,
      "reward": 1.7822656631469727,
      "reward_std": 0.5052198618650436,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.298691526055336,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.29547789692878723,
      "rewards/ReflectionBalance/mean": 0.7744531333446503,
      "rewards/ReflectionBalance/std": 0.297401562333107,
      "step": 1419,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 516.71484375,
      "completions/min_length": 297.5,
      "epoch": 7.6738544474393535,
      "grad_norm": 3.0680288531234465,
      "kl": 2.18359375,
      "learning_rate": 1.3001994610866695e-07,
      "loss": 0.15675948560237885,
      "memory(GiB)": 75.26,
      "reward": 2.0394253730773926,
      "reward_std": 0.5456578433513641,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.3046875,
      "rewards/MultiModalAccuracyORM/std": 0.460112988948822,
      "rewards/ReflectionBalance/mean": 0.8050504624843597,
      "rewards/ReflectionBalance/std": 0.246697798371315,
      "step": 1420,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 505.1875,
      "completions/min_length": 270.5,
      "epoch": 7.679245283018868,
      "grad_norm": 6.761874091260806,
      "kl": 2.5,
      "learning_rate": 1.2944343072385948e-07,
      "loss": 0.1836153268814087,
      "memory(GiB)": 75.26,
      "reward": 1.8199958801269531,
      "reward_std": 0.5096909999847412,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2829600051045418,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.31232215464115143,
      "rewards/ReflectionBalance/mean": 0.8004646897315979,
      "rewards/ReflectionBalance/std": 0.3083065450191498,
      "step": 1421,
      "train_speed(iter/s)": 0.00711
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 467.9921875,
      "completions/min_length": 252.5,
      "epoch": 7.684636118598383,
      "grad_norm": 3.2430787853858596,
      "kl": 1.41015625,
      "learning_rate": 1.288680062202771e-07,
      "loss": 0.1187075674533844,
      "memory(GiB)": 75.26,
      "reward": 2.131144165992737,
      "reward_std": 0.41233763098716736,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.2156769335269928,
      "rewards/MultiModalAccuracyORM/mean": 0.34765625,
      "rewards/MultiModalAccuracyORM/std": 0.47594553232192993,
      "rewards/ReflectionBalance/mean": 0.8342691957950592,
      "rewards/ReflectionBalance/std": 0.20493227243423462,
      "step": 1422,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 875.0,
      "completions/mean_length": 454.234375,
      "completions/min_length": 233.0,
      "epoch": 7.690026954177897,
      "grad_norm": 3.000947845497312,
      "kl": 1.1640625,
      "learning_rate": 1.2829367429191096e-07,
      "loss": 0.10568338632583618,
      "memory(GiB)": 75.26,
      "reward": 2.0929280519485474,
      "reward_std": 0.40539322793483734,
      "rewards/Format/mean": 0.96875,
      "rewards/Format/std": 0.17319392412900925,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.4497225433588028,
      "rewards/ReflectionBalance/mean": 0.8390218019485474,
      "rewards/ReflectionBalance/std": 0.23049545288085938,
      "step": 1423,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 507.3515625,
      "completions/min_length": 236.0,
      "epoch": 7.695417789757412,
      "grad_norm": 3.0581981707554227,
      "kl": 2.1015625,
      "learning_rate": 1.2772043662953675e-07,
      "loss": 0.16290318965911865,
      "memory(GiB)": 75.26,
      "reward": 1.851924180984497,
      "reward_std": 0.4554077982902527,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.30740001052618027,
      "rewards/ReflectionBalance/mean": 0.8167679309844971,
      "rewards/ReflectionBalance/std": 0.24858571588993073,
      "step": 1424,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 500.8203125,
      "completions/min_length": 252.5,
      "epoch": 7.7008086253369274,
      "grad_norm": 2.829339915351366,
      "kl": 2.6796875,
      "learning_rate": 1.2714829492070867e-07,
      "loss": 0.2072518765926361,
      "memory(GiB)": 75.26,
      "reward": 1.9890022277832031,
      "reward_std": 0.4349946677684784,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.44320113956928253,
      "rewards/ReflectionBalance/mean": 0.7936897873878479,
      "rewards/ReflectionBalance/std": 0.2723981589078903,
      "step": 1425,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 508.6484375,
      "completions/min_length": 171.5,
      "epoch": 7.706199460916442,
      "grad_norm": 3.100896192579468,
      "kl": 1.5546875,
      "learning_rate": 1.265772508497538e-07,
      "loss": 0.11108261346817017,
      "memory(GiB)": 75.26,
      "reward": 1.8517838716506958,
      "reward_std": 0.4379274249076843,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.32267695665359497,
      "rewards/ReflectionBalance/mean": 0.8127213716506958,
      "rewards/ReflectionBalance/std": 0.21451987326145172,
      "step": 1426,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 496.29296875,
      "completions/min_length": 253.5,
      "epoch": 7.711590296495957,
      "grad_norm": 3.856721239399032,
      "kl": 2.81640625,
      "learning_rate": 1.2600730609776872e-07,
      "loss": 0.22108624875545502,
      "memory(GiB)": 75.26,
      "reward": 1.8423625230789185,
      "reward_std": 0.49653588235378265,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3586205691099167,
      "rewards/ReflectionBalance/mean": 0.7876749336719513,
      "rewards/ReflectionBalance/std": 0.29025764763355255,
      "step": 1427,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 505.48046875,
      "completions/min_length": 252.5,
      "epoch": 7.716981132075472,
      "grad_norm": 3.2556658534022573,
      "kl": 2.7265625,
      "learning_rate": 1.2543846234261323e-07,
      "loss": 0.21470217406749725,
      "memory(GiB)": 75.26,
      "reward": 1.910764992237091,
      "reward_std": 0.5285927057266235,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41857391595840454,
      "rewards/ReflectionBalance/mean": 0.7857649624347687,
      "rewards/ReflectionBalance/std": 0.2854742854833603,
      "step": 1428,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 509.515625,
      "completions/min_length": 273.0,
      "epoch": 7.722371967654986,
      "grad_norm": 3.0432290077906656,
      "kl": 1.83203125,
      "learning_rate": 1.2487072125890608e-07,
      "loss": 0.14410389959812164,
      "memory(GiB)": 75.26,
      "reward": 1.8498095870018005,
      "reward_std": 0.44200974702835083,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.305100217461586,
      "rewards/ReflectionBalance/mean": 0.8068408370018005,
      "rewards/ReflectionBalance/std": 0.23543577641248703,
      "step": 1429,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 493.0546875,
      "completions/min_length": 272.5,
      "epoch": 7.727762803234501,
      "grad_norm": 3.0269541539928335,
      "kl": 2.38671875,
      "learning_rate": 1.243040845180201e-07,
      "loss": 0.20707567036151886,
      "memory(GiB)": 75.26,
      "reward": 1.8853580355644226,
      "reward_std": 0.5068142712116241,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3678421527147293,
      "rewards/ReflectionBalance/mean": 0.815045565366745,
      "rewards/ReflectionBalance/std": 0.2354145050048828,
      "step": 1430,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 509.13671875,
      "completions/min_length": 244.5,
      "epoch": 7.7331536388140165,
      "grad_norm": 3.311534005445417,
      "kl": 2.921875,
      "learning_rate": 1.237385537880763e-07,
      "loss": 0.2617681622505188,
      "memory(GiB)": 75.26,
      "reward": 1.8367470502853394,
      "reward_std": 0.5217285752296448,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3437131494283676,
      "rewards/ReflectionBalance/mean": 0.7898721098899841,
      "rewards/ReflectionBalance/std": 0.3189447969198227,
      "step": 1431,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 497.6796875,
      "completions/min_length": 247.5,
      "epoch": 7.738544474393531,
      "grad_norm": 3.4591760668888143,
      "kl": 2.234375,
      "learning_rate": 1.231741307339406e-07,
      "loss": 0.22056585550308228,
      "memory(GiB)": 75.26,
      "reward": 2.0209227204322815,
      "reward_std": 0.42004716396331787,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24301259219646454,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.423653244972229,
      "rewards/ReflectionBalance/mean": 0.849047839641571,
      "rewards/ReflectionBalance/std": 0.1776280775666237,
      "step": 1432,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 492.01171875,
      "completions/min_length": 283.5,
      "epoch": 7.743935309973046,
      "grad_norm": 3.914229652447943,
      "kl": 2.1796875,
      "learning_rate": 1.2261081701721754e-07,
      "loss": 0.18866993486881256,
      "memory(GiB)": 75.26,
      "reward": 1.933567762374878,
      "reward_std": 0.4633399546146393,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38864077627658844,
      "rewards/ReflectionBalance/mean": 0.8241927027702332,
      "rewards/ReflectionBalance/std": 0.2385716438293457,
      "step": 1433,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 928.5,
      "completions/mean_length": 495.26171875,
      "completions/min_length": 252.0,
      "epoch": 7.74932614555256,
      "grad_norm": 2.7203914946759884,
      "kl": 1.94921875,
      "learning_rate": 1.220486142962458e-07,
      "loss": 0.1634945273399353,
      "memory(GiB)": 75.26,
      "reward": 1.9320279955863953,
      "reward_std": 0.45749934017658234,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3804178833961487,
      "rewards/ReflectionBalance/mean": 0.8226530253887177,
      "rewards/ReflectionBalance/std": 0.21146193146705627,
      "step": 1434,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 462.890625,
      "completions/min_length": 261.0,
      "epoch": 7.754716981132075,
      "grad_norm": 3.1161457202721237,
      "kl": 2.109375,
      "learning_rate": 1.2148752422609388e-07,
      "loss": 0.1527515947818756,
      "memory(GiB)": 75.26,
      "reward": 1.9311132431030273,
      "reward_std": 0.48142339289188385,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29434798657894135,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.39412054419517517,
      "rewards/ReflectionBalance/mean": 0.8139257729053497,
      "rewards/ReflectionBalance/std": 0.2620967626571655,
      "step": 1435,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.0,
      "completions/mean_length": 481.78125,
      "completions/min_length": 268.0,
      "epoch": 7.7601078167115904,
      "grad_norm": 4.004534783395115,
      "kl": 2.5859375,
      "learning_rate": 1.2092754845855424e-07,
      "loss": 0.2421227991580963,
      "memory(GiB)": 75.26,
      "reward": 1.9155473709106445,
      "reward_std": 0.5093860924243927,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38597656786441803,
      "rewards/ReflectionBalance/mean": 0.8139848411083221,
      "rewards/ReflectionBalance/std": 0.275194987654686,
      "step": 1436,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 520.08984375,
      "completions/min_length": 296.5,
      "epoch": 7.765498652291106,
      "grad_norm": 5.111727083619353,
      "kl": 3.1171875,
      "learning_rate": 1.2036868864213921e-07,
      "loss": 0.24468597769737244,
      "memory(GiB)": 75.26,
      "reward": 1.8767513036727905,
      "reward_std": 0.5539287626743317,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3318064510822296,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3978758305311203,
      "rewards/ReflectionBalance/mean": 0.8064388036727905,
      "rewards/ReflectionBalance/std": 0.26493869721889496,
      "step": 1437,
      "train_speed(iter/s)": 0.007111
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 505.1484375,
      "completions/min_length": 274.5,
      "epoch": 7.77088948787062,
      "grad_norm": 3.36725991001568,
      "kl": 2.90625,
      "learning_rate": 1.1981094642207567e-07,
      "loss": 0.2588980793952942,
      "memory(GiB)": 75.26,
      "reward": 1.853847622871399,
      "reward_std": 0.524268239736557,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3846261203289032,
      "rewards/ReflectionBalance/mean": 0.7835351526737213,
      "rewards/ReflectionBalance/std": 0.2834545373916626,
      "step": 1438,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 490.60546875,
      "completions/min_length": 262.5,
      "epoch": 7.776280323450135,
      "grad_norm": 3.0188179574050245,
      "kl": 2.6796875,
      "learning_rate": 1.1925432344030085e-07,
      "loss": 0.22868146002292633,
      "memory(GiB)": 75.26,
      "reward": 1.9027766585350037,
      "reward_std": 0.5210115909576416,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39110109210014343,
      "rewards/ReflectionBalance/mean": 0.8168392181396484,
      "rewards/ReflectionBalance/std": 0.268661230802536,
      "step": 1439,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 493.2265625,
      "completions/min_length": 291.5,
      "epoch": 7.781671159029649,
      "grad_norm": 3.6972618130061137,
      "kl": 2.375,
      "learning_rate": 1.1869882133545655e-07,
      "loss": 0.17933757603168488,
      "memory(GiB)": 75.26,
      "reward": 1.98911452293396,
      "reward_std": 0.4286678731441498,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4270860105752945,
      "rewards/ReflectionBalance/mean": 0.8250520825386047,
      "rewards/ReflectionBalance/std": 0.22072140127420425,
      "step": 1440,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 481.84375,
      "completions/min_length": 291.0,
      "epoch": 7.787061994609164,
      "grad_norm": 59.333234245203045,
      "kl": 3.203125,
      "learning_rate": 1.1814444174288495e-07,
      "loss": 0.26118984818458557,
      "memory(GiB)": 75.26,
      "reward": 1.8538134694099426,
      "reward_std": 0.5027024298906326,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3734995424747467,
      "rewards/ReflectionBalance/mean": 0.7913134694099426,
      "rewards/ReflectionBalance/std": 0.2614557668566704,
      "step": 1441,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 491.98046875,
      "completions/min_length": 251.0,
      "epoch": 7.7924528301886795,
      "grad_norm": 3.1151304101290203,
      "kl": 2.734375,
      "learning_rate": 1.1759118629462417e-07,
      "loss": 0.22362378239631653,
      "memory(GiB)": 75.26,
      "reward": 1.8983317017555237,
      "reward_std": 0.44350965321063995,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.34719423204660416,
      "rewards/ReflectionBalance/mean": 0.8045817017555237,
      "rewards/ReflectionBalance/std": 0.2552070617675781,
      "step": 1442,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 457.26171875,
      "completions/min_length": 261.0,
      "epoch": 7.797843665768194,
      "grad_norm": 32.52035768811225,
      "kl": 2.890625,
      "learning_rate": 1.1703905661940189e-07,
      "loss": 0.22568613290786743,
      "memory(GiB)": 75.26,
      "reward": 1.9442114233970642,
      "reward_std": 0.4844870865345001,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41983383893966675,
      "rewards/ReflectionBalance/mean": 0.8113989233970642,
      "rewards/ReflectionBalance/std": 0.28908713161945343,
      "step": 1443,
      "train_speed(iter/s)": 0.007112
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 502.60546875,
      "completions/min_length": 144.0,
      "epoch": 7.803234501347709,
      "grad_norm": 4.955973717301882,
      "kl": 3.78125,
      "learning_rate": 1.1648805434263276e-07,
      "loss": 0.2892777919769287,
      "memory(GiB)": 75.26,
      "reward": 1.80472332239151,
      "reward_std": 0.5179704129695892,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3511015325784683,
      "rewards/ReflectionBalance/mean": 0.76566082239151,
      "rewards/ReflectionBalance/std": 0.32784712314605713,
      "step": 1444,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 510.65234375,
      "completions/min_length": 200.0,
      "epoch": 7.808625336927224,
      "grad_norm": 3.5300688588919447,
      "kl": 3.6875,
      "learning_rate": 1.1593818108641185e-07,
      "loss": 0.2927936911582947,
      "memory(GiB)": 75.26,
      "reward": 1.7648274898529053,
      "reward_std": 0.6724550426006317,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3389271944761276,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35675957798957825,
      "rewards/ReflectionBalance/mean": 0.7492024600505829,
      "rewards/ReflectionBalance/std": 0.32702963054180145,
      "step": 1445,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 494.84765625,
      "completions/min_length": 232.5,
      "epoch": 7.814016172506738,
      "grad_norm": 4.728755585162096,
      "kl": 2.625,
      "learning_rate": 1.153894384695105e-07,
      "loss": 0.20123931765556335,
      "memory(GiB)": 75.26,
      "reward": 1.8990185260772705,
      "reward_std": 0.5343738496303558,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3935103267431259,
      "rewards/ReflectionBalance/mean": 0.8052685558795929,
      "rewards/ReflectionBalance/std": 0.24816370755434036,
      "step": 1446,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 492.22265625,
      "completions/min_length": 256.5,
      "epoch": 7.819407008086253,
      "grad_norm": 3.733805957815473,
      "kl": 3.203125,
      "learning_rate": 1.1484182810737203e-07,
      "loss": 0.24685662984848022,
      "memory(GiB)": 75.26,
      "reward": 1.9300684332847595,
      "reward_std": 0.588347315788269,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.4150650203227997,
      "rewards/ReflectionBalance/mean": 0.8011621236801147,
      "rewards/ReflectionBalance/std": 0.2699384465813637,
      "step": 1447,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 494.85546875,
      "completions/min_length": 286.0,
      "epoch": 7.824797843665769,
      "grad_norm": 3.933239821827737,
      "kl": 2.6875,
      "learning_rate": 1.1429535161210601e-07,
      "loss": 0.19477036595344543,
      "memory(GiB)": 75.26,
      "reward": 1.930873990058899,
      "reward_std": 0.5315434336662292,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41441819071769714,
      "rewards/ReflectionBalance/mean": 0.8058740496635437,
      "rewards/ReflectionBalance/std": 0.2563336566090584,
      "step": 1448,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 499.05859375,
      "completions/min_length": 254.0,
      "epoch": 7.830188679245283,
      "grad_norm": 3.888946425757714,
      "kl": 2.453125,
      "learning_rate": 1.1375001059248424e-07,
      "loss": 0.1833297312259674,
      "memory(GiB)": 75.26,
      "reward": 1.9366071820259094,
      "reward_std": 0.4924318790435791,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.24400590360164642,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39416566491127014,
      "rewards/ReflectionBalance/mean": 0.8155135214328766,
      "rewards/ReflectionBalance/std": 0.2270166277885437,
      "step": 1449,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 474.43359375,
      "completions/min_length": 251.0,
      "epoch": 7.835579514824798,
      "grad_norm": 3.2289739217139615,
      "kl": 2.1640625,
      "learning_rate": 1.1320580665393564e-07,
      "loss": 0.1946832239627838,
      "memory(GiB)": 75.26,
      "reward": 2.0887938737869263,
      "reward_std": 0.49940451979637146,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23196382820606232,
      "rewards/MultiModalAccuracyORM/mean": 0.33984375,
      "rewards/MultiModalAccuracyORM/std": 0.47405435144901276,
      "rewards/ReflectionBalance/mean": 0.807543933391571,
      "rewards/ReflectionBalance/std": 0.28844045102596283,
      "step": 1450,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 487.84375,
      "completions/min_length": 237.5,
      "epoch": 7.840970350404312,
      "grad_norm": 3.3240786750862568,
      "kl": 2.01953125,
      "learning_rate": 1.1266274139854203e-07,
      "loss": 0.17889618873596191,
      "memory(GiB)": 75.26,
      "reward": 1.8819931149482727,
      "reward_std": 0.4715070128440857,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3643587827682495,
      "rewards/ReflectionBalance/mean": 0.8038679957389832,
      "rewards/ReflectionBalance/std": 0.27990925312042236,
      "step": 1451,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 484.96484375,
      "completions/min_length": 248.5,
      "epoch": 7.846361185983827,
      "grad_norm": 3.8980535662857254,
      "kl": 2.6171875,
      "learning_rate": 1.1212081642503279e-07,
      "loss": 0.20400182902812958,
      "memory(GiB)": 75.26,
      "reward": 1.9376139640808105,
      "reward_std": 0.4224194586277008,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3700246214866638,
      "rewards/ReflectionBalance/mean": 0.8282389044761658,
      "rewards/ReflectionBalance/std": 0.2041444033384323,
      "step": 1452,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 497.546875,
      "completions/min_length": 260.5,
      "epoch": 7.8517520215633425,
      "grad_norm": 2.8743323840341035,
      "kl": 2.07421875,
      "learning_rate": 1.1158003332878042e-07,
      "loss": 0.15542185306549072,
      "memory(GiB)": 75.26,
      "reward": 1.9121776819229126,
      "reward_std": 0.5101892650127411,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3853000998497009,
      "rewards/ReflectionBalance/mean": 0.822333961725235,
      "rewards/ReflectionBalance/std": 0.2292342185974121,
      "step": 1453,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 496.3671875,
      "completions/min_length": 271.0,
      "epoch": 7.857142857142857,
      "grad_norm": 3.0304525863582255,
      "kl": 1.94921875,
      "learning_rate": 1.1104039370179569e-07,
      "loss": 0.1620834767818451,
      "memory(GiB)": 75.26,
      "reward": 2.0648714303970337,
      "reward_std": 0.4432583302259445,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24678732454776764,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.45157207548618317,
      "rewards/ReflectionBalance/mean": 0.8461214303970337,
      "rewards/ReflectionBalance/std": 0.20577041804790497,
      "step": 1454,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 502.9296875,
      "completions/min_length": 176.5,
      "epoch": 7.862533692722372,
      "grad_norm": 2.7282188114129786,
      "kl": 2.06640625,
      "learning_rate": 1.1050189913272368e-07,
      "loss": 0.18120460212230682,
      "memory(GiB)": 75.26,
      "reward": 1.983858346939087,
      "reward_std": 0.46007925271987915,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.4129126965999603,
      "rewards/ReflectionBalance/mean": 0.8158896565437317,
      "rewards/ReflectionBalance/std": 0.2506645396351814,
      "step": 1455,
      "train_speed(iter/s)": 0.007113
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 492.44140625,
      "completions/min_length": 279.5,
      "epoch": 7.867924528301887,
      "grad_norm": 3.053734708257764,
      "kl": 2.375,
      "learning_rate": 1.0996455120683807e-07,
      "loss": 0.18969519436359406,
      "memory(GiB)": 75.26,
      "reward": 1.962367296218872,
      "reward_std": 0.4940577298402786,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.42014752328395844,
      "rewards/ReflectionBalance/mean": 0.8139298558235168,
      "rewards/ReflectionBalance/std": 0.24755582213401794,
      "step": 1456,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 486.27734375,
      "completions/min_length": 244.0,
      "epoch": 7.873315363881401,
      "grad_norm": 3.004495111123692,
      "kl": 2.1015625,
      "learning_rate": 1.094283515060369e-07,
      "loss": 0.17930157482624054,
      "memory(GiB)": 75.26,
      "reward": 2.017088234424591,
      "reward_std": 0.4206719398498535,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.4298042804002762,
      "rewards/ReflectionBalance/mean": 0.8256819546222687,
      "rewards/ReflectionBalance/std": 0.21753829717636108,
      "step": 1457,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 502.60546875,
      "completions/min_length": 195.0,
      "epoch": 7.878706199460916,
      "grad_norm": 14.230328988711566,
      "kl": 2.69140625,
      "learning_rate": 1.0889330160883809e-07,
      "loss": 0.2075778841972351,
      "memory(GiB)": 75.26,
      "reward": 1.8771345019340515,
      "reward_std": 0.38482779264450073,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.22757095843553543,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.2899702489376068,
      "rewards/ReflectionBalance/mean": 0.8302594423294067,
      "rewards/ReflectionBalance/std": 0.22839194536209106,
      "step": 1458,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 489.2890625,
      "completions/min_length": 267.0,
      "epoch": 7.884097035040432,
      "grad_norm": 3.928949238098475,
      "kl": 1.82421875,
      "learning_rate": 1.0835940309037489e-07,
      "loss": 0.1482020765542984,
      "memory(GiB)": 75.26,
      "reward": 1.9750032424926758,
      "reward_std": 0.4571365714073181,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41107751429080963,
      "rewards/ReflectionBalance/mean": 0.8187532424926758,
      "rewards/ReflectionBalance/std": 0.23011957854032516,
      "step": 1459,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 486.48046875,
      "completions/min_length": 282.5,
      "epoch": 7.889487870619946,
      "grad_norm": 5.152520061584115,
      "kl": 2.91015625,
      "learning_rate": 1.0782665752239062e-07,
      "loss": 0.20930327475070953,
      "memory(GiB)": 75.26,
      "reward": 1.9407161474227905,
      "reward_std": 0.4886406660079956,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2795119285583496,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4187151938676834,
      "rewards/ReflectionBalance/mean": 0.8039973974227905,
      "rewards/ReflectionBalance/std": 0.312626376748085,
      "step": 1460,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 502.74609375,
      "completions/min_length": 291.5,
      "epoch": 7.894878706199461,
      "grad_norm": 2.968864090430243,
      "kl": 2.03125,
      "learning_rate": 1.072950664732345e-07,
      "loss": 0.15701818466186523,
      "memory(GiB)": 75.26,
      "reward": 1.8485920429229736,
      "reward_std": 0.4248371869325638,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.3110392540693283,
      "rewards/ReflectionBalance/mean": 0.8251546621322632,
      "rewards/ReflectionBalance/std": 0.2411416620016098,
      "step": 1461,
      "train_speed(iter/s)": 0.007114
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 531.109375,
      "completions/min_length": 188.5,
      "epoch": 7.900269541778976,
      "grad_norm": 3.9051230850352345,
      "kl": 4.1484375,
      "learning_rate": 1.0676463150785736e-07,
      "loss": 0.31058940291404724,
      "memory(GiB)": 75.26,
      "reward": 1.7732243537902832,
      "reward_std": 0.6825430393218994,
      "rewards/Format/mean": 0.8359375,
      "rewards/Format/std": 0.36935214698314667,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39110109210014343,
      "rewards/ReflectionBalance/mean": 0.745880514383316,
      "rewards/ReflectionBalance/std": 0.32558318972587585,
      "step": 1462,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 491.0859375,
      "completions/min_length": 269.5,
      "epoch": 7.90566037735849,
      "grad_norm": 3.66447282201421,
      "kl": 2.7421875,
      "learning_rate": 1.0623535418780588e-07,
      "loss": 0.19636157155036926,
      "memory(GiB)": 75.26,
      "reward": 1.818116843700409,
      "reward_std": 0.4839913845062256,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.31518664956092834,
      "rewards/ReflectionBalance/mean": 0.7985856533050537,
      "rewards/ReflectionBalance/std": 0.25732680410146713,
      "step": 1463,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 475.3984375,
      "completions/min_length": 249.0,
      "epoch": 7.9110512129380055,
      "grad_norm": 3.274169208208407,
      "kl": 2.34375,
      "learning_rate": 1.0570723607121957e-07,
      "loss": 0.19425471127033234,
      "memory(GiB)": 75.26,
      "reward": 1.9394123554229736,
      "reward_std": 0.4612477123737335,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4120787978172302,
      "rewards/ReflectionBalance/mean": 0.8065999448299408,
      "rewards/ReflectionBalance/std": 0.25456763803958893,
      "step": 1464,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 497.48046875,
      "completions/min_length": 258.5,
      "epoch": 7.916442048517521,
      "grad_norm": 3.2398569770541257,
      "kl": 1.89453125,
      "learning_rate": 1.0518027871282481e-07,
      "loss": 0.1564529836177826,
      "memory(GiB)": 75.26,
      "reward": 2.08451509475708,
      "reward_std": 0.4148552715778351,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2255912721157074,
      "rewards/MultiModalAccuracyORM/mean": 0.2890625,
      "rewards/MultiModalAccuracyORM/std": 0.45510825514793396,
      "rewards/ReflectionBalance/mean": 0.8501399755477905,
      "rewards/ReflectionBalance/std": 0.18686231970787048,
      "step": 1465,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 500.0625,
      "completions/min_length": 249.0,
      "epoch": 7.921832884097035,
      "grad_norm": 3.50339607043163,
      "kl": 3.75,
      "learning_rate": 1.0465448366393086e-07,
      "loss": 0.3148254156112671,
      "memory(GiB)": 75.26,
      "reward": 1.8117545247077942,
      "reward_std": 0.5948812365531921,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3405093401670456,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3589489758014679,
      "rewards/ReflectionBalance/mean": 0.7805045247077942,
      "rewards/ReflectionBalance/std": 0.3166462928056717,
      "step": 1466,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 484.6953125,
      "completions/min_length": 184.0,
      "epoch": 7.92722371967655,
      "grad_norm": 3.0427713759191564,
      "kl": 2.6953125,
      "learning_rate": 1.0412985247242573e-07,
      "loss": 0.19880037009716034,
      "memory(GiB)": 75.26,
      "reward": 2.0049185752868652,
      "reward_std": 0.5623062252998352,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.32421875,
      "rewards/MultiModalAccuracyORM/std": 0.46975335478782654,
      "rewards/ReflectionBalance/mean": 0.7939811050891876,
      "rewards/ReflectionBalance/std": 0.25861337035894394,
      "step": 1467,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 479.02734375,
      "completions/min_length": 268.5,
      "epoch": 7.932614555256064,
      "grad_norm": 3.173093319217606,
      "kl": 2.8828125,
      "learning_rate": 1.0360638668277066e-07,
      "loss": 0.22687606513500214,
      "memory(GiB)": 75.26,
      "reward": 1.9804086089134216,
      "reward_std": 0.4920293092727661,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4312909245491028,
      "rewards/ReflectionBalance/mean": 0.8163460195064545,
      "rewards/ReflectionBalance/std": 0.2355993241071701,
      "step": 1468,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 496.46875,
      "completions/min_length": 180.0,
      "epoch": 7.938005390835579,
      "grad_norm": 3.0980935552299855,
      "kl": 3.1328125,
      "learning_rate": 1.030840878359962e-07,
      "loss": 0.24468031525611877,
      "memory(GiB)": 75.26,
      "reward": 1.9235399961471558,
      "reward_std": 0.5867301523685455,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41635647416114807,
      "rewards/ReflectionBalance/mean": 0.7985400259494781,
      "rewards/ReflectionBalance/std": 0.28379252552986145,
      "step": 1469,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 489.81640625,
      "completions/min_length": 267.5,
      "epoch": 7.943396226415095,
      "grad_norm": 3.7539070093132714,
      "kl": 2.12890625,
      "learning_rate": 1.0256295746969739e-07,
      "loss": 0.1818377673625946,
      "memory(GiB)": 75.26,
      "reward": 1.9386314153671265,
      "reward_std": 0.5108246207237244,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.38974663615226746,
      "rewards/ReflectionBalance/mean": 0.821444034576416,
      "rewards/ReflectionBalance/std": 0.24983952194452286,
      "step": 1470,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 508.34375,
      "completions/min_length": 267.5,
      "epoch": 7.948787061994609,
      "grad_norm": 3.2387900737998025,
      "kl": 2.140625,
      "learning_rate": 1.0204299711802988e-07,
      "loss": 0.17544889450073242,
      "memory(GiB)": 75.26,
      "reward": 1.9665511846542358,
      "reward_std": 0.46817196905612946,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4089515954256058,
      "rewards/ReflectionBalance/mean": 0.8259260952472687,
      "rewards/ReflectionBalance/std": 0.2503955140709877,
      "step": 1471,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 471.49609375,
      "completions/min_length": 209.0,
      "epoch": 7.954177897574124,
      "grad_norm": 4.877319962397528,
      "kl": 2.1953125,
      "learning_rate": 1.015242083117045e-07,
      "loss": 0.12275996804237366,
      "memory(GiB)": 75.26,
      "reward": 2.0172810554504395,
      "reward_std": 0.40043748915195465,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.419345960021019,
      "rewards/ReflectionBalance/mean": 0.8336873650550842,
      "rewards/ReflectionBalance/std": 0.26682232320308685,
      "step": 1472,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 481.09375,
      "completions/min_length": 263.5,
      "epoch": 7.959568733153639,
      "grad_norm": 4.026589791315362,
      "kl": 2.7890625,
      "learning_rate": 1.0100659257798299e-07,
      "loss": 0.19430339336395264,
      "memory(GiB)": 75.26,
      "reward": 2.044716715812683,
      "reward_std": 0.4249168932437897,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.27645355463027954,
      "rewards/MultiModalAccuracyORM/mean": 0.3125,
      "rewards/MultiModalAccuracyORM/std": 0.4652567654848099,
      "rewards/ReflectionBalance/mean": 0.8220605552196503,
      "rewards/ReflectionBalance/std": 0.23359161615371704,
      "step": 1473,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 484.875,
      "completions/min_length": 154.5,
      "epoch": 7.964959568733153,
      "grad_norm": 16.26493061275133,
      "kl": 3.5625,
      "learning_rate": 1.0049015144067441e-07,
      "loss": 0.26124799251556396,
      "memory(GiB)": 75.26,
      "reward": 1.889939785003662,
      "reward_std": 0.5316371917724609,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4020542651414871,
      "rewards/ReflectionBalance/mean": 0.7844710350036621,
      "rewards/ReflectionBalance/std": 0.27510926127433777,
      "step": 1474,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 514.6171875,
      "completions/min_length": 209.0,
      "epoch": 7.9703504043126685,
      "grad_norm": 4.193695223641592,
      "kl": 3.4921875,
      "learning_rate": 9.997488642012891e-08,
      "loss": 0.28647246956825256,
      "memory(GiB)": 75.26,
      "reward": 1.8617627024650574,
      "reward_std": 0.6171380579471588,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3407054841518402,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.40395987033843994,
      "rewards/ReflectionBalance/mean": 0.7836377024650574,
      "rewards/ReflectionBalance/std": 0.2819419652223587,
      "step": 1475,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 490.77734375,
      "completions/min_length": 264.0,
      "epoch": 7.975741239892184,
      "grad_norm": 2.8486003687823307,
      "kl": 2.3671875,
      "learning_rate": 9.946079903323529e-08,
      "loss": 0.188214510679245,
      "memory(GiB)": 75.26,
      "reward": 2.1119205951690674,
      "reward_std": 0.49360191822052,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.359375,
      "rewards/MultiModalAccuracyORM/std": 0.4678487181663513,
      "rewards/ReflectionBalance/mean": 0.8306705951690674,
      "rewards/ReflectionBalance/std": 0.22508712112903595,
      "step": 1476,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 506.00390625,
      "completions/min_length": 289.0,
      "epoch": 7.981132075471698,
      "grad_norm": 3.3214818421360697,
      "kl": 2.71875,
      "learning_rate": 9.894789079341481e-08,
      "loss": 0.23193205893039703,
      "memory(GiB)": 75.26,
      "reward": 1.9076839685440063,
      "reward_std": 0.535961925983429,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4067413955926895,
      "rewards/ReflectionBalance/mean": 0.7983089685440063,
      "rewards/ReflectionBalance/std": 0.29196588695049286,
      "step": 1477,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 485.15625,
      "completions/min_length": 282.0,
      "epoch": 7.986522911051213,
      "grad_norm": 3.128712108103406,
      "kl": 1.65234375,
      "learning_rate": 9.843616321061765e-08,
      "loss": 0.14132434129714966,
      "memory(GiB)": 75.26,
      "reward": 2.0267984867095947,
      "reward_std": 0.4391954690217972,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23443959653377533,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.41336533427238464,
      "rewards/ReflectionBalance/mean": 0.8314860165119171,
      "rewards/ReflectionBalance/std": 0.24789579957723618,
      "step": 1478,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 497.140625,
      "completions/min_length": 261.5,
      "epoch": 7.991913746630727,
      "grad_norm": 2.7712693155058425,
      "kl": 1.68359375,
      "learning_rate": 9.792561779131864e-08,
      "loss": 0.12784813344478607,
      "memory(GiB)": 75.26,
      "reward": 1.941311776638031,
      "reward_std": 0.4040307253599167,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3558929115533829,
      "rewards/ReflectionBalance/mean": 0.8241243362426758,
      "rewards/ReflectionBalance/std": 0.24710779637098312,
      "step": 1479,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 474.1796875,
      "completions/min_length": 245.0,
      "epoch": 7.997304582210242,
      "grad_norm": 3.302471458789951,
      "kl": 1.640625,
      "learning_rate": 9.74162560385115e-08,
      "loss": 0.13428816199302673,
      "memory(GiB)": 75.26,
      "reward": 2.1028295755386353,
      "reward_std": 0.363287091255188,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.3203125,
      "rewards/MultiModalAccuracyORM/std": 0.4484759718179703,
      "rewards/ReflectionBalance/mean": 0.8372046053409576,
      "rewards/ReflectionBalance/std": 0.18347440659999847,
      "step": 1480,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 506.52734375,
      "completions/min_length": 272.5,
      "epoch": 8.005390835579515,
      "grad_norm": 3.173351669133357,
      "kl": 2.625,
      "learning_rate": 9.690807945170637e-08,
      "loss": 0.21736015379428864,
      "memory(GiB)": 75.26,
      "reward": 1.9588770270347595,
      "reward_std": 0.5298084318637848,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.42014752328395844,
      "rewards/ReflectionBalance/mean": 0.8182519674301147,
      "rewards/ReflectionBalance/std": 0.2559177875518799,
      "step": 1481,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 517.23046875,
      "completions/min_length": 271.0,
      "epoch": 8.01078167115903,
      "grad_norm": 3.1035752648717065,
      "kl": 2.9453125,
      "learning_rate": 9.640108952692388e-08,
      "loss": 0.2329084575176239,
      "memory(GiB)": 75.26,
      "reward": 1.8365308046340942,
      "reward_std": 0.46692168712615967,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.26755452156066895,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3375232368707657,
      "rewards/ReflectionBalance/mean": 0.7857495248317719,
      "rewards/ReflectionBalance/std": 0.2876732796430588,
      "step": 1482,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 468.16015625,
      "completions/min_length": 267.5,
      "epoch": 8.016172506738544,
      "grad_norm": 3.1175516919554784,
      "kl": 1.71484375,
      "learning_rate": 9.58952877566911e-08,
      "loss": 0.14310839772224426,
      "memory(GiB)": 75.26,
      "reward": 2.0081908106803894,
      "reward_std": 0.3699336647987366,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.4193098843097687,
      "rewards/ReflectionBalance/mean": 0.840222179889679,
      "rewards/ReflectionBalance/std": 0.19335830956697464,
      "step": 1483,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.0,
      "completions/mean_length": 475.4921875,
      "completions/min_length": 252.0,
      "epoch": 8.021563342318059,
      "grad_norm": 3.8906960343781094,
      "kl": 1.53125,
      "learning_rate": 9.539067563003778e-08,
      "loss": 0.12060795724391937,
      "memory(GiB)": 75.26,
      "reward": 1.901586890220642,
      "reward_std": 0.44697824120521545,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24081215262413025,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3511015325784683,
      "rewards/ReflectionBalance/mean": 0.8117431700229645,
      "rewards/ReflectionBalance/std": 0.25146324932575226,
      "step": 1484,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 493.81640625,
      "completions/min_length": 272.5,
      "epoch": 8.026954177897574,
      "grad_norm": 3.5288213335498573,
      "kl": 2.7890625,
      "learning_rate": 9.488725463249109e-08,
      "loss": 0.23884700238704681,
      "memory(GiB)": 75.26,
      "reward": 1.8968220949172974,
      "reward_std": 0.5118696987628937,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3804178833961487,
      "rewards/ReflectionBalance/mean": 0.8147908449172974,
      "rewards/ReflectionBalance/std": 0.23759589344263077,
      "step": 1485,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 521.33984375,
      "completions/min_length": 261.0,
      "epoch": 8.032345013477089,
      "grad_norm": 3.533909488613213,
      "kl": 2.671875,
      "learning_rate": 9.438502624607164e-08,
      "loss": 0.20041625201702118,
      "memory(GiB)": 75.26,
      "reward": 1.9328621625900269,
      "reward_std": 0.48214878141880035,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3068038523197174,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.42696477472782135,
      "rewards/ReflectionBalance/mean": 0.7961434125900269,
      "rewards/ReflectionBalance/std": 0.250078022480011,
      "step": 1486,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 497.609375,
      "completions/min_length": 283.0,
      "epoch": 8.037735849056604,
      "grad_norm": 3.268147236528935,
      "kl": 3.4921875,
      "learning_rate": 9.388399194928948e-08,
      "loss": 0.28786414861679077,
      "memory(GiB)": 75.26,
      "reward": 1.86322420835495,
      "reward_std": 0.4849090278148651,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30283892154693604,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3762422651052475,
      "rewards/ReflectionBalance/mean": 0.7850992977619171,
      "rewards/ReflectionBalance/std": 0.3038640171289444,
      "step": 1487,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 485.4140625,
      "completions/min_length": 259.5,
      "epoch": 8.04312668463612,
      "grad_norm": 3.2517423774534056,
      "kl": 2.7421875,
      "learning_rate": 9.3384153217139e-08,
      "loss": 0.20773261785507202,
      "memory(GiB)": 75.26,
      "reward": 1.885144829750061,
      "reward_std": 0.505158394575119,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3329402506351471,
      "rewards/ReflectionBalance/mean": 0.7953011095523834,
      "rewards/ReflectionBalance/std": 0.2826884984970093,
      "step": 1488,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 476.55078125,
      "completions/min_length": 247.5,
      "epoch": 8.048517520215633,
      "grad_norm": 4.915920286627614,
      "kl": 2.71875,
      "learning_rate": 9.288551152109503e-08,
      "loss": 0.2489793449640274,
      "memory(GiB)": 75.26,
      "reward": 1.977796196937561,
      "reward_std": 0.4461077153682709,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.41492627561092377,
      "rewards/ReflectionBalance/mean": 0.8254524767398834,
      "rewards/ReflectionBalance/std": 0.21612033993005753,
      "step": 1489,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 509.125,
      "completions/min_length": 292.0,
      "epoch": 8.053908355795148,
      "grad_norm": 3.57688342046701,
      "kl": 2.8125,
      "learning_rate": 9.238806832910845e-08,
      "loss": 0.20332278311252594,
      "memory(GiB)": 75.26,
      "reward": 1.8756347298622131,
      "reward_std": 0.5201267302036285,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39068229496479034,
      "rewards/ReflectionBalance/mean": 0.7975097894668579,
      "rewards/ReflectionBalance/std": 0.28807251900434494,
      "step": 1490,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 486.33203125,
      "completions/min_length": 264.0,
      "epoch": 8.059299191374663,
      "grad_norm": 3.0564060630661514,
      "kl": 2.46875,
      "learning_rate": 9.189182510560217e-08,
      "loss": 0.2066911906003952,
      "memory(GiB)": 75.26,
      "reward": 1.8181664943695068,
      "reward_std": 0.4951584190130234,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3390307128429413,
      "rewards/ReflectionBalance/mean": 0.7947290539741516,
      "rewards/ReflectionBalance/std": 0.2859930992126465,
      "step": 1491,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 490.91015625,
      "completions/min_length": 268.5,
      "epoch": 8.064690026954178,
      "grad_norm": 5.100296967906714,
      "kl": 3.890625,
      "learning_rate": 9.139678331146611e-08,
      "loss": 0.29863089323043823,
      "memory(GiB)": 75.26,
      "reward": 1.8621156215667725,
      "reward_std": 0.5367448627948761,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.310598224401474,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.38974663615226746,
      "rewards/ReflectionBalance/mean": 0.7761780619621277,
      "rewards/ReflectionBalance/std": 0.26430652290582657,
      "step": 1492,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 536.46875,
      "completions/min_length": 193.0,
      "epoch": 8.070080862533693,
      "grad_norm": 3.80046808761875,
      "kl": 3.125,
      "learning_rate": 9.090294440405339e-08,
      "loss": 0.21508248150348663,
      "memory(GiB)": 75.26,
      "reward": 1.8617513179779053,
      "reward_std": 0.5215583592653275,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3219822645187378,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38837648928165436,
      "rewards/ReflectionBalance/mean": 0.7953450679779053,
      "rewards/ReflectionBalance/std": 0.2866382449865341,
      "step": 1493,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 504.61328125,
      "completions/min_length": 289.5,
      "epoch": 8.075471698113208,
      "grad_norm": 3.1630203958272904,
      "kl": 2.4296875,
      "learning_rate": 9.041030983717629e-08,
      "loss": 0.20606090128421783,
      "memory(GiB)": 75.26,
      "reward": 1.88187175989151,
      "reward_std": 0.5189220905303955,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3949224203824997,
      "rewards/ReflectionBalance/mean": 0.78812175989151,
      "rewards/ReflectionBalance/std": 0.3123411685228348,
      "step": 1494,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 488.5,
      "completions/min_length": 258.5,
      "epoch": 8.080862533692722,
      "grad_norm": 3.24932089337769,
      "kl": 1.890625,
      "learning_rate": 8.991888106110102e-08,
      "loss": 0.14061376452445984,
      "memory(GiB)": 75.26,
      "reward": 1.9668880701065063,
      "reward_std": 0.5024103373289108,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.42014752328395844,
      "rewards/ReflectionBalance/mean": 0.8223567306995392,
      "rewards/ReflectionBalance/std": 0.23842703551054,
      "step": 1495,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 485.859375,
      "completions/min_length": 167.5,
      "epoch": 8.086253369272237,
      "grad_norm": 3.18251121886562,
      "kl": 3.1484375,
      "learning_rate": 8.942865952254469e-08,
      "loss": 0.23874977231025696,
      "memory(GiB)": 75.26,
      "reward": 1.9354809522628784,
      "reward_std": 0.5714271366596222,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3285723924636841,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.4427928179502487,
      "rewards/ReflectionBalance/mean": 0.7909497022628784,
      "rewards/ReflectionBalance/std": 0.29253970086574554,
      "step": 1496,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 488.96484375,
      "completions/min_length": 252.0,
      "epoch": 8.091644204851752,
      "grad_norm": 2.863360466244581,
      "kl": 1.49609375,
      "learning_rate": 8.893964666467002e-08,
      "loss": 0.12216698378324509,
      "memory(GiB)": 75.26,
      "reward": 1.976902723312378,
      "reward_std": 0.25535061210393906,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20146947354078293,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.370050385594368,
      "rewards/ReflectionBalance/mean": 0.8362776637077332,
      "rewards/ReflectionBalance/std": 0.1793215572834015,
      "step": 1497,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 509.8125,
      "completions/min_length": 253.5,
      "epoch": 8.097035040431267,
      "grad_norm": 3.1168861131879333,
      "kl": 2.6171875,
      "learning_rate": 8.84518439270815e-08,
      "loss": 0.23248544335365295,
      "memory(GiB)": 75.26,
      "reward": 1.8688639402389526,
      "reward_std": 0.4888351559638977,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2918590381741524,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3460448384284973,
      "rewards/ReflectionBalance/mean": 0.8258951902389526,
      "rewards/ReflectionBalance/std": 0.20903410762548447,
      "step": 1498,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 479.74609375,
      "completions/min_length": 258.0,
      "epoch": 8.102425876010782,
      "grad_norm": 10.125180585821518,
      "kl": 1.84765625,
      "learning_rate": 8.796525274582156e-08,
      "loss": 0.15594235062599182,
      "memory(GiB)": 75.26,
      "reward": 1.9477376341819763,
      "reward_std": 0.45160824060440063,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3853000998497009,
      "rewards/ReflectionBalance/mean": 0.8227376341819763,
      "rewards/ReflectionBalance/std": 0.2469528391957283,
      "step": 1499,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 503.9453125,
      "completions/min_length": 299.5,
      "epoch": 8.107816711590296,
      "grad_norm": 8.882583738151702,
      "kl": 2.375,
      "learning_rate": 8.747987455336541e-08,
      "loss": 0.21987922489643097,
      "memory(GiB)": 75.26,
      "reward": 1.969408392906189,
      "reward_std": 0.5217000842094421,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4187151938676834,
      "rewards/ReflectionBalance/mean": 0.8131583631038666,
      "rewards/ReflectionBalance/std": 0.2890012413263321,
      "step": 1500,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 506.0,
      "completions/min_length": 269.0,
      "epoch": 8.11320754716981,
      "grad_norm": 3.5316409881673594,
      "kl": 1.8203125,
      "learning_rate": 8.699571077861773e-08,
      "loss": 0.17015163600444794,
      "memory(GiB)": 75.26,
      "reward": 1.9194514751434326,
      "reward_std": 0.4492191821336746,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3666747808456421,
      "rewards/ReflectionBalance/mean": 0.8178889453411102,
      "rewards/ReflectionBalance/std": 0.2480148747563362,
      "step": 1501,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 484.79296875,
      "completions/min_length": 243.0,
      "epoch": 8.118598382749326,
      "grad_norm": 3.4272252716310714,
      "kl": 1.61328125,
      "learning_rate": 8.651276284690778e-08,
      "loss": 0.12311924993991852,
      "memory(GiB)": 75.26,
      "reward": 1.932382047176361,
      "reward_std": 0.41926634311676025,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24081215262413025,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3707977831363678,
      "rewards/ReflectionBalance/mean": 0.8269132077693939,
      "rewards/ReflectionBalance/std": 0.2051931470632553,
      "step": 1502,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 492.671875,
      "completions/min_length": 281.5,
      "epoch": 8.123989218328841,
      "grad_norm": 3.192350466770904,
      "kl": 2.01171875,
      "learning_rate": 8.603103217998559e-08,
      "loss": 0.19014140963554382,
      "memory(GiB)": 75.26,
      "reward": 1.9638363122940063,
      "reward_std": 0.4363660216331482,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.39705096185207367,
      "rewards/ReflectionBalance/mean": 0.823211282491684,
      "rewards/ReflectionBalance/std": 0.2651531547307968,
      "step": 1503,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 489.5078125,
      "completions/min_length": 239.0,
      "epoch": 8.129380053908356,
      "grad_norm": 6.366318939267174,
      "kl": 2.921875,
      "learning_rate": 8.555052019601794e-08,
      "loss": 0.24365468323230743,
      "memory(GiB)": 75.26,
      "reward": 1.8858724236488342,
      "reward_std": 0.5574371516704559,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32746291160583496,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.41230008006095886,
      "rewards/ReflectionBalance/mean": 0.7921223938465118,
      "rewards/ReflectionBalance/std": 0.3082384020090103,
      "step": 1504,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 907.5,
      "completions/mean_length": 455.6875,
      "completions/min_length": 265.0,
      "epoch": 8.134770889487871,
      "grad_norm": 3.0963946355701792,
      "kl": 1.4521484375,
      "learning_rate": 8.507122830958358e-08,
      "loss": 0.12909017503261566,
      "memory(GiB)": 75.26,
      "reward": 2.165962040424347,
      "reward_std": 0.39745567739009857,
      "rewards/Format/mean": 0.9609375,
      "rewards/Format/std": 0.1347113400697708,
      "rewards/MultiModalAccuracyORM/mean": 0.359375,
      "rewards/MultiModalAccuracyORM/std": 0.4697897136211395,
      "rewards/ReflectionBalance/mean": 0.845649391412735,
      "rewards/ReflectionBalance/std": 0.15824264287948608,
      "step": 1505,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 485.609375,
      "completions/min_length": 278.0,
      "epoch": 8.140161725067385,
      "grad_norm": 2.8056085006993126,
      "kl": 1.35546875,
      "learning_rate": 8.45931579316696e-08,
      "loss": 0.125803142786026,
      "memory(GiB)": 75.26,
      "reward": 2.056676506996155,
      "reward_std": 0.4031402915716171,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.20884444564580917,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.43903933465480804,
      "rewards/ReflectionBalance/mean": 0.83402019739151,
      "rewards/ReflectionBalance/std": 0.20722748339176178,
      "step": 1506,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 493.98828125,
      "completions/min_length": 210.0,
      "epoch": 8.1455525606469,
      "grad_norm": 2.747643116444821,
      "kl": 1.978515625,
      "learning_rate": 8.411631046966694e-08,
      "loss": 0.15591591596603394,
      "memory(GiB)": 75.26,
      "reward": 1.9304231405258179,
      "reward_std": 0.4121400862932205,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.25334735214710236,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38489311933517456,
      "rewards/ReflectionBalance/mean": 0.8288606405258179,
      "rewards/ReflectionBalance/std": 0.21155836433172226,
      "step": 1507,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 481.0,
      "completions/min_length": 266.0,
      "epoch": 8.150943396226415,
      "grad_norm": 3.15052370596474,
      "kl": 2.53125,
      "learning_rate": 8.364068732736674e-08,
      "loss": 0.19134482741355896,
      "memory(GiB)": 75.26,
      "reward": 1.9344449043273926,
      "reward_std": 0.5140318870544434,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4067341536283493,
      "rewards/ReflectionBalance/mean": 0.8133512437343597,
      "rewards/ReflectionBalance/std": 0.2182609960436821,
      "step": 1508,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 484.265625,
      "completions/min_length": 272.0,
      "epoch": 8.15633423180593,
      "grad_norm": 3.4050304645085783,
      "kl": 2.28515625,
      "learning_rate": 8.31662899049555e-08,
      "loss": 0.18175268173217773,
      "memory(GiB)": 75.26,
      "reward": 1.997438132762909,
      "reward_std": 0.49004851281642914,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4492545574903488,
      "rewards/ReflectionBalance/mean": 0.8138444125652313,
      "rewards/ReflectionBalance/std": 0.2460038885474205,
      "step": 1509,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 473.375,
      "completions/min_length": 271.5,
      "epoch": 8.161725067385445,
      "grad_norm": 3.9187443984326533,
      "kl": 2.515625,
      "learning_rate": 8.269311959901137e-08,
      "loss": 0.19602778553962708,
      "memory(GiB)": 75.26,
      "reward": 1.9177880883216858,
      "reward_std": 0.46024686098098755,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4044223427772522,
      "rewards/ReflectionBalance/mean": 0.8006005585193634,
      "rewards/ReflectionBalance/std": 0.2865821421146393,
      "step": 1510,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 495.79296875,
      "completions/min_length": 286.5,
      "epoch": 8.167115902964959,
      "grad_norm": 2.8559757016413556,
      "kl": 2.265625,
      "learning_rate": 8.222117780250037e-08,
      "loss": 0.19209271669387817,
      "memory(GiB)": 75.26,
      "reward": 1.8856136202812195,
      "reward_std": 0.421516090631485,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.336664155125618,
      "rewards/ReflectionBalance/mean": 0.8309260904788971,
      "rewards/ReflectionBalance/std": 0.21038121730089188,
      "step": 1511,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 472.265625,
      "completions/min_length": 197.0,
      "epoch": 8.172506738544474,
      "grad_norm": 3.0267164594005047,
      "kl": 2.515625,
      "learning_rate": 8.175046590477119e-08,
      "loss": 0.18950536847114563,
      "memory(GiB)": 75.26,
      "reward": 1.9159117937088013,
      "reward_std": 0.4221895933151245,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.26183828711509705,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38837648928165436,
      "rewards/ReflectionBalance/mean": 0.8182555437088013,
      "rewards/ReflectionBalance/std": 0.2119358405470848,
      "step": 1512,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 505.93359375,
      "completions/min_length": 284.0,
      "epoch": 8.177897574123989,
      "grad_norm": 3.295957544073556,
      "kl": 2.625,
      "learning_rate": 8.128098529155252e-08,
      "loss": 0.209526926279068,
      "memory(GiB)": 75.26,
      "reward": 1.763247013092041,
      "reward_std": 0.4909578263759613,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.078125,
      "rewards/MultiModalAccuracyORM/std": 0.25334735214710236,
      "rewards/ReflectionBalance/mean": 0.7905907928943634,
      "rewards/ReflectionBalance/std": 0.281817302107811,
      "step": 1513,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 520.55859375,
      "completions/min_length": 163.0,
      "epoch": 8.183288409703504,
      "grad_norm": 4.018889062262381,
      "kl": 3.4296875,
      "learning_rate": 8.081273734494826e-08,
      "loss": 0.25310924649238586,
      "memory(GiB)": 75.26,
      "reward": 1.8588272333145142,
      "reward_std": 0.5563302934169769,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.344347283244133,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.39601969718933105,
      "rewards/ReflectionBalance/mean": 0.804139643907547,
      "rewards/ReflectionBalance/std": 0.2381543070077896,
      "step": 1514,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 501.0703125,
      "completions/min_length": 280.0,
      "epoch": 8.18867924528302,
      "grad_norm": 4.072411272159075,
      "kl": 4.0625,
      "learning_rate": 8.034572344343265e-08,
      "loss": 0.30995744466781616,
      "memory(GiB)": 75.26,
      "reward": 1.9561663269996643,
      "reward_std": 0.5858877301216125,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32746291160583496,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.45274218916893005,
      "rewards/ReflectionBalance/mean": 0.7921038866043091,
      "rewards/ReflectionBalance/std": 0.28413210064172745,
      "step": 1515,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 477.34375,
      "completions/min_length": 256.0,
      "epoch": 8.194070080862534,
      "grad_norm": 3.2112943593616796,
      "kl": 3.0390625,
      "learning_rate": 7.987994496184802e-08,
      "loss": 0.24697348475456238,
      "memory(GiB)": 75.26,
      "reward": 1.9149837493896484,
      "reward_std": 0.5155171155929565,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38784652948379517,
      "rewards/ReflectionBalance/mean": 0.805608719587326,
      "rewards/ReflectionBalance/std": 0.2665465995669365,
      "step": 1516,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.0,
      "completions/mean_length": 479.1328125,
      "completions/min_length": 245.0,
      "epoch": 8.199460916442048,
      "grad_norm": 3.362512686123735,
      "kl": 2.796875,
      "learning_rate": 7.941540327139917e-08,
      "loss": 0.22805126011371613,
      "memory(GiB)": 75.26,
      "reward": 1.8781022429466248,
      "reward_std": 0.5395990610122681,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3786114752292633,
      "rewards/ReflectionBalance/mean": 0.8077897131443024,
      "rewards/ReflectionBalance/std": 0.23284442722797394,
      "step": 1517,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 531.59375,
      "completions/min_length": 262.5,
      "epoch": 8.204851752021563,
      "grad_norm": 3.681943665157458,
      "kl": 3.3984375,
      "learning_rate": 7.895209973964994e-08,
      "loss": 0.27360856533050537,
      "memory(GiB)": 75.26,
      "reward": 1.8401156067848206,
      "reward_std": 0.4762252867221832,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.32267695665359497,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3526121824979782,
      "rewards/ReflectionBalance/mean": 0.8127717971801758,
      "rewards/ReflectionBalance/std": 0.23814328014850616,
      "step": 1518,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 477.78125,
      "completions/min_length": 264.5,
      "epoch": 8.210242587601078,
      "grad_norm": 5.098943248259632,
      "kl": 2.8125,
      "learning_rate": 7.849003573051955e-08,
      "loss": 0.19708313047885895,
      "memory(GiB)": 75.26,
      "reward": 1.8918359279632568,
      "reward_std": 0.4436578005552292,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3842218816280365,
      "rewards/ReflectionBalance/mean": 0.8019921779632568,
      "rewards/ReflectionBalance/std": 0.2797245681285858,
      "step": 1519,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 481.01171875,
      "completions/min_length": 278.0,
      "epoch": 8.215633423180593,
      "grad_norm": 3.6105779807812364,
      "kl": 2.6328125,
      "learning_rate": 7.802921260427769e-08,
      "loss": 0.20849117636680603,
      "memory(GiB)": 75.26,
      "reward": 1.995371162891388,
      "reward_std": 0.5068108290433884,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.3046875,
      "rewards/MultiModalAccuracyORM/std": 0.46176913380622864,
      "rewards/ReflectionBalance/mean": 0.7961523234844208,
      "rewards/ReflectionBalance/std": 0.26377037912607193,
      "step": 1520,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 519.5078125,
      "completions/min_length": 171.0,
      "epoch": 8.221024258760108,
      "grad_norm": 3.591901647367374,
      "kl": 2.8125,
      "learning_rate": 7.756963171754133e-08,
      "loss": 0.21197712421417236,
      "memory(GiB)": 75.26,
      "reward": 1.950312614440918,
      "reward_std": 0.5524044185876846,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3265802264213562,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.45274218916893005,
      "rewards/ReflectionBalance/mean": 0.7901562452316284,
      "rewards/ReflectionBalance/std": 0.28008826822042465,
      "step": 1521,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 513.06640625,
      "completions/min_length": 293.0,
      "epoch": 8.226415094339623,
      "grad_norm": 3.607939581862826,
      "kl": 3.3359375,
      "learning_rate": 7.711129442327019e-08,
      "loss": 0.2654925286769867,
      "memory(GiB)": 75.26,
      "reward": 1.871019721031189,
      "reward_std": 0.5253527164459229,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3247780203819275,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.35859281569719315,
      "rewards/ReflectionBalance/mean": 0.7928946912288666,
      "rewards/ReflectionBalance/std": 0.28396619856357574,
      "step": 1522,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 905.5,
      "completions/mean_length": 487.4453125,
      "completions/min_length": 241.5,
      "epoch": 8.231805929919137,
      "grad_norm": 2.926248680146787,
      "kl": 2.37109375,
      "learning_rate": 7.665420207076334e-08,
      "loss": 0.2009916603565216,
      "memory(GiB)": 75.26,
      "reward": 2.04250168800354,
      "reward_std": 0.4230402857065201,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.2379390001296997,
      "rewards/MultiModalAccuracyORM/mean": 0.28125,
      "rewards/MultiModalAccuracyORM/std": 0.45002390444278717,
      "rewards/ReflectionBalance/mean": 0.8237516283988953,
      "rewards/ReflectionBalance/std": 0.24307524412870407,
      "step": 1523,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 494.91015625,
      "completions/min_length": 255.5,
      "epoch": 8.237196765498652,
      "grad_norm": 4.143793600437475,
      "kl": 3.40625,
      "learning_rate": 7.619835600565443e-08,
      "loss": 0.26206570863723755,
      "memory(GiB)": 75.26,
      "reward": 1.9346550107002258,
      "reward_std": 0.4538128972053528,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4065109044313431,
      "rewards/ReflectionBalance/mean": 0.817467451095581,
      "rewards/ReflectionBalance/std": 0.26979272067546844,
      "step": 1524,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 494.34765625,
      "completions/min_length": 252.0,
      "epoch": 8.242587601078167,
      "grad_norm": 3.092410679668142,
      "kl": 2.8515625,
      "learning_rate": 7.574375756990842e-08,
      "loss": 0.23402148485183716,
      "memory(GiB)": 75.26,
      "reward": 1.8692659139633179,
      "reward_std": 0.5163249671459198,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3679620176553726,
      "rewards/ReflectionBalance/mean": 0.8028597235679626,
      "rewards/ReflectionBalance/std": 0.25145217776298523,
      "step": 1525,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 513.46875,
      "completions/min_length": 284.5,
      "epoch": 8.247978436657682,
      "grad_norm": 3.1718178126344103,
      "kl": 2.53125,
      "learning_rate": 7.529040810181759e-08,
      "loss": 0.216567724943161,
      "memory(GiB)": 75.26,
      "reward": 1.8918229937553406,
      "reward_std": 0.48004113137722015,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.35259076952934265,
      "rewards/ReflectionBalance/mean": 0.8410416841506958,
      "rewards/ReflectionBalance/std": 0.20442699640989304,
      "step": 1526,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 512.59765625,
      "completions/min_length": 250.5,
      "epoch": 8.253369272237197,
      "grad_norm": 3.349044864949932,
      "kl": 2.12890625,
      "learning_rate": 7.483830893599664e-08,
      "loss": 0.1502997726202011,
      "memory(GiB)": 75.26,
      "reward": 1.8692490458488464,
      "reward_std": 0.43059808015823364,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.27558503299951553,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3607245087623596,
      "rewards/ReflectionBalance/mean": 0.802842766046524,
      "rewards/ReflectionBalance/std": 0.2563943490386009,
      "step": 1527,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 519.9375,
      "completions/min_length": 291.5,
      "epoch": 8.25876010781671,
      "grad_norm": 3.055342897595432,
      "kl": 2.8828125,
      "learning_rate": 7.438746140338031e-08,
      "loss": 0.25025561451911926,
      "memory(GiB)": 75.26,
      "reward": 1.8838549852371216,
      "reward_std": 0.5415955483913422,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3682406395673752,
      "rewards/ReflectionBalance/mean": 0.8096362352371216,
      "rewards/ReflectionBalance/std": 0.27266034483909607,
      "step": 1528,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 491.65625,
      "completions/min_length": 259.5,
      "epoch": 8.264150943396226,
      "grad_norm": 2.9255401259135523,
      "kl": 2.33984375,
      "learning_rate": 7.393786683121833e-08,
      "loss": 0.203220397233963,
      "memory(GiB)": 75.26,
      "reward": 1.9412280321121216,
      "reward_std": 0.4794440418481827,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4077630192041397,
      "rewards/ReflectionBalance/mean": 0.804509311914444,
      "rewards/ReflectionBalance/std": 0.2633124887943268,
      "step": 1529,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 489.15234375,
      "completions/min_length": 259.0,
      "epoch": 8.269541778975741,
      "grad_norm": 3.1075548035341924,
      "kl": 1.77734375,
      "learning_rate": 7.348952654307161e-08,
      "loss": 0.16968992352485657,
      "memory(GiB)": 75.26,
      "reward": 2.0238672494888306,
      "reward_std": 0.47652195394039154,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4331994354724884,
      "rewards/ReflectionBalance/mean": 0.8480859398841858,
      "rewards/ReflectionBalance/std": 0.18495460599660873,
      "step": 1530,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 502.7890625,
      "completions/min_length": 281.5,
      "epoch": 8.274932614555256,
      "grad_norm": 3.584535981346829,
      "kl": 1.99609375,
      "learning_rate": 7.304244185880926e-08,
      "loss": 0.16065357625484467,
      "memory(GiB)": 75.26,
      "reward": 1.8930988907814026,
      "reward_std": 0.500692754983902,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2829600051045418,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.7954427301883698,
      "rewards/ReflectionBalance/std": 0.27998265624046326,
      "step": 1531,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 493.15234375,
      "completions/min_length": 280.0,
      "epoch": 8.280323450134771,
      "grad_norm": 91.01658522257762,
      "kl": 2.1640625,
      "learning_rate": 7.259661409460305e-08,
      "loss": 0.18556803464889526,
      "memory(GiB)": 75.26,
      "reward": 1.9579133987426758,
      "reward_std": 0.4432370215654373,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.4164551943540573,
      "rewards/ReflectionBalance/mean": 0.8094758987426758,
      "rewards/ReflectionBalance/std": 0.23376046866178513,
      "step": 1532,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 888.5,
      "completions/mean_length": 485.3984375,
      "completions/min_length": 196.0,
      "epoch": 8.285714285714286,
      "grad_norm": 2.800215466072086,
      "kl": 1.6796875,
      "learning_rate": 7.215204456292534e-08,
      "loss": 0.1253637671470642,
      "memory(GiB)": 75.26,
      "reward": 1.9870353937149048,
      "reward_std": 0.3737320303916931,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.19582369551062584,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.38315413892269135,
      "rewards/ReflectionBalance/mean": 0.8503165543079376,
      "rewards/ReflectionBalance/std": 0.17742080241441727,
      "step": 1533,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 953.5,
      "completions/mean_length": 507.12109375,
      "completions/min_length": 253.5,
      "epoch": 8.2911051212938,
      "grad_norm": 2.985488523710387,
      "kl": 1.408203125,
      "learning_rate": 7.1708734572544e-08,
      "loss": 0.10909222066402435,
      "memory(GiB)": 75.26,
      "reward": 2.0047411918640137,
      "reward_std": 0.4161745309829712,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2280246615409851,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4169354736804962,
      "rewards/ReflectionBalance/mean": 0.8289599716663361,
      "rewards/ReflectionBalance/std": 0.23364467173814774,
      "step": 1534,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 494.82421875,
      "completions/min_length": 267.0,
      "epoch": 8.296495956873315,
      "grad_norm": 2.8832146156756258,
      "kl": 2.375,
      "learning_rate": 7.126668542851889e-08,
      "loss": 0.18521153926849365,
      "memory(GiB)": 75.26,
      "reward": 1.8839803338050842,
      "reward_std": 0.43692004680633545,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3716365247964859,
      "rewards/ReflectionBalance/mean": 0.8058553338050842,
      "rewards/ReflectionBalance/std": 0.2551134526729584,
      "step": 1535,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 496.89453125,
      "completions/min_length": 264.0,
      "epoch": 8.30188679245283,
      "grad_norm": 3.4482617908116184,
      "kl": 2.6171875,
      "learning_rate": 7.082589843219833e-08,
      "loss": 0.21646231412887573,
      "memory(GiB)": 75.26,
      "reward": 1.9521784782409668,
      "reward_std": 0.47854918241500854,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41328008472919464,
      "rewards/ReflectionBalance/mean": 0.8310848474502563,
      "rewards/ReflectionBalance/std": 0.21382179111242294,
      "step": 1536,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 508.67578125,
      "completions/min_length": 271.0,
      "epoch": 8.307277628032345,
      "grad_norm": 2.758775468652206,
      "kl": 1.50390625,
      "learning_rate": 7.038637488121486e-08,
      "loss": 0.11967888474464417,
      "memory(GiB)": 75.26,
      "reward": 2.0081608295440674,
      "reward_std": 0.32769832015037537,
      "rewards/Format/mean": 0.953125,
      "rewards/Format/std": 0.20884444564580917,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.37897978723049164,
      "rewards/ReflectionBalance/mean": 0.8519108295440674,
      "rewards/ReflectionBalance/std": 0.16998812183737755,
      "step": 1537,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 469.7578125,
      "completions/min_length": 276.5,
      "epoch": 8.31266846361186,
      "grad_norm": 3.9357997709199775,
      "kl": 1.5078125,
      "learning_rate": 6.994811606948142e-08,
      "loss": 0.12554237246513367,
      "memory(GiB)": 75.26,
      "reward": 1.9468716979026794,
      "reward_std": 0.40342114865779877,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.39950306713581085,
      "rewards/ReflectionBalance/mean": 0.8257780075073242,
      "rewards/ReflectionBalance/std": 0.18025831878185272,
      "step": 1538,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 941.0,
      "completions/mean_length": 485.46484375,
      "completions/min_length": 274.5,
      "epoch": 8.318059299191376,
      "grad_norm": 3.0520308463354944,
      "kl": 2.68359375,
      "learning_rate": 6.951112328718783e-08,
      "loss": 0.19708997011184692,
      "memory(GiB)": 75.26,
      "reward": 1.8372704982757568,
      "reward_std": 0.49762727320194244,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3208130896091461,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.36930377781391144,
      "rewards/ReflectionBalance/mean": 0.7825829982757568,
      "rewards/ReflectionBalance/std": 0.2472488135099411,
      "step": 1539,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 514.94921875,
      "completions/min_length": 281.0,
      "epoch": 8.323450134770889,
      "grad_norm": 3.14113224972472,
      "kl": 2.4765625,
      "learning_rate": 6.907539782079696e-08,
      "loss": 0.19601969420909882,
      "memory(GiB)": 75.26,
      "reward": 1.841844081878662,
      "reward_std": 0.5129604935646057,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3526121824979782,
      "rewards/ReflectionBalance/mean": 0.7871565520763397,
      "rewards/ReflectionBalance/std": 0.2871025502681732,
      "step": 1540,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 937.5,
      "completions/mean_length": 462.81640625,
      "completions/min_length": 228.5,
      "epoch": 8.328840970350404,
      "grad_norm": 3.140337881537254,
      "kl": 2.76171875,
      "learning_rate": 6.864094095304069e-08,
      "loss": 0.22311076521873474,
      "memory(GiB)": 75.26,
      "reward": 1.931787133216858,
      "reward_std": 0.5030151605606079,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.28863146156072617,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41328008472919464,
      "rewards/ReflectionBalance/mean": 0.8106933534145355,
      "rewards/ReflectionBalance/std": 0.23575058579444885,
      "step": 1541,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 500.1640625,
      "completions/min_length": 203.0,
      "epoch": 8.33423180592992,
      "grad_norm": 4.105139223659487,
      "kl": 1.65625,
      "learning_rate": 6.820775396291606e-08,
      "loss": 0.12278472632169724,
      "memory(GiB)": 75.26,
      "reward": 2.0125374794006348,
      "reward_std": 0.4883074164390564,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.265625,
      "rewards/MultiModalAccuracyORM/std": 0.439003586769104,
      "rewards/ReflectionBalance/mean": 0.81722491979599,
      "rewards/ReflectionBalance/std": 0.21540524810552597,
      "step": 1542,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 486.953125,
      "completions/min_length": 251.5,
      "epoch": 8.339622641509434,
      "grad_norm": 3.658946699426066,
      "kl": 2.5234375,
      "learning_rate": 6.777583812568233e-08,
      "loss": 0.22760477662086487,
      "memory(GiB)": 75.26,
      "reward": 1.8989111185073853,
      "reward_std": 0.44325776398181915,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3586205691099167,
      "rewards/ReflectionBalance/mean": 0.8285986483097076,
      "rewards/ReflectionBalance/std": 0.22968700528144836,
      "step": 1543,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 505.53515625,
      "completions/min_length": 258.5,
      "epoch": 8.34501347708895,
      "grad_norm": 2.9930075721108422,
      "kl": 2.6953125,
      "learning_rate": 6.734519471285582e-08,
      "loss": 0.21831901371479034,
      "memory(GiB)": 75.26,
      "reward": 1.8571581840515137,
      "reward_std": 0.4933706670999527,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3643587827682495,
      "rewards/ReflectionBalance/mean": 0.7829394638538361,
      "rewards/ReflectionBalance/std": 0.28510333597660065,
      "step": 1544,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 499.7421875,
      "completions/min_length": 198.0,
      "epoch": 8.350404312668463,
      "grad_norm": 3.224217444589681,
      "kl": 2.6953125,
      "learning_rate": 6.69158249922076e-08,
      "loss": 0.2285172939300537,
      "memory(GiB)": 75.26,
      "reward": 1.9511873126029968,
      "reward_std": 0.5451172739267349,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4300411641597748,
      "rewards/ReflectionBalance/mean": 0.8105623126029968,
      "rewards/ReflectionBalance/std": 0.2538045421242714,
      "step": 1545,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 490.90234375,
      "completions/min_length": 239.0,
      "epoch": 8.355795148247978,
      "grad_norm": 2.95087980129944,
      "kl": 2.765625,
      "learning_rate": 6.648773022775905e-08,
      "loss": 0.2425607293844223,
      "memory(GiB)": 75.26,
      "reward": 1.961523413658142,
      "reward_std": 0.5529196560382843,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2875154986977577,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.44036537408828735,
      "rewards/ReflectionBalance/mean": 0.7818359136581421,
      "rewards/ReflectionBalance/std": 0.291030153632164,
      "step": 1546,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.0,
      "completions/mean_length": 479.57421875,
      "completions/min_length": 259.0,
      "epoch": 8.361185983827493,
      "grad_norm": 3.147506731226104,
      "kl": 2.6171875,
      "learning_rate": 6.606091167977767e-08,
      "loss": 0.2208828330039978,
      "memory(GiB)": 75.26,
      "reward": 1.7895084619522095,
      "reward_std": 0.4611152410507202,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.078125,
      "rewards/MultiModalAccuracyORM/std": 0.2690252959728241,
      "rewards/ReflectionBalance/mean": 0.8168522417545319,
      "rewards/ReflectionBalance/std": 0.2518806606531143,
      "step": 1547,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 508.9921875,
      "completions/min_length": 290.5,
      "epoch": 8.366576819407008,
      "grad_norm": 3.375819565528694,
      "kl": 3.140625,
      "learning_rate": 6.563537060477459e-08,
      "loss": 0.22563277184963226,
      "memory(GiB)": 75.26,
      "reward": 1.7442976832389832,
      "reward_std": 0.5352703630924225,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3488178104162216,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.26531852036714554,
      "rewards/ReflectionBalance/mean": 0.7872664332389832,
      "rewards/ReflectionBalance/std": 0.2947675734758377,
      "step": 1548,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 531.98046875,
      "completions/min_length": 274.5,
      "epoch": 8.371967654986523,
      "grad_norm": 3.432450000339759,
      "kl": 2.96875,
      "learning_rate": 6.521110825549974e-08,
      "loss": 0.24058571457862854,
      "memory(GiB)": 75.26,
      "reward": 1.8389273881912231,
      "reward_std": 0.5115049481391907,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3681531548500061,
      "rewards/ReflectionBalance/mean": 0.7842398881912231,
      "rewards/ReflectionBalance/std": 0.32660040259361267,
      "step": 1549,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.0,
      "completions/mean_length": 504.75,
      "completions/min_length": 258.0,
      "epoch": 8.377358490566039,
      "grad_norm": 3.0699465493423577,
      "kl": 2.125,
      "learning_rate": 6.478812588093863e-08,
      "loss": 0.17080199718475342,
      "memory(GiB)": 75.26,
      "reward": 1.7755957245826721,
      "reward_std": 0.4765811860561371,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.06640625,
      "rewards/MultiModalAccuracyORM/std": 0.24678732454776764,
      "rewards/ReflectionBalance/mean": 0.8107519745826721,
      "rewards/ReflectionBalance/std": 0.2679991126060486,
      "step": 1550,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 950.0,
      "completions/mean_length": 488.7734375,
      "completions/min_length": 261.5,
      "epoch": 8.382749326145552,
      "grad_norm": 3.698515767287646,
      "kl": 1.33203125,
      "learning_rate": 6.43664247263091e-08,
      "loss": 0.08742914348840714,
      "memory(GiB)": 75.26,
      "reward": 1.9485937356948853,
      "reward_std": 0.39430877566337585,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38047872483730316,
      "rewards/ReflectionBalance/mean": 0.8392187356948853,
      "rewards/ReflectionBalance/std": 0.1933797523379326,
      "step": 1551,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1015.0,
      "completions/mean_length": 505.9296875,
      "completions/min_length": 233.0,
      "epoch": 8.388140161725067,
      "grad_norm": 3.031548539601977,
      "kl": 2.78125,
      "learning_rate": 6.394600603305667e-08,
      "loss": 0.1968688666820526,
      "memory(GiB)": 75.26,
      "reward": 1.7747843861579895,
      "reward_std": 0.587814062833786,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3247780203819275,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3240884244441986,
      "rewards/ReflectionBalance/mean": 0.7708781063556671,
      "rewards/ReflectionBalance/std": 0.3226960599422455,
      "step": 1552,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 482.1875,
      "completions/min_length": 260.5,
      "epoch": 8.393530997304582,
      "grad_norm": 3.6003882071105138,
      "kl": 2.4375,
      "learning_rate": 6.352687103885169e-08,
      "loss": 0.1865113526582718,
      "memory(GiB)": 75.26,
      "reward": 1.992470622062683,
      "reward_std": 0.5541642606258392,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.296875,
      "rewards/MultiModalAccuracyORM/std": 0.45209212601184845,
      "rewards/ReflectionBalance/mean": 0.8049707114696503,
      "rewards/ReflectionBalance/std": 0.23556455224752426,
      "step": 1553,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 497.83984375,
      "completions/min_length": 263.0,
      "epoch": 8.398921832884097,
      "grad_norm": 3.3144057111931624,
      "kl": 2.7890625,
      "learning_rate": 6.310902097758536e-08,
      "loss": 0.22943831980228424,
      "memory(GiB)": 75.26,
      "reward": 1.8974837064743042,
      "reward_std": 0.5363713800907135,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2813730239868164,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3854355216026306,
      "rewards/ReflectionBalance/mean": 0.8037337064743042,
      "rewards/ReflectionBalance/std": 0.2978953421115875,
      "step": 1554,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 466.375,
      "completions/min_length": 261.0,
      "epoch": 8.404312668463612,
      "grad_norm": 3.103740218828811,
      "kl": 2.7578125,
      "learning_rate": 6.269245707936604e-08,
      "loss": 0.2599206566810608,
      "memory(GiB)": 75.26,
      "reward": 1.8268635869026184,
      "reward_std": 0.47201862931251526,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.26755452156066895,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.3032590448856354,
      "rewards/ReflectionBalance/mean": 0.8073323965072632,
      "rewards/ReflectionBalance/std": 0.26830407977104187,
      "step": 1555,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 524.1875,
      "completions/min_length": 242.5,
      "epoch": 8.409703504043126,
      "grad_norm": 3.3714016520146752,
      "kl": 3.6875,
      "learning_rate": 6.227718057051624e-08,
      "loss": 0.3174428343772888,
      "memory(GiB)": 75.26,
      "reward": 1.8930379748344421,
      "reward_std": 0.525348961353302,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32612966001033783,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.40016549825668335,
      "rewards/ReflectionBalance/mean": 0.8071004152297974,
      "rewards/ReflectionBalance/std": 0.27089786529541016,
      "step": 1556,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 499.99609375,
      "completions/min_length": 289.5,
      "epoch": 8.415094339622641,
      "grad_norm": 3.4474977826673685,
      "kl": 2.6640625,
      "learning_rate": 6.186319267356793e-08,
      "loss": 0.21051159501075745,
      "memory(GiB)": 75.26,
      "reward": 1.90669184923172,
      "reward_std": 0.46643367409706116,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.39601969718933105,
      "rewards/ReflectionBalance/mean": 0.8129419088363647,
      "rewards/ReflectionBalance/std": 0.2355044260621071,
      "step": 1557,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 499.68359375,
      "completions/min_length": 295.0,
      "epoch": 8.420485175202156,
      "grad_norm": 2.9578439414139313,
      "kl": 2.36328125,
      "learning_rate": 6.145049460725981e-08,
      "loss": 0.2018032819032669,
      "memory(GiB)": 75.26,
      "reward": 1.9101463556289673,
      "reward_std": 0.3819589763879776,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3723055571317673,
      "rewards/ReflectionBalance/mean": 0.8163964748382568,
      "rewards/ReflectionBalance/std": 0.2609846889972687,
      "step": 1558,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 509.93359375,
      "completions/min_length": 291.5,
      "epoch": 8.425876010781671,
      "grad_norm": 3.0669479688603682,
      "kl": 3.2265625,
      "learning_rate": 6.103908758653342e-08,
      "loss": 0.28645965456962585,
      "memory(GiB)": 75.26,
      "reward": 1.9439375400543213,
      "reward_std": 0.5720501840114594,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31333550810813904,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.44438336789608,
      "rewards/ReflectionBalance/mean": 0.7837812602519989,
      "rewards/ReflectionBalance/std": 0.2888075113296509,
      "step": 1559,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 496.8828125,
      "completions/min_length": 174.5,
      "epoch": 8.431266846361186,
      "grad_norm": 3.2714705214808895,
      "kl": 2.06640625,
      "learning_rate": 6.062897282252971e-08,
      "loss": 0.1741049736738205,
      "memory(GiB)": 75.26,
      "reward": 1.8973644971847534,
      "reward_std": 0.5138214081525803,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3749772012233734,
      "rewards/ReflectionBalance/mean": 0.8114270865917206,
      "rewards/ReflectionBalance/std": 0.25933176279067993,
      "step": 1560,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 516.875,
      "completions/min_length": 195.0,
      "epoch": 8.436657681940702,
      "grad_norm": 3.3731859942724083,
      "kl": 2.28125,
      "learning_rate": 6.02201515225852e-08,
      "loss": 0.1590953767299652,
      "memory(GiB)": 75.26,
      "reward": 1.945004940032959,
      "reward_std": 0.48786282539367676,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.41096827387809753,
      "rewards/ReflectionBalance/mean": 0.8200048804283142,
      "rewards/ReflectionBalance/std": 0.22488851100206375,
      "step": 1561,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 483.0859375,
      "completions/min_length": 275.5,
      "epoch": 8.442048517520215,
      "grad_norm": 3.166868800272938,
      "kl": 3.015625,
      "learning_rate": 5.981262489022848e-08,
      "loss": 0.22205641865730286,
      "memory(GiB)": 75.26,
      "reward": 1.98964524269104,
      "reward_std": 0.5248235762119293,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3320184051990509,
      "rewards/MultiModalAccuracyORM/mean": 0.3046875,
      "rewards/MultiModalAccuracyORM/std": 0.460112988948822,
      "rewards/ReflectionBalance/mean": 0.8099576830863953,
      "rewards/ReflectionBalance/std": 0.26276516914367676,
      "step": 1562,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 494.4140625,
      "completions/min_length": 287.5,
      "epoch": 8.44743935309973,
      "grad_norm": 3.762433240330657,
      "kl": 2.421875,
      "learning_rate": 5.9406394125177284e-08,
      "loss": 0.20780736207962036,
      "memory(GiB)": 75.26,
      "reward": 1.9998160600662231,
      "reward_std": 0.44360795617103577,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.2890625,
      "rewards/MultiModalAccuracyORM/std": 0.4547791928052902,
      "rewards/ReflectionBalance/mean": 0.7927848100662231,
      "rewards/ReflectionBalance/std": 0.2845708355307579,
      "step": 1563,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 484.46875,
      "completions/min_length": 287.5,
      "epoch": 8.452830188679245,
      "grad_norm": 4.168058228983483,
      "kl": 2.6015625,
      "learning_rate": 5.9001460423333595e-08,
      "loss": 0.2411036640405655,
      "memory(GiB)": 75.26,
      "reward": 1.932349443435669,
      "reward_std": 0.48600953817367554,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.40095221996307373,
      "rewards/ReflectionBalance/mean": 0.811255693435669,
      "rewards/ReflectionBalance/std": 0.22526399046182632,
      "step": 1564,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 489.12109375,
      "completions/min_length": 269.5,
      "epoch": 8.45822102425876,
      "grad_norm": 2.6153445167858083,
      "kl": 1.314453125,
      "learning_rate": 5.859782497678184e-08,
      "loss": 0.1283893585205078,
      "memory(GiB)": 75.26,
      "reward": 2.01604163646698,
      "reward_std": 0.37368375062942505,
      "rewards/Format/mean": 0.96484375,
      "rewards/Format/std": 0.1845906376838684,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.39185720682144165,
      "rewards/ReflectionBalance/mean": 0.85588538646698,
      "rewards/ReflectionBalance/std": 0.16625933349132538,
      "step": 1565,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 491.3046875,
      "completions/min_length": 273.0,
      "epoch": 8.463611859838275,
      "grad_norm": 2.7727144080828086,
      "kl": 1.96484375,
      "learning_rate": 5.819548897378396e-08,
      "loss": 0.1495371162891388,
      "memory(GiB)": 75.26,
      "reward": 1.9990184903144836,
      "reward_std": 0.484265074133873,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4112040400505066,
      "rewards/ReflectionBalance/mean": 0.8193310499191284,
      "rewards/ReflectionBalance/std": 0.24152955412864685,
      "step": 1566,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 502.42578125,
      "completions/min_length": 257.0,
      "epoch": 8.46900269541779,
      "grad_norm": 3.006524499694424,
      "kl": 2.6015625,
      "learning_rate": 5.779445359877661e-08,
      "loss": 0.20860835909843445,
      "memory(GiB)": 75.26,
      "reward": 1.9373111724853516,
      "reward_std": 0.5116670429706573,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4323427081108093,
      "rewards/ReflectionBalance/mean": 0.7927799224853516,
      "rewards/ReflectionBalance/std": 0.27145490050315857,
      "step": 1567,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 509.01171875,
      "completions/min_length": 292.5,
      "epoch": 8.474393530997304,
      "grad_norm": 4.304508576839499,
      "kl": 2.453125,
      "learning_rate": 5.739472003236762e-08,
      "loss": 0.21252043545246124,
      "memory(GiB)": 75.26,
      "reward": 1.8516837358474731,
      "reward_std": 0.48926717042922974,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.32746291160583496,
      "rewards/ReflectionBalance/mean": 0.8087150454521179,
      "rewards/ReflectionBalance/std": 0.2827114462852478,
      "step": 1568,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1034.0,
      "completions/mean_length": 502.7109375,
      "completions/min_length": 283.5,
      "epoch": 8.479784366576819,
      "grad_norm": 3.1520419475739203,
      "kl": 2.2734375,
      "learning_rate": 5.699628945133239e-08,
      "loss": 0.1967446506023407,
      "memory(GiB)": 75.26,
      "reward": 2.0847867727279663,
      "reward_std": 0.5249457657337189,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.328125,
      "rewards/MultiModalAccuracyORM/std": 0.4695202708244324,
      "rewards/ReflectionBalance/mean": 0.8269743025302887,
      "rewards/ReflectionBalance/std": 0.2511618658900261,
      "step": 1569,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1020.0,
      "completions/mean_length": 492.1796875,
      "completions/min_length": 239.5,
      "epoch": 8.485175202156334,
      "grad_norm": 2.910946841846052,
      "kl": 2.46875,
      "learning_rate": 5.659916302861034e-08,
      "loss": 0.20528988540172577,
      "memory(GiB)": 75.26,
      "reward": 1.8847030997276306,
      "reward_std": 0.5148428678512573,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32746291160583496,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3980395793914795,
      "rewards/ReflectionBalance/mean": 0.8026718497276306,
      "rewards/ReflectionBalance/std": 0.2750084549188614,
      "step": 1570,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 506.3046875,
      "completions/min_length": 196.0,
      "epoch": 8.49056603773585,
      "grad_norm": 3.7541841342992,
      "kl": 2.9140625,
      "learning_rate": 5.6203341933301615e-08,
      "loss": 0.21951228380203247,
      "memory(GiB)": 75.26,
      "reward": 1.8679655194282532,
      "reward_std": 0.44446389377117157,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3774644732475281,
      "rewards/ReflectionBalance/mean": 0.8015592396259308,
      "rewards/ReflectionBalance/std": 0.2688775509595871,
      "step": 1571,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 475.43359375,
      "completions/min_length": 272.5,
      "epoch": 8.495956873315365,
      "grad_norm": 4.018323641568099,
      "kl": 2.171875,
      "learning_rate": 5.580882733066383e-08,
      "loss": 0.17688193917274475,
      "memory(GiB)": 75.26,
      "reward": 1.9858251810073853,
      "reward_std": 0.4445549547672272,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.4298042804002762,
      "rewards/ReflectionBalance/mean": 0.8139502108097076,
      "rewards/ReflectionBalance/std": 0.25048378109931946,
      "step": 1572,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 487.08203125,
      "completions/min_length": 245.0,
      "epoch": 8.501347708894878,
      "grad_norm": 2.8026383273729647,
      "kl": 1.76171875,
      "learning_rate": 5.541562038210823e-08,
      "loss": 0.1357879936695099,
      "memory(GiB)": 75.26,
      "reward": 1.9415266513824463,
      "reward_std": 0.40599238872528076,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2488820105791092,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3762422651052475,
      "rewards/ReflectionBalance/mean": 0.8282454311847687,
      "rewards/ReflectionBalance/std": 0.23000343888998032,
      "step": 1573,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 489.24609375,
      "completions/min_length": 265.5,
      "epoch": 8.506738544474393,
      "grad_norm": 3.7773772939643617,
      "kl": 2.828125,
      "learning_rate": 5.5023722245196246e-08,
      "loss": 0.24405556917190552,
      "memory(GiB)": 75.26,
      "reward": 1.832395076751709,
      "reward_std": 0.4414885491132736,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27079907804727554,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.3208130896091461,
      "rewards/ReflectionBalance/mean": 0.7972387671470642,
      "rewards/ReflectionBalance/std": 0.2632836401462555,
      "step": 1574,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 481.6328125,
      "completions/min_length": 291.5,
      "epoch": 8.512129380053908,
      "grad_norm": 3.16828295287666,
      "kl": 2.40625,
      "learning_rate": 5.46331340736369e-08,
      "loss": 0.18962404131889343,
      "memory(GiB)": 75.26,
      "reward": 1.8905436396598816,
      "reward_std": 0.4008728712797165,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2813730239868164,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.3291765823960304,
      "rewards/ReflectionBalance/mean": 0.8280436396598816,
      "rewards/ReflectionBalance/std": 0.22170713543891907,
      "step": 1575,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 492.62890625,
      "completions/min_length": 243.5,
      "epoch": 8.517520215633423,
      "grad_norm": 3.160865443431781,
      "kl": 2.1484375,
      "learning_rate": 5.424385701728201e-08,
      "loss": 0.16742119193077087,
      "memory(GiB)": 75.26,
      "reward": 2.0199055671691895,
      "reward_std": 0.44332945346832275,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.265625,
      "rewards/MultiModalAccuracyORM/std": 0.44331194460392,
      "rewards/ReflectionBalance/mean": 0.8206868469715118,
      "rewards/ReflectionBalance/std": 0.2487816959619522,
      "step": 1576,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 955.0,
      "completions/mean_length": 486.56640625,
      "completions/min_length": 255.0,
      "epoch": 8.522911051212938,
      "grad_norm": 3.974576256300827,
      "kl": 2.046875,
      "learning_rate": 5.385589222212428e-08,
      "loss": 0.17082767188549042,
      "memory(GiB)": 75.26,
      "reward": 1.9826563000679016,
      "reward_std": 0.4596785008907318,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4323427081108093,
      "rewards/ReflectionBalance/mean": 0.8225000202655792,
      "rewards/ReflectionBalance/std": 0.2352335974574089,
      "step": 1577,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 478.46484375,
      "completions/min_length": 253.5,
      "epoch": 8.528301886792454,
      "grad_norm": 3.324825167078974,
      "kl": 2.625,
      "learning_rate": 5.346924083029314e-08,
      "loss": 0.21409036219120026,
      "memory(GiB)": 75.26,
      "reward": 1.8902954459190369,
      "reward_std": 0.5022547394037247,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38837648928165436,
      "rewards/ReflectionBalance/mean": 0.7887329161167145,
      "rewards/ReflectionBalance/std": 0.29827944934368134,
      "step": 1578,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 524.03515625,
      "completions/min_length": 203.5,
      "epoch": 8.533692722371967,
      "grad_norm": 4.499230541041895,
      "kl": 3.5703125,
      "learning_rate": 5.308390398005103e-08,
      "loss": 0.25820472836494446,
      "memory(GiB)": 75.26,
      "reward": 1.9452197551727295,
      "reward_std": 0.5280018150806427,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4312909245491028,
      "rewards/ReflectionBalance/mean": 0.7967822253704071,
      "rewards/ReflectionBalance/std": 0.27204084396362305,
      "step": 1579,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 482.33203125,
      "completions/min_length": 276.0,
      "epoch": 8.539083557951482,
      "grad_norm": 2.9805533402624453,
      "kl": 2.4609375,
      "learning_rate": 5.2699882805791016e-08,
      "loss": 0.19278523325920105,
      "memory(GiB)": 75.26,
      "reward": 2.0618343353271484,
      "reward_std": 0.4874502122402191,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2566775679588318,
      "rewards/MultiModalAccuracyORM/mean": 0.30859375,
      "rewards/MultiModalAccuracyORM/std": 0.46136754751205444,
      "rewards/ReflectionBalance/mean": 0.823553055524826,
      "rewards/ReflectionBalance/std": 0.21422803401947021,
      "step": 1580,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 509.765625,
      "completions/min_length": 291.0,
      "epoch": 8.544474393530997,
      "grad_norm": 8.478045469146023,
      "kl": 2.390625,
      "learning_rate": 5.231717843803274e-08,
      "loss": 0.20929498970508575,
      "memory(GiB)": 75.26,
      "reward": 1.967226505279541,
      "reward_std": 0.48523564636707306,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.420418843626976,
      "rewards/ReflectionBalance/mean": 0.8187890648841858,
      "rewards/ReflectionBalance/std": 0.24777282029390335,
      "step": 1581,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 497.1640625,
      "completions/min_length": 233.5,
      "epoch": 8.549865229110512,
      "grad_norm": 3.1950675598350124,
      "kl": 2.8046875,
      "learning_rate": 5.193579200341924e-08,
      "loss": 0.2547110617160797,
      "memory(GiB)": 75.26,
      "reward": 1.8193880319595337,
      "reward_std": 0.4622965008020401,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.339918777346611,
      "rewards/ReflectionBalance/mean": 0.7959505021572113,
      "rewards/ReflectionBalance/std": 0.2775277718901634,
      "step": 1582,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 495.53515625,
      "completions/min_length": 257.0,
      "epoch": 8.555256064690028,
      "grad_norm": 3.0703787287189397,
      "kl": 2.7578125,
      "learning_rate": 5.155572462471402e-08,
      "loss": 0.22581225633621216,
      "memory(GiB)": 75.26,
      "reward": 1.8182942867279053,
      "reward_std": 0.5051331520080566,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3229459524154663,
      "rewards/ReflectionBalance/mean": 0.8026692271232605,
      "rewards/ReflectionBalance/std": 0.27078548818826675,
      "step": 1583,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 858.0,
      "completions/mean_length": 461.56640625,
      "completions/min_length": 228.5,
      "epoch": 8.560646900269543,
      "grad_norm": 3.3557142454758417,
      "kl": 2.0,
      "learning_rate": 5.1176977420796854e-08,
      "loss": 0.15457969903945923,
      "memory(GiB)": 75.26,
      "reward": 2.044217109680176,
      "reward_std": 0.41050684452056885,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24678732454776764,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.38492654263973236,
      "rewards/ReflectionBalance/mean": 0.8410921096801758,
      "rewards/ReflectionBalance/std": 0.1936645731329918,
      "step": 1584,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 481.796875,
      "completions/min_length": 260.0,
      "epoch": 8.566037735849056,
      "grad_norm": 3.3179174693684077,
      "kl": 2.171875,
      "learning_rate": 5.079955150666154e-08,
      "loss": 0.17896384000778198,
      "memory(GiB)": 75.26,
      "reward": 1.9499626159667969,
      "reward_std": 0.4361727088689804,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23443959653377533,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38837648928165436,
      "rewards/ReflectionBalance/mean": 0.8249625563621521,
      "rewards/ReflectionBalance/std": 0.2588091939687729,
      "step": 1585,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 494.26953125,
      "completions/min_length": 254.5,
      "epoch": 8.571428571428571,
      "grad_norm": 3.8644862527781023,
      "kl": 2.7109375,
      "learning_rate": 5.0423447993411974e-08,
      "loss": 0.19189289212226868,
      "memory(GiB)": 75.26,
      "reward": 1.8196842670440674,
      "reward_std": 0.5054992139339447,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2897924929857254,
      "rewards/MultiModalAccuracyORM/mean": 0.109375,
      "rewards/MultiModalAccuracyORM/std": 0.29547789692878723,
      "rewards/ReflectionBalance/mean": 0.804059237241745,
      "rewards/ReflectionBalance/std": 0.24308130145072937,
      "step": 1586,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 507.55859375,
      "completions/min_length": 269.5,
      "epoch": 8.576819407008086,
      "grad_norm": 4.165879099457681,
      "kl": 4.046875,
      "learning_rate": 5.004866798825885e-08,
      "loss": 0.33485615253448486,
      "memory(GiB)": 75.26,
      "reward": 1.762112557888031,
      "reward_std": 0.6368812620639801,
      "rewards/Format/mean": 0.83984375,
      "rewards/Format/std": 0.36721789836883545,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3634065240621567,
      "rewards/ReflectionBalance/mean": 0.7621126472949982,
      "rewards/ReflectionBalance/std": 0.28543201088905334,
      "step": 1587,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 472.2734375,
      "completions/min_length": 198.0,
      "epoch": 8.582210242587601,
      "grad_norm": 3.0496517937553076,
      "kl": 2.953125,
      "learning_rate": 4.9675212594517015e-08,
      "loss": 0.21985755860805511,
      "memory(GiB)": 75.26,
      "reward": 1.9118525385856628,
      "reward_std": 0.5190428495407104,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.422762006521225,
      "rewards/ReflectionBalance/mean": 0.7829462885856628,
      "rewards/ReflectionBalance/std": 0.28477728366851807,
      "step": 1588,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 493.08203125,
      "completions/min_length": 173.5,
      "epoch": 8.587601078167117,
      "grad_norm": 3.62746602523217,
      "kl": 1.98828125,
      "learning_rate": 4.9303082911601456e-08,
      "loss": 0.12751375138759613,
      "memory(GiB)": 75.26,
      "reward": 1.886598289012909,
      "reward_std": 0.406248539686203,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2566775679588318,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.33968591690063477,
      "rewards/ReflectionBalance/mean": 0.8162858486175537,
      "rewards/ReflectionBalance/std": 0.23157693445682526,
      "step": 1589,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 491.35546875,
      "completions/min_length": 263.0,
      "epoch": 8.59299191374663,
      "grad_norm": 3.6435096889810312,
      "kl": 2.6640625,
      "learning_rate": 4.893228003502442e-08,
      "loss": 0.20459794998168945,
      "memory(GiB)": 75.26,
      "reward": 1.8938220739364624,
      "reward_std": 0.4694209545850754,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.28448406606912613,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36417925357818604,
      "rewards/ReflectionBalance/mean": 0.8235096037387848,
      "rewards/ReflectionBalance/std": 0.2480502724647522,
      "step": 1590,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1037.0,
      "completions/mean_length": 504.64453125,
      "completions/min_length": 237.5,
      "epoch": 8.598382749326145,
      "grad_norm": 3.7069999141605097,
      "kl": 3.24609375,
      "learning_rate": 4.8562805056392155e-08,
      "loss": 0.289611279964447,
      "memory(GiB)": 75.26,
      "reward": 1.827001929283142,
      "reward_std": 0.5527986884117126,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.30740001052618027,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.33949992060661316,
      "rewards/ReflectionBalance/mean": 0.7957519590854645,
      "rewards/ReflectionBalance/std": 0.28709328919649124,
      "step": 1591,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 498.15234375,
      "completions/min_length": 273.0,
      "epoch": 8.60377358490566,
      "grad_norm": 3.431068205264889,
      "kl": 3.328125,
      "learning_rate": 4.8194659063402034e-08,
      "loss": 0.23821431398391724,
      "memory(GiB)": 75.26,
      "reward": 1.8554931282997131,
      "reward_std": 0.5565241128206253,
      "rewards/Format/mean": 0.859375,
      "rewards/Format/std": 0.3422359675168991,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41635647416114807,
      "rewards/ReflectionBalance/mean": 0.7734619379043579,
      "rewards/ReflectionBalance/std": 0.3172908127307892,
      "step": 1592,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 481.80859375,
      "completions/min_length": 230.0,
      "epoch": 8.609164420485175,
      "grad_norm": 5.02227332260497,
      "kl": 2.078125,
      "learning_rate": 4.782784313983868e-08,
      "loss": 0.19345903396606445,
      "memory(GiB)": 75.26,
      "reward": 1.9882991313934326,
      "reward_std": 0.5631588995456696,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.4369315803050995,
      "rewards/ReflectionBalance/mean": 0.8125178813934326,
      "rewards/ReflectionBalance/std": 0.24276767671108246,
      "step": 1593,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 490.109375,
      "completions/min_length": 260.0,
      "epoch": 8.61455525606469,
      "grad_norm": 3.5255236258191975,
      "kl": 2.6328125,
      "learning_rate": 4.746235836557111e-08,
      "loss": 0.2213125377893448,
      "memory(GiB)": 75.26,
      "reward": 1.8819059133529663,
      "reward_std": 0.5143303275108337,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38201163709163666,
      "rewards/ReflectionBalance/mean": 0.8037809133529663,
      "rewards/ReflectionBalance/std": 0.26611846685409546,
      "step": 1594,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1014.5,
      "completions/mean_length": 478.5390625,
      "completions/min_length": 274.0,
      "epoch": 8.619946091644206,
      "grad_norm": 4.936674774947547,
      "kl": 2.36328125,
      "learning_rate": 4.7098205816549944e-08,
      "loss": 0.19041626155376434,
      "memory(GiB)": 75.26,
      "reward": 1.9944140315055847,
      "reward_std": 0.4670017659664154,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2524137645959854,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4187151938676834,
      "rewards/ReflectionBalance/mean": 0.8303515911102295,
      "rewards/ReflectionBalance/std": 0.25776340067386627,
      "step": 1595,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 505.2734375,
      "completions/min_length": 275.5,
      "epoch": 8.625336927223719,
      "grad_norm": 3.195114920766768,
      "kl": 2.6484375,
      "learning_rate": 4.67353865648033e-08,
      "loss": 0.2105686515569687,
      "memory(GiB)": 75.26,
      "reward": 2.0220881700515747,
      "reward_std": 0.5379084348678589,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.31640625,
      "rewards/MultiModalAccuracyORM/std": 0.46596668660640717,
      "rewards/ReflectionBalance/mean": 0.8111506998538971,
      "rewards/ReflectionBalance/std": 0.27548253536224365,
      "step": 1596,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1037.5,
      "completions/mean_length": 511.015625,
      "completions/min_length": 279.0,
      "epoch": 8.630727762803234,
      "grad_norm": 4.831925654009394,
      "kl": 3.0625,
      "learning_rate": 4.63739016784348e-08,
      "loss": 0.2655859887599945,
      "memory(GiB)": 75.26,
      "reward": 1.854467749595642,
      "reward_std": 0.5067737400531769,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3110392540693283,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3538244515657425,
      "rewards/ReflectionBalance/mean": 0.8075927495956421,
      "rewards/ReflectionBalance/std": 0.2727551907300949,
      "step": 1597,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 492.41796875,
      "completions/min_length": 280.5,
      "epoch": 8.63611859838275,
      "grad_norm": 4.048827672977875,
      "kl": 2.7890625,
      "learning_rate": 4.6013752221619406e-08,
      "loss": 0.23521116375923157,
      "memory(GiB)": 75.26,
      "reward": 1.9529476165771484,
      "reward_std": 0.5032622069120407,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41983383893966675,
      "rewards/ReflectionBalance/mean": 0.8279476165771484,
      "rewards/ReflectionBalance/std": 0.19972140341997147,
      "step": 1598,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 486.72265625,
      "completions/min_length": 272.0,
      "epoch": 8.641509433962264,
      "grad_norm": 3.895039880686871,
      "kl": 2.11328125,
      "learning_rate": 4.5654939254600844e-08,
      "loss": 0.1709611564874649,
      "memory(GiB)": 75.26,
      "reward": 1.97760409116745,
      "reward_std": 0.418158620595932,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42764075100421906,
      "rewards/ReflectionBalance/mean": 0.8291666507720947,
      "rewards/ReflectionBalance/std": 0.22657500952482224,
      "step": 1599,
      "train_speed(iter/s)": 0.007124
    },
    {
      "epoch": 8.64690026954178,
      "grad_norm": 3.5532379514012735,
      "learning_rate": 4.529746383368843e-08,
      "loss": 0.09180903434753418,
      "memory(GiB)": 75.26,
      "step": 1600,
      "train_speed(iter/s)": 0.007125
    },
    {
      "epoch": 8.64690026954178,
      "eval_clip_ratio": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 1025.0,
      "eval_completions/mean_length": 514.80859375,
      "eval_completions/min_length": 271.0,
      "eval_kl": 3.0,
      "eval_loss": 0.24236825108528137,
      "eval_reward": 1.8347667455673218,
      "eval_reward_std": 0.4896717965602875,
      "eval_rewards/Format/mean": 0.90625,
      "eval_rewards/Format/std": 0.2896772623062134,
      "eval_rewards/MultiModalAccuracyORM/mean": 0.140625,
      "eval_rewards/MultiModalAccuracyORM/std": 0.3307749703526497,
      "eval_rewards/ReflectionBalance/mean": 0.7878917306661606,
      "eval_rewards/ReflectionBalance/std": 0.2871466726064682,
      "eval_runtime": 248.9065,
      "eval_samples_per_second": 0.237,
      "eval_steps_per_second": 0.004,
      "step": 1600
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 968.75,
      "completions/mean_length": 481.6015625,
      "completions/min_length": 275.5,
      "epoch": 8.652291105121293,
      "grad_norm": 3.5614168930749264,
      "kl": 1.7734375,
      "learning_rate": 4.4941327011253806e-08,
      "loss": 0.1833464652299881,
      "memory(GiB)": 75.26,
      "reward": 2.01108518242836,
      "reward_std": 0.4837544485926628,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2536158002912998,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.42860229313373566,
      "rewards/ReflectionBalance/mean": 0.8118664622306824,
      "rewards/ReflectionBalance/std": 0.2609814219176769,
      "step": 1601,
      "train_speed(iter/s)": 0.007115
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 491.0625,
      "completions/min_length": 265.5,
      "epoch": 8.657681940700808,
      "grad_norm": 3.2202108597920964,
      "kl": 2.25,
      "learning_rate": 4.458652983572786e-08,
      "loss": 0.15656769275665283,
      "memory(GiB)": 75.26,
      "reward": 1.8978555798530579,
      "reward_std": 0.5114020705223083,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3786114752292633,
      "rewards/ReflectionBalance/mean": 0.8158243596553802,
      "rewards/ReflectionBalance/std": 0.25323258340358734,
      "step": 1602,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 491.90625,
      "completions/min_length": 279.5,
      "epoch": 8.663072776280323,
      "grad_norm": 3.3804433639574785,
      "kl": 2.75,
      "learning_rate": 4.42330733515977e-08,
      "loss": 0.22422140836715698,
      "memory(GiB)": 75.26,
      "reward": 1.88979572057724,
      "reward_std": 0.5416781604290009,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30669571459293365,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.40016549825668335,
      "rewards/ReflectionBalance/mean": 0.78823322057724,
      "rewards/ReflectionBalance/std": 0.27847443521022797,
      "step": 1603,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 488.71484375,
      "completions/min_length": 150.0,
      "epoch": 8.668463611859838,
      "grad_norm": 3.190776373937242,
      "kl": 2.0390625,
      "learning_rate": 4.388095859940377e-08,
      "loss": 0.15982143580913544,
      "memory(GiB)": 75.26,
      "reward": 1.9946093559265137,
      "reward_std": 0.37562763690948486,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.42748305201530457,
      "rewards/ReflectionBalance/mean": 0.8305468857288361,
      "rewards/ReflectionBalance/std": 0.2195785641670227,
      "step": 1604,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 478.11328125,
      "completions/min_length": 245.0,
      "epoch": 8.673854447439354,
      "grad_norm": 3.4073117443913143,
      "kl": 2.515625,
      "learning_rate": 4.353018661573637e-08,
      "loss": 0.19876478612422943,
      "memory(GiB)": 75.26,
      "reward": 1.9212629795074463,
      "reward_std": 0.5367026925086975,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4018339514732361,
      "rewards/ReflectionBalance/mean": 0.8079817891120911,
      "rewards/ReflectionBalance/std": 0.2766219973564148,
      "step": 1605,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 496.61328125,
      "completions/min_length": 251.0,
      "epoch": 8.679245283018869,
      "grad_norm": 3.572721734611506,
      "kl": 3.1171875,
      "learning_rate": 4.318075843323288e-08,
      "loss": 0.253447949886322,
      "memory(GiB)": 75.26,
      "reward": 1.9145117402076721,
      "reward_std": 0.438040554523468,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3050640821456909,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41441819071769714,
      "rewards/ReflectionBalance/mean": 0.7973242402076721,
      "rewards/ReflectionBalance/std": 0.23492535203695297,
      "step": 1606,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 471.41796875,
      "completions/min_length": 253.5,
      "epoch": 8.684636118598382,
      "grad_norm": 4.297516356748265,
      "kl": 2.015625,
      "learning_rate": 4.283267508057459e-08,
      "loss": 0.17670638859272003,
      "memory(GiB)": 75.26,
      "reward": 1.9554020166397095,
      "reward_std": 0.46819207072257996,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2829600051045418,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.4179472029209137,
      "rewards/ReflectionBalance/mean": 0.8030582666397095,
      "rewards/ReflectionBalance/std": 0.23100672662258148,
      "step": 1607,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 536.71484375,
      "completions/min_length": 295.5,
      "epoch": 8.690026954177897,
      "grad_norm": 3.502656519832994,
      "kl": 3.8203125,
      "learning_rate": 4.2485937582484034e-08,
      "loss": 0.3191593289375305,
      "memory(GiB)": 75.26,
      "reward": 1.777039349079132,
      "reward_std": 0.6034540832042694,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3285723924636841,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.31518664956092834,
      "rewards/ReflectionBalance/mean": 0.7887581586837769,
      "rewards/ReflectionBalance/std": 0.3040229380130768,
      "step": 1608,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 493.8515625,
      "completions/min_length": 169.5,
      "epoch": 8.695417789757412,
      "grad_norm": 3.3522722016209294,
      "kl": 3.03515625,
      "learning_rate": 4.214054695972141e-08,
      "loss": 0.2569359540939331,
      "memory(GiB)": 75.26,
      "reward": 1.9389005899429321,
      "reward_std": 0.598906010389328,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4346197694540024,
      "rewards/ReflectionBalance/mean": 0.7943693101406097,
      "rewards/ReflectionBalance/std": 0.32294870913028717,
      "step": 1609,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 504.88671875,
      "completions/min_length": 310.0,
      "epoch": 8.700808625336927,
      "grad_norm": 4.184127262613145,
      "kl": 2.87109375,
      "learning_rate": 4.1796504229081875e-08,
      "loss": 0.23956841230392456,
      "memory(GiB)": 75.26,
      "reward": 1.92062246799469,
      "reward_std": 0.4588871896266937,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26276856660842896,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38864077627658844,
      "rewards/ReflectionBalance/mean": 0.8151537775993347,
      "rewards/ReflectionBalance/std": 0.28031371533870697,
      "step": 1610,
      "train_speed(iter/s)": 0.007116
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 508.33203125,
      "completions/min_length": 262.5,
      "epoch": 8.706199460916443,
      "grad_norm": 3.377852153140346,
      "kl": 2.359375,
      "learning_rate": 4.145381040339257e-08,
      "loss": 0.18846938014030457,
      "memory(GiB)": 75.26,
      "reward": 1.907433271408081,
      "reward_std": 0.4448632001876831,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39068229496479034,
      "rewards/ReflectionBalance/mean": 0.821495771408081,
      "rewards/ReflectionBalance/std": 0.24021045118570328,
      "step": 1611,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 509.921875,
      "completions/min_length": 244.0,
      "epoch": 8.711590296495956,
      "grad_norm": 51.265961050203984,
      "kl": 2.84375,
      "learning_rate": 4.1112466491509735e-08,
      "loss": 0.2428557574748993,
      "memory(GiB)": 75.26,
      "reward": 2.0329800248146057,
      "reward_std": 0.5305788218975067,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.3046875,
      "rewards/MultiModalAccuracyORM/std": 0.458208367228508,
      "rewards/ReflectionBalance/mean": 0.8142301440238953,
      "rewards/ReflectionBalance/std": 0.21176885813474655,
      "step": 1612,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 513.16015625,
      "completions/min_length": 246.0,
      "epoch": 8.716981132075471,
      "grad_norm": 3.2552408532060744,
      "kl": 2.5234375,
      "learning_rate": 4.077247349831542e-08,
      "loss": 0.19721779227256775,
      "memory(GiB)": 75.26,
      "reward": 1.9183804988861084,
      "reward_std": 0.46096834540367126,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3576768934726715,
      "rewards/ReflectionBalance/mean": 0.8090055584907532,
      "rewards/ReflectionBalance/std": 0.2582410201430321,
      "step": 1613,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 487.57421875,
      "completions/min_length": 225.5,
      "epoch": 8.722371967654986,
      "grad_norm": 35.72171763981569,
      "kl": 2.7109375,
      "learning_rate": 4.043383242471465e-08,
      "loss": 0.20670591294765472,
      "memory(GiB)": 75.26,
      "reward": 1.9931584596633911,
      "reward_std": 0.5026209354400635,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4492545574903488,
      "rewards/ReflectionBalance/mean": 0.8056583404541016,
      "rewards/ReflectionBalance/std": 0.24230989813804626,
      "step": 1614,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 538.0234375,
      "completions/min_length": 242.5,
      "epoch": 8.727762803234501,
      "grad_norm": 5.775486183798094,
      "kl": 3.53125,
      "learning_rate": 4.009654426763292e-08,
      "loss": 0.2796545624732971,
      "memory(GiB)": 75.26,
      "reward": 1.9030582308769226,
      "reward_std": 0.681813508272171,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3320184051990509,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4264890104532242,
      "rewards/ReflectionBalance/mean": 0.789777010679245,
      "rewards/ReflectionBalance/std": 0.29235750436782837,
      "step": 1615,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1031.5,
      "completions/mean_length": 469.3046875,
      "completions/min_length": 246.5,
      "epoch": 8.733153638814017,
      "grad_norm": 4.272133231343721,
      "kl": 2.3046875,
      "learning_rate": 3.976061002001219e-08,
      "loss": 0.17821738123893738,
      "memory(GiB)": 75.26,
      "reward": 1.917508840560913,
      "reward_std": 0.42556822299957275,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.356930136680603,
      "rewards/ReflectionBalance/mean": 0.8432901799678802,
      "rewards/ReflectionBalance/std": 0.21849443763494492,
      "step": 1616,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 488.89453125,
      "completions/min_length": 266.5,
      "epoch": 8.738544474393532,
      "grad_norm": 4.2616467950364365,
      "kl": 2.23828125,
      "learning_rate": 3.9426030670809405e-08,
      "loss": 0.1914825141429901,
      "memory(GiB)": 75.26,
      "reward": 2.0031039118766785,
      "reward_std": 0.42062027752399445,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24678732454776764,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4324023276567459,
      "rewards/ReflectionBalance/mean": 0.8234163522720337,
      "rewards/ReflectionBalance/std": 0.26168499886989594,
      "step": 1617,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 484.4296875,
      "completions/min_length": 192.0,
      "epoch": 8.743935309973045,
      "grad_norm": 3.1405871072811027,
      "kl": 2.7734375,
      "learning_rate": 3.909280720499231e-08,
      "loss": 0.2410615235567093,
      "memory(GiB)": 75.26,
      "reward": 1.9838932156562805,
      "reward_std": 0.5049064755439758,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4300411641597748,
      "rewards/ReflectionBalance/mean": 0.8159244954586029,
      "rewards/ReflectionBalance/std": 0.2527238577604294,
      "step": 1618,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 492.80859375,
      "completions/min_length": 290.5,
      "epoch": 8.74932614555256,
      "grad_norm": 3.4271611682860676,
      "kl": 1.8671875,
      "learning_rate": 3.876094060353724e-08,
      "loss": 0.15722163021564484,
      "memory(GiB)": 75.26,
      "reward": 2.0265170335769653,
      "reward_std": 0.3673842251300812,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24301259219646454,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4312909245491028,
      "rewards/ReflectionBalance/mean": 0.8390169441699982,
      "rewards/ReflectionBalance/std": 0.18582727760076523,
      "step": 1619,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 494.4375,
      "completions/min_length": 266.5,
      "epoch": 8.754716981132075,
      "grad_norm": 3.3637352719757434,
      "kl": 2.4921875,
      "learning_rate": 3.8430431843426144e-08,
      "loss": 0.18961003422737122,
      "memory(GiB)": 75.26,
      "reward": 2.0129451751708984,
      "reward_std": 0.5441280007362366,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.3046875,
      "rewards/MultiModalAccuracyORM/std": 0.46208351850509644,
      "rewards/ReflectionBalance/mean": 0.809820145368576,
      "rewards/ReflectionBalance/std": 0.26866473257541656,
      "step": 1620,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 966.5,
      "completions/mean_length": 489.0546875,
      "completions/min_length": 281.0,
      "epoch": 8.76010781671159,
      "grad_norm": 3.3699605757113216,
      "kl": 2.87109375,
      "learning_rate": 3.810128189764344e-08,
      "loss": 0.21730664372444153,
      "memory(GiB)": 75.26,
      "reward": 1.9397835731506348,
      "reward_std": 0.5244423151016235,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.30283892154693604,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.41856054961681366,
      "rewards/ReflectionBalance/mean": 0.80697101354599,
      "rewards/ReflectionBalance/std": 0.2609207034111023,
      "step": 1621,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 485.47265625,
      "completions/min_length": 158.0,
      "epoch": 8.765498652291106,
      "grad_norm": 3.41769775981531,
      "kl": 2.7578125,
      "learning_rate": 3.777349173517352e-08,
      "loss": 0.201604962348938,
      "memory(GiB)": 75.26,
      "reward": 1.9284815192222595,
      "reward_std": 0.5277608036994934,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.40301957726478577,
      "rewards/ReflectionBalance/mean": 0.8073876798152924,
      "rewards/ReflectionBalance/std": 0.263211652636528,
      "step": 1622,
      "train_speed(iter/s)": 0.007117
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 861.0,
      "completions/mean_length": 450.7578125,
      "completions/min_length": 245.0,
      "epoch": 8.77088948787062,
      "grad_norm": 3.3078570086302994,
      "kl": 1.73828125,
      "learning_rate": 3.744706232099743e-08,
      "loss": 0.13569796085357666,
      "memory(GiB)": 75.26,
      "reward": 1.9574235081672668,
      "reward_std": 0.434821754693985,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3996165096759796,
      "rewards/ReflectionBalance/mean": 0.8246109783649445,
      "rewards/ReflectionBalance/std": 0.17301632463932037,
      "step": 1623,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 492.22265625,
      "completions/min_length": 232.0,
      "epoch": 8.776280323450134,
      "grad_norm": 5.057243724424012,
      "kl": 2.4140625,
      "learning_rate": 3.712199461609067e-08,
      "loss": 0.17675188183784485,
      "memory(GiB)": 75.26,
      "reward": 1.871202826499939,
      "reward_std": 0.4417945295572281,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.33583175390958786,
      "rewards/ReflectionBalance/mean": 0.8008902966976166,
      "rewards/ReflectionBalance/std": 0.27037808299064636,
      "step": 1624,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 503.53125,
      "completions/min_length": 284.5,
      "epoch": 8.78167115902965,
      "grad_norm": 3.637835415306483,
      "kl": 2.5546875,
      "learning_rate": 3.679828957741976e-08,
      "loss": 0.1952856183052063,
      "memory(GiB)": 75.26,
      "reward": 1.8073567748069763,
      "reward_std": 0.5127585977315903,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.09765625,
      "rewards/MultiModalAccuracyORM/std": 0.29434798657894135,
      "rewards/ReflectionBalance/mean": 0.8073567748069763,
      "rewards/ReflectionBalance/std": 0.2642689347267151,
      "step": 1625,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 498.37890625,
      "completions/min_length": 241.0,
      "epoch": 8.787061994609164,
      "grad_norm": 3.118376983733812,
      "kl": 2.484375,
      "learning_rate": 3.6475948157939694e-08,
      "loss": 0.20609091222286224,
      "memory(GiB)": 75.26,
      "reward": 1.9353970885276794,
      "reward_std": 0.509901762008667,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.396040678024292,
      "rewards/ReflectionBalance/mean": 0.8143033385276794,
      "rewards/ReflectionBalance/std": 0.23993322998285294,
      "step": 1626,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 987.0,
      "completions/mean_length": 499.33984375,
      "completions/min_length": 193.0,
      "epoch": 8.79245283018868,
      "grad_norm": 4.813654305086892,
      "kl": 1.890625,
      "learning_rate": 3.615497130659134e-08,
      "loss": 0.14451725780963898,
      "memory(GiB)": 75.26,
      "reward": 1.9752147793769836,
      "reward_std": 0.47918251156806946,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.4164551943540573,
      "rewards/ReflectionBalance/mean": 0.8228710889816284,
      "rewards/ReflectionBalance/std": 0.2313404455780983,
      "step": 1627,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 486.01953125,
      "completions/min_length": 255.5,
      "epoch": 8.797843665768195,
      "grad_norm": 3.0002304542761804,
      "kl": 2.15234375,
      "learning_rate": 3.583535996829795e-08,
      "loss": 0.16641388833522797,
      "memory(GiB)": 75.26,
      "reward": 1.8930549621582031,
      "reward_std": 0.4919341206550598,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.378810778260231,
      "rewards/ReflectionBalance/mean": 0.8032113015651703,
      "rewards/ReflectionBalance/std": 0.2810792624950409,
      "step": 1628,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 481.83203125,
      "completions/min_length": 173.0,
      "epoch": 8.80323450134771,
      "grad_norm": 3.0686848520312715,
      "kl": 1.70703125,
      "learning_rate": 3.551711508396338e-08,
      "loss": 0.14572779834270477,
      "memory(GiB)": 75.26,
      "reward": 1.9524576663970947,
      "reward_std": 0.3791739344596863,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.2379390001296997,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3854355216026306,
      "rewards/ReflectionBalance/mean": 0.8352701663970947,
      "rewards/ReflectionBalance/std": 0.23630796372890472,
      "step": 1629,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 507.734375,
      "completions/min_length": 284.0,
      "epoch": 8.808625336927223,
      "grad_norm": 3.5530094196636965,
      "kl": 2.5625,
      "learning_rate": 3.5200237590468306e-08,
      "loss": 0.1988043636083603,
      "memory(GiB)": 75.26,
      "reward": 1.912306308746338,
      "reward_std": 0.4172956645488739,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.368965208530426,
      "rewards/ReflectionBalance/mean": 0.8263688087463379,
      "rewards/ReflectionBalance/std": 0.2222806215286255,
      "step": 1630,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 491.578125,
      "completions/min_length": 278.5,
      "epoch": 8.814016172506738,
      "grad_norm": 3.3134525779981328,
      "kl": 3.4375,
      "learning_rate": 3.488472842066825e-08,
      "loss": 0.29145950078964233,
      "memory(GiB)": 75.26,
      "reward": 1.8449886441230774,
      "reward_std": 0.5392736494541168,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3177158683538437,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3761351853609085,
      "rewards/ReflectionBalance/mean": 0.7903010845184326,
      "rewards/ReflectionBalance/std": 0.29117347300052643,
      "step": 1631,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 490.734375,
      "completions/min_length": 157.5,
      "epoch": 8.819407008086253,
      "grad_norm": 3.4835071973361424,
      "kl": 2.5703125,
      "learning_rate": 3.457058850339056e-08,
      "loss": 0.20454388856887817,
      "memory(GiB)": 75.26,
      "reward": 1.9306591749191284,
      "reward_std": 0.5004628300666809,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.3985402286052704,
      "rewards/ReflectionBalance/mean": 0.790034145116806,
      "rewards/ReflectionBalance/std": 0.2743731588125229,
      "step": 1632,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 476.78515625,
      "completions/min_length": 255.5,
      "epoch": 8.824797843665769,
      "grad_norm": 3.094316320578632,
      "kl": 2.0546875,
      "learning_rate": 3.4257818763431255e-08,
      "loss": 0.16733556985855103,
      "memory(GiB)": 75.26,
      "reward": 1.930201768875122,
      "reward_std": 0.4568460136651993,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.37408973276615143,
      "rewards/ReflectionBalance/mean": 0.8169205784797668,
      "rewards/ReflectionBalance/std": 0.21093285828828812,
      "step": 1633,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 459.6796875,
      "completions/min_length": 244.0,
      "epoch": 8.830188679245284,
      "grad_norm": 3.3867633907040617,
      "kl": 2.4453125,
      "learning_rate": 3.394642012155313e-08,
      "loss": 0.1999972015619278,
      "memory(GiB)": 75.26,
      "reward": 1.9636019468307495,
      "reward_std": 0.43942122161388397,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4202718883752823,
      "rewards/ReflectionBalance/mean": 0.8073518872261047,
      "rewards/ReflectionBalance/std": 0.2519194260239601,
      "step": 1634,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 493.07421875,
      "completions/min_length": 259.5,
      "epoch": 8.835579514824797,
      "grad_norm": 4.823950049911954,
      "kl": 3.25,
      "learning_rate": 3.363639349448244e-08,
      "loss": 0.22587928175926208,
      "memory(GiB)": 75.26,
      "reward": 1.9043115377426147,
      "reward_std": 0.4950973093509674,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3068038523197174,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42764075100421906,
      "rewards/ReflectionBalance/mean": 0.7714990079402924,
      "rewards/ReflectionBalance/std": 0.30867964774370193,
      "step": 1635,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 503.89453125,
      "completions/min_length": 236.5,
      "epoch": 8.840970350404312,
      "grad_norm": 3.3055866756280765,
      "kl": 2.578125,
      "learning_rate": 3.332773979490622e-08,
      "loss": 0.21383188664913177,
      "memory(GiB)": 75.26,
      "reward": 1.8680248856544495,
      "reward_std": 0.525248721241951,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.32267695665359497,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3946705758571625,
      "rewards/ReflectionBalance/mean": 0.7938061952590942,
      "rewards/ReflectionBalance/std": 0.2602197080850601,
      "step": 1636,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 458.6171875,
      "completions/min_length": 235.0,
      "epoch": 8.846361185983827,
      "grad_norm": 3.114760867660601,
      "kl": 1.62890625,
      "learning_rate": 3.3020459931470066e-08,
      "loss": 0.14544326066970825,
      "memory(GiB)": 75.26,
      "reward": 2.0727733373641968,
      "reward_std": 0.4920021593570709,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.30859375,
      "rewards/MultiModalAccuracyORM/std": 0.4621499925851822,
      "rewards/ReflectionBalance/mean": 0.8266797065734863,
      "rewards/ReflectionBalance/std": 0.2229848951101303,
      "step": 1637,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 479.0859375,
      "completions/min_length": 256.5,
      "epoch": 8.851752021563343,
      "grad_norm": 2.9566162121861987,
      "kl": 2.09765625,
      "learning_rate": 3.271455480877483e-08,
      "loss": 0.16096211969852448,
      "memory(GiB)": 75.26,
      "reward": 1.9507437944412231,
      "reward_std": 0.4778040945529938,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.258706197142601,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.42152732610702515,
      "rewards/ReflectionBalance/mean": 0.7984000742435455,
      "rewards/ReflectionBalance/std": 0.27902182936668396,
      "step": 1638,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 514.7421875,
      "completions/min_length": 222.0,
      "epoch": 8.857142857142858,
      "grad_norm": 8.385682704391535,
      "kl": 2.3515625,
      "learning_rate": 3.241002532737436e-08,
      "loss": 0.18753598630428314,
      "memory(GiB)": 75.26,
      "reward": 1.9252417087554932,
      "reward_std": 0.4915670305490494,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3773547112941742,
      "rewards/ReflectionBalance/mean": 0.8041479885578156,
      "rewards/ReflectionBalance/std": 0.27817532420158386,
      "step": 1639,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1035.0,
      "completions/mean_length": 535.66015625,
      "completions/min_length": 276.0,
      "epoch": 8.862533692722373,
      "grad_norm": 4.08520142174244,
      "kl": 2.8984375,
      "learning_rate": 3.210687238377286e-08,
      "loss": 0.2137603759765625,
      "memory(GiB)": 75.26,
      "reward": 1.8694050908088684,
      "reward_std": 0.4799867272377014,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.34734736382961273,
      "rewards/ReflectionBalance/mean": 0.8029988706111908,
      "rewards/ReflectionBalance/std": 0.24447108805179596,
      "step": 1640,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 922.0,
      "completions/mean_length": 474.69140625,
      "completions/min_length": 267.0,
      "epoch": 8.867924528301886,
      "grad_norm": 4.523467202199336,
      "kl": 1.1640625,
      "learning_rate": 3.18050968704221e-08,
      "loss": 0.11837199330329895,
      "memory(GiB)": 75.26,
      "reward": 2.0839778780937195,
      "reward_std": 0.3463009297847748,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.20146947354078293,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.40798476338386536,
      "rewards/ReflectionBalance/mean": 0.8652278184890747,
      "rewards/ReflectionBalance/std": 0.1505906879901886,
      "step": 1641,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 513.05078125,
      "completions/min_length": 271.0,
      "epoch": 8.873315363881401,
      "grad_norm": 3.3288731933543603,
      "kl": 2.81640625,
      "learning_rate": 3.1504699675718724e-08,
      "loss": 0.2383785992860794,
      "memory(GiB)": 75.26,
      "reward": 1.8011263012886047,
      "reward_std": 0.4817189872264862,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.28863146156072617,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3035084083676338,
      "rewards/ReflectionBalance/mean": 0.7855013012886047,
      "rewards/ReflectionBalance/std": 0.3143925666809082,
      "step": 1642,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 497.23828125,
      "completions/min_length": 251.0,
      "epoch": 8.878706199460916,
      "grad_norm": 4.181422094596375,
      "kl": 1.8515625,
      "learning_rate": 3.1205681684001785e-08,
      "loss": 0.15207815170288086,
      "memory(GiB)": 75.26,
      "reward": 1.9454866647720337,
      "reward_std": 0.38206905126571655,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3761351853609085,
      "rewards/ReflectionBalance/mean": 0.8361116647720337,
      "rewards/ReflectionBalance/std": 0.1884942650794983,
      "step": 1643,
      "train_speed(iter/s)": 0.007118
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 513.4921875,
      "completions/min_length": 263.0,
      "epoch": 8.884097035040432,
      "grad_norm": 4.0984897132597515,
      "kl": 2.23828125,
      "learning_rate": 3.090804377555029e-08,
      "loss": 0.17378626763820648,
      "memory(GiB)": 75.26,
      "reward": 1.8674390316009521,
      "reward_std": 0.47710245847702026,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3786114752292633,
      "rewards/ReflectionBalance/mean": 0.7932202219963074,
      "rewards/ReflectionBalance/std": 0.2613835409283638,
      "step": 1644,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 491.33203125,
      "completions/min_length": 254.5,
      "epoch": 8.889487870619947,
      "grad_norm": 3.2428414543836444,
      "kl": 1.9609375,
      "learning_rate": 3.0611786826580166e-08,
      "loss": 0.160008043050766,
      "memory(GiB)": 75.26,
      "reward": 1.9293619394302368,
      "reward_std": 0.48831233382225037,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39068229496479034,
      "rewards/ReflectionBalance/mean": 0.8121744692325592,
      "rewards/ReflectionBalance/std": 0.2582196071743965,
      "step": 1645,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 489.96875,
      "completions/min_length": 267.0,
      "epoch": 8.89487870619946,
      "grad_norm": 3.1735481119392617,
      "kl": 2.26171875,
      "learning_rate": 3.031691170924194e-08,
      "loss": 0.21550416946411133,
      "memory(GiB)": 75.26,
      "reward": 2.1531949043273926,
      "reward_std": 0.534711167216301,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24678732454776764,
      "rewards/MultiModalAccuracyORM/mean": 0.390625,
      "rewards/MultiModalAccuracyORM/std": 0.48974156379699707,
      "rewards/ReflectionBalance/mean": 0.8289762735366821,
      "rewards/ReflectionBalance/std": 0.20817043632268906,
      "step": 1646,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 460.42578125,
      "completions/min_length": 242.5,
      "epoch": 8.900269541778975,
      "grad_norm": 3.370990253636784,
      "kl": 2.29296875,
      "learning_rate": 3.0023419291618556e-08,
      "loss": 0.17074242234230042,
      "memory(GiB)": 75.26,
      "reward": 1.9100211262702942,
      "reward_std": 0.5086431205272675,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.39185720682144165,
      "rewards/ReflectionBalance/mean": 0.8123648762702942,
      "rewards/ReflectionBalance/std": 0.2442808300256729,
      "step": 1647,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.0,
      "completions/mean_length": 495.9921875,
      "completions/min_length": 244.5,
      "epoch": 8.90566037735849,
      "grad_norm": 3.1983713436547236,
      "kl": 2.4765625,
      "learning_rate": 2.973131043772181e-08,
      "loss": 0.19550204277038574,
      "memory(GiB)": 75.26,
      "reward": 1.9098836183547974,
      "reward_std": 0.4759047329425812,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39416566491127014,
      "rewards/ReflectionBalance/mean": 0.8083211183547974,
      "rewards/ReflectionBalance/std": 0.24761837720870972,
      "step": 1648,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 521.30859375,
      "completions/min_length": 282.0,
      "epoch": 8.911051212938006,
      "grad_norm": 3.0648636054333513,
      "kl": 2.78125,
      "learning_rate": 2.9440586007490996e-08,
      "loss": 0.23551489412784576,
      "memory(GiB)": 75.26,
      "reward": 1.9686980247497559,
      "reward_std": 0.5360698103904724,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.4219265580177307,
      "rewards/ReflectionBalance/mean": 0.8202604055404663,
      "rewards/ReflectionBalance/std": 0.24282445758581161,
      "step": 1649,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 956.0,
      "completions/mean_length": 493.07421875,
      "completions/min_length": 267.5,
      "epoch": 8.91644204851752,
      "grad_norm": 3.9030242144974556,
      "kl": 1.6484375,
      "learning_rate": 2.9151246856789436e-08,
      "loss": 0.11874755471944809,
      "memory(GiB)": 75.26,
      "reward": 1.9420602321624756,
      "reward_std": 0.3382181227207184,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.36039385199546814,
      "rewards/ReflectionBalance/mean": 0.844404011964798,
      "rewards/ReflectionBalance/std": 0.1685032993555069,
      "step": 1650,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 493.43359375,
      "completions/min_length": 274.5,
      "epoch": 8.921832884097036,
      "grad_norm": 2.7995265749221563,
      "kl": 2.1796875,
      "learning_rate": 2.886329383740238e-08,
      "loss": 0.16837003827095032,
      "memory(GiB)": 75.26,
      "reward": 1.9803451895713806,
      "reward_std": 0.43869270384311676,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.4098135679960251,
      "rewards/ReflectionBalance/mean": 0.8240950405597687,
      "rewards/ReflectionBalance/std": 0.28224676102399826,
      "step": 1651,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 496.53125,
      "completions/min_length": 275.0,
      "epoch": 8.92722371967655,
      "grad_norm": 3.28991849751851,
      "kl": 2.6796875,
      "learning_rate": 2.8576727797034737e-08,
      "loss": 0.2180728018283844,
      "memory(GiB)": 75.26,
      "reward": 1.8769091963768005,
      "reward_std": 0.5468700081110001,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3897709399461746,
      "rewards/ReflectionBalance/mean": 0.7909716963768005,
      "rewards/ReflectionBalance/std": 0.29087333381175995,
      "step": 1652,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 497.91015625,
      "completions/min_length": 267.5,
      "epoch": 8.932614555256064,
      "grad_norm": 3.5677140773018867,
      "kl": 2.08984375,
      "learning_rate": 2.829154957930796e-08,
      "loss": 0.15955522656440735,
      "memory(GiB)": 75.26,
      "reward": 1.9480209350585938,
      "reward_std": 0.44063428044319153,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.4002292603254318,
      "rewards/ReflectionBalance/mean": 0.8191145956516266,
      "rewards/ReflectionBalance/std": 0.2276834473013878,
      "step": 1653,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 922.5,
      "completions/mean_length": 480.72265625,
      "completions/min_length": 236.0,
      "epoch": 8.93800539083558,
      "grad_norm": 87.33699164053816,
      "kl": 2.041015625,
      "learning_rate": 2.800776002375799e-08,
      "loss": 0.1621113419532776,
      "memory(GiB)": 75.26,
      "reward": 1.9225423336029053,
      "reward_std": 0.4176352769136429,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.36175550520420074,
      "rewards/ReflectionBalance/mean": 0.8170736134052277,
      "rewards/ReflectionBalance/std": 0.2230573520064354,
      "step": 1654,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 476.09375,
      "completions/min_length": 269.0,
      "epoch": 8.943396226415095,
      "grad_norm": 3.1492956048260634,
      "kl": 2.13671875,
      "learning_rate": 2.772535996583264e-08,
      "loss": 0.18132595717906952,
      "memory(GiB)": 75.26,
      "reward": 1.9018115401268005,
      "reward_std": 0.44129478931427,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3761351853609085,
      "rewards/ReflectionBalance/mean": 0.8002490103244781,
      "rewards/ReflectionBalance/std": 0.26391320675611496,
      "step": 1655,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 493.8984375,
      "completions/min_length": 169.0,
      "epoch": 8.94878706199461,
      "grad_norm": 3.665223583981984,
      "kl": 2.8203125,
      "learning_rate": 2.7444350236889347e-08,
      "loss": 0.23055602610111237,
      "memory(GiB)": 75.26,
      "reward": 1.8766422271728516,
      "reward_std": 0.46385449171066284,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3734995424747467,
      "rewards/ReflectionBalance/mean": 0.8102359771728516,
      "rewards/ReflectionBalance/std": 0.2751603275537491,
      "step": 1656,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.0,
      "completions/mean_length": 511.66015625,
      "completions/min_length": 289.5,
      "epoch": 8.954177897574123,
      "grad_norm": 2.763130274504443,
      "kl": 2.390625,
      "learning_rate": 2.7164731664192474e-08,
      "loss": 0.202149897813797,
      "memory(GiB)": 75.26,
      "reward": 1.869387149810791,
      "reward_std": 0.45979224145412445,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.15234375,
      "rewards/MultiModalAccuracyORM/std": 0.3511015325784683,
      "rewards/ReflectionBalance/mean": 0.8029809296131134,
      "rewards/ReflectionBalance/std": 0.2935011088848114,
      "step": 1657,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 502.17578125,
      "completions/min_length": 296.0,
      "epoch": 8.959568733153638,
      "grad_norm": 4.54347499417797,
      "kl": 2.30859375,
      "learning_rate": 2.688650507091089e-08,
      "loss": 0.1888635903596878,
      "memory(GiB)": 75.26,
      "reward": 2.007566809654236,
      "reward_std": 0.5137800872325897,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.27558503299951553,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.43559013307094574,
      "rewards/ReflectionBalance/mean": 0.8239730000495911,
      "rewards/ReflectionBalance/std": 0.21521373838186264,
      "step": 1658,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 480.3125,
      "completions/min_length": 227.5,
      "epoch": 8.964959568733153,
      "grad_norm": 3.463654201436334,
      "kl": 2.328125,
      "learning_rate": 2.660967127611563e-08,
      "loss": 0.18228571116924286,
      "memory(GiB)": 75.26,
      "reward": 1.9176570177078247,
      "reward_std": 0.5266140103340149,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.4037879854440689,
      "rewards/ReflectionBalance/mean": 0.8121883273124695,
      "rewards/ReflectionBalance/std": 0.2598586902022362,
      "step": 1659,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 529.41796875,
      "completions/min_length": 304.5,
      "epoch": 8.970350404312669,
      "grad_norm": 3.1384833426024525,
      "kl": 2.65625,
      "learning_rate": 2.6334231094777505e-08,
      "loss": 0.2206539511680603,
      "memory(GiB)": 75.26,
      "reward": 1.858801245689392,
      "reward_std": 0.49941496551036835,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.34826576709747314,
      "rewards/ReflectionBalance/mean": 0.8158324956893921,
      "rewards/ReflectionBalance/std": 0.25956177711486816,
      "step": 1660,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 998.5,
      "completions/mean_length": 479.9375,
      "completions/min_length": 251.5,
      "epoch": 8.975741239892184,
      "grad_norm": 3.2103685685890144,
      "kl": 2.0703125,
      "learning_rate": 2.6060185337764773e-08,
      "loss": 0.17212337255477905,
      "memory(GiB)": 75.26,
      "reward": 1.9839110970497131,
      "reward_std": 0.4219912141561508,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.253920242190361,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41404466331005096,
      "rewards/ReflectionBalance/mean": 0.8393798768520355,
      "rewards/ReflectionBalance/std": 0.204652838408947,
      "step": 1661,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 509.9921875,
      "completions/min_length": 244.0,
      "epoch": 8.981132075471699,
      "grad_norm": 6.313936145121259,
      "kl": 2.2109375,
      "learning_rate": 2.5787534811840485e-08,
      "loss": 0.16863319277763367,
      "memory(GiB)": 75.26,
      "reward": 1.9913965463638306,
      "reward_std": 0.4461279809474945,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.22826264798641205,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41328008472919464,
      "rewards/ReflectionBalance/mean": 0.8273339867591858,
      "rewards/ReflectionBalance/std": 0.2460727021098137,
      "step": 1662,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 510.23828125,
      "completions/min_length": 286.5,
      "epoch": 8.986522911051212,
      "grad_norm": 3.1435312883165225,
      "kl": 1.8125,
      "learning_rate": 2.55162803196603e-08,
      "loss": 0.13554629683494568,
      "memory(GiB)": 75.26,
      "reward": 1.9817813634872437,
      "reward_std": 0.4803917706012726,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41328008472919464,
      "rewards/ReflectionBalance/mean": 0.837250143289566,
      "rewards/ReflectionBalance/std": 0.20653767883777618,
      "step": 1663,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 491.0859375,
      "completions/min_length": 239.0,
      "epoch": 8.991913746630727,
      "grad_norm": 3.268622479922711,
      "kl": 3.125,
      "learning_rate": 2.5246422659770227e-08,
      "loss": 0.2411789447069168,
      "memory(GiB)": 75.26,
      "reward": 1.9500390887260437,
      "reward_std": 0.49731212854385376,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.4401901513338089,
      "rewards/ReflectionBalance/mean": 0.7898827791213989,
      "rewards/ReflectionBalance/std": 0.297363743185997,
      "step": 1664,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 511.48828125,
      "completions/min_length": 260.5,
      "epoch": 8.997304582210242,
      "grad_norm": 3.0149456521348577,
      "kl": 2.2421875,
      "learning_rate": 2.49779626266039e-08,
      "loss": 0.18747985363006592,
      "memory(GiB)": 75.26,
      "reward": 1.9051263332366943,
      "reward_std": 0.39505375921726227,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.37818245589733124,
      "rewards/ReflectionBalance/mean": 0.8270012736320496,
      "rewards/ReflectionBalance/std": 0.22356101870536804,
      "step": 1665,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 520.109375,
      "completions/min_length": 277.0,
      "epoch": 9.005390835579515,
      "grad_norm": 4.456259451515414,
      "kl": 3.140625,
      "learning_rate": 2.471090101048068e-08,
      "loss": 0.25718775391578674,
      "memory(GiB)": 75.26,
      "reward": 1.7895393371582031,
      "reward_std": 0.5528254508972168,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.33513282239437103,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.33636194467544556,
      "rewards/ReflectionBalance/mean": 0.7895393967628479,
      "rewards/ReflectionBalance/std": 0.3132112920284271,
      "step": 1666,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 499.44140625,
      "completions/min_length": 260.0,
      "epoch": 9.01078167115903,
      "grad_norm": 3.431763603533021,
      "kl": 1.46484375,
      "learning_rate": 2.4445238597603222e-08,
      "loss": 0.10571329295635223,
      "memory(GiB)": 75.26,
      "reward": 1.9545295238494873,
      "reward_std": 0.4165172874927521,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.2156769335269928,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.37408973276615143,
      "rewards/ReflectionBalance/mean": 0.8295296430587769,
      "rewards/ReflectionBalance/std": 0.21766380965709686,
      "step": 1667,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 506.8203125,
      "completions/min_length": 255.5,
      "epoch": 9.016172506738544,
      "grad_norm": 3.259591108760096,
      "kl": 2.5390625,
      "learning_rate": 2.4180976170054658e-08,
      "loss": 0.21169717609882355,
      "memory(GiB)": 75.26,
      "reward": 1.9409717321395874,
      "reward_std": 0.5312439501285553,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4256902486085892,
      "rewards/ReflectionBalance/mean": 0.796440452337265,
      "rewards/ReflectionBalance/std": 0.2744702696800232,
      "step": 1668,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 476.41015625,
      "completions/min_length": 274.0,
      "epoch": 9.021563342318059,
      "grad_norm": 4.21398302322771,
      "kl": 1.75390625,
      "learning_rate": 2.391811450579717e-08,
      "loss": 0.1636945605278015,
      "memory(GiB)": 75.26,
      "reward": 1.869775414466858,
      "reward_std": 0.3983634412288666,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24356544017791748,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.31763872504234314,
      "rewards/ReflectionBalance/mean": 0.8229003548622131,
      "rewards/ReflectionBalance/std": 0.23538093268871307,
      "step": 1669,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 498.515625,
      "completions/min_length": 261.0,
      "epoch": 9.026954177897574,
      "grad_norm": 3.2028677115990867,
      "kl": 3.2890625,
      "learning_rate": 2.3656654378668928e-08,
      "loss": 0.27323973178863525,
      "memory(GiB)": 75.26,
      "reward": 1.9055549502372742,
      "reward_std": 0.4736045151948929,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.41230008006095886,
      "rewards/ReflectionBalance/mean": 0.788367509841919,
      "rewards/ReflectionBalance/std": 0.27890798449516296,
      "step": 1670,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 488.31640625,
      "completions/min_length": 241.0,
      "epoch": 9.032345013477089,
      "grad_norm": 3.5024917433502387,
      "kl": 2.984375,
      "learning_rate": 2.339659655838222e-08,
      "loss": 0.23588252067565918,
      "memory(GiB)": 75.26,
      "reward": 1.8538640141487122,
      "reward_std": 0.4344460666179657,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2918590381741524,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.3562464565038681,
      "rewards/ReflectionBalance/mean": 0.8030827045440674,
      "rewards/ReflectionBalance/std": 0.25054971128702164,
      "step": 1671,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 506.859375,
      "completions/min_length": 265.5,
      "epoch": 9.037735849056604,
      "grad_norm": 3.443428093403757,
      "kl": 2.8828125,
      "learning_rate": 2.313794181052109e-08,
      "loss": 0.22788861393928528,
      "memory(GiB)": 75.26,
      "reward": 1.9260741472244263,
      "reward_std": 0.4737909883260727,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.413447842001915,
      "rewards/ReflectionBalance/mean": 0.797167956829071,
      "rewards/ReflectionBalance/std": 0.2870986983180046,
      "step": 1672,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 478.5390625,
      "completions/min_length": 189.0,
      "epoch": 9.04312668463612,
      "grad_norm": 3.0699116820350683,
      "kl": 1.484375,
      "learning_rate": 2.2880690896539156e-08,
      "loss": 0.1276373565196991,
      "memory(GiB)": 75.26,
      "reward": 2.009127616882324,
      "reward_std": 0.4625282287597656,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2566775679588318,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.42214564979076385,
      "rewards/ReflectionBalance/mean": 0.8489713370800018,
      "rewards/ReflectionBalance/std": 0.20089460164308548,
      "step": 1673,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 480.3046875,
      "completions/min_length": 130.5,
      "epoch": 9.048517520215633,
      "grad_norm": 11.830551319303757,
      "kl": 3.046875,
      "learning_rate": 2.2624844573757072e-08,
      "loss": 0.24332892894744873,
      "memory(GiB)": 75.26,
      "reward": 1.9100553393363953,
      "reward_std": 0.46860575675964355,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.40284374356269836,
      "rewards/ReflectionBalance/mean": 0.7967740893363953,
      "rewards/ReflectionBalance/std": 0.25648467242717743,
      "step": 1674,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 494.77734375,
      "completions/min_length": 239.5,
      "epoch": 9.053908355795148,
      "grad_norm": 3.5025279487929666,
      "kl": 2.5,
      "learning_rate": 2.237040359536069e-08,
      "loss": 0.19700443744659424,
      "memory(GiB)": 75.26,
      "reward": 1.9974235892295837,
      "reward_std": 0.49533940851688385,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.44526244699954987,
      "rewards/ReflectionBalance/mean": 0.8060172498226166,
      "rewards/ReflectionBalance/std": 0.252975657582283,
      "step": 1675,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 501.1484375,
      "completions/min_length": 258.0,
      "epoch": 9.059299191374663,
      "grad_norm": 3.006739692145133,
      "kl": 2.1171875,
      "learning_rate": 2.2117368710398655e-08,
      "loss": 0.16986089944839478,
      "memory(GiB)": 75.26,
      "reward": 1.9224552512168884,
      "reward_std": 0.5157036036252975,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3996165096759796,
      "rewards/ReflectionBalance/mean": 0.7974552810192108,
      "rewards/ReflectionBalance/std": 0.28674203902482986,
      "step": 1676,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 474.9609375,
      "completions/min_length": 258.5,
      "epoch": 9.064690026954178,
      "grad_norm": 3.1749918135738118,
      "kl": 1.95703125,
      "learning_rate": 2.1865740663780164e-08,
      "loss": 0.14863455295562744,
      "memory(GiB)": 75.26,
      "reward": 1.94928377866745,
      "reward_std": 0.4450217932462692,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38864077627658844,
      "rewards/ReflectionBalance/mean": 0.8360026180744171,
      "rewards/ReflectionBalance/std": 0.20855122059583664,
      "step": 1677,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 501.94921875,
      "completions/min_length": 260.5,
      "epoch": 9.070080862533693,
      "grad_norm": 3.146805628299836,
      "kl": 3.0859375,
      "learning_rate": 2.161552019627283e-08,
      "loss": 0.26323747634887695,
      "memory(GiB)": 75.26,
      "reward": 1.7821402549743652,
      "reward_std": 0.5598337054252625,
      "rewards/Format/mean": 0.86328125,
      "rewards/Format/std": 0.33660952746868134,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.33029159903526306,
      "rewards/ReflectionBalance/mean": 0.78214031457901,
      "rewards/ReflectionBalance/std": 0.2735815569758415,
      "step": 1678,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 468.6484375,
      "completions/min_length": 188.5,
      "epoch": 9.075471698113208,
      "grad_norm": 2.900015280128123,
      "kl": 1.6796875,
      "learning_rate": 2.1366708044500638e-08,
      "loss": 0.11905308067798615,
      "memory(GiB)": 75.26,
      "reward": 1.972395896911621,
      "reward_std": 0.44040389358997345,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.3740888237953186,
      "rewards/ReflectionBalance/mean": 0.8278645873069763,
      "rewards/ReflectionBalance/std": 0.2306700274348259,
      "step": 1679,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 487.921875,
      "completions/min_length": 240.5,
      "epoch": 9.080862533692722,
      "grad_norm": 4.856511067076279,
      "kl": 2.4765625,
      "learning_rate": 2.111930494094133e-08,
      "loss": 0.18161864578723907,
      "memory(GiB)": 75.26,
      "reward": 1.9233707785606384,
      "reward_std": 0.4681987315416336,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4053537845611572,
      "rewards/ReflectionBalance/mean": 0.8061832785606384,
      "rewards/ReflectionBalance/std": 0.24108624458312988,
      "step": 1680,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 508.22265625,
      "completions/min_length": 286.0,
      "epoch": 9.086253369272237,
      "grad_norm": 4.420919709828258,
      "kl": 2.5625,
      "learning_rate": 2.0873311613924806e-08,
      "loss": 0.17326754331588745,
      "memory(GiB)": 75.26,
      "reward": 1.8788753151893616,
      "reward_std": 0.3876798748970032,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3716365247964859,
      "rewards/ReflectionBalance/mean": 0.8046566247940063,
      "rewards/ReflectionBalance/std": 0.28272831439971924,
      "step": 1681,
      "train_speed(iter/s)": 0.007121
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 964.0,
      "completions/mean_length": 472.03515625,
      "completions/min_length": 256.0,
      "epoch": 9.091644204851752,
      "grad_norm": 3.210212366013507,
      "kl": 1.50390625,
      "learning_rate": 2.0628728787630723e-08,
      "loss": 0.12529432773590088,
      "memory(GiB)": 75.26,
      "reward": 1.9231180548667908,
      "reward_std": 0.39495500922203064,
      "rewards/Format/mean": 0.94921875,
      "rewards/Format/std": 0.2156769335269928,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3358502835035324,
      "rewards/ReflectionBalance/mean": 0.829367995262146,
      "rewards/ReflectionBalance/std": 0.2280733361840248,
      "step": 1682,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 494.75,
      "completions/min_length": 283.0,
      "epoch": 9.097035040431267,
      "grad_norm": 3.6281747037124092,
      "kl": 2.8046875,
      "learning_rate": 2.0385557182086066e-08,
      "loss": 0.242225781083107,
      "memory(GiB)": 75.26,
      "reward": 1.99332195520401,
      "reward_std": 0.5239094197750092,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.43741196393966675,
      "rewards/ReflectionBalance/mean": 0.8136344254016876,
      "rewards/ReflectionBalance/std": 0.2316809445619583,
      "step": 1683,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 492.7109375,
      "completions/min_length": 83.0,
      "epoch": 9.102425876010782,
      "grad_norm": 3.2993597464136846,
      "kl": 2.78125,
      "learning_rate": 2.014379751316392e-08,
      "loss": 0.19858458638191223,
      "memory(GiB)": 75.26,
      "reward": 1.899230182170868,
      "reward_std": 0.5349125415086746,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4067413955926895,
      "rewards/ReflectionBalance/mean": 0.7937614321708679,
      "rewards/ReflectionBalance/std": 0.26853759586811066,
      "step": 1684,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.0,
      "completions/mean_length": 513.43359375,
      "completions/min_length": 239.0,
      "epoch": 9.107816711590296,
      "grad_norm": 3.6041273443229747,
      "kl": 3.484375,
      "learning_rate": 1.990345049258002e-08,
      "loss": 0.26570871472358704,
      "memory(GiB)": 75.26,
      "reward": 1.8701944947242737,
      "reward_std": 0.599218338727951,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.3338889628648758,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.38770951330661774,
      "rewards/ReflectionBalance/mean": 0.7920694947242737,
      "rewards/ReflectionBalance/std": 0.27445466816425323,
      "step": 1685,
      "train_speed(iter/s)": 0.007122
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 884.5,
      "completions/mean_length": 493.55859375,
      "completions/min_length": 284.0,
      "epoch": 9.11320754716981,
      "grad_norm": 3.63964858656152,
      "kl": 2.828125,
      "learning_rate": 1.9664516827892053e-08,
      "loss": 0.22499990463256836,
      "memory(GiB)": 75.26,
      "reward": 1.889724612236023,
      "reward_std": 0.6314757168292999,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3191515803337097,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.3908827602863312,
      "rewards/ReflectionBalance/mean": 0.792068362236023,
      "rewards/ReflectionBalance/std": 0.29178905487060547,
      "step": 1686,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 485.12109375,
      "completions/min_length": 268.5,
      "epoch": 9.118598382749326,
      "grad_norm": 3.0897119102056214,
      "kl": 1.5859375,
      "learning_rate": 1.9426997222496522e-08,
      "loss": 0.1320197582244873,
      "memory(GiB)": 75.26,
      "reward": 1.954410433769226,
      "reward_std": 0.5002721846103668,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2566775679588318,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.3890078365802765,
      "rewards/ReflectionBalance/mean": 0.8215979933738708,
      "rewards/ReflectionBalance/std": 0.24905723333358765,
      "step": 1687,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 475.37109375,
      "completions/min_length": 174.0,
      "epoch": 9.123989218328841,
      "grad_norm": 3.4868747073787656,
      "kl": 2.47265625,
      "learning_rate": 1.9190892375627267e-08,
      "loss": 0.18075209856033325,
      "memory(GiB)": 75.26,
      "reward": 1.8961995244026184,
      "reward_std": 0.5006348639726639,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3934052139520645,
      "rewards/ReflectionBalance/mean": 0.7907307744026184,
      "rewards/ReflectionBalance/std": 0.2628839835524559,
      "step": 1688,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 492.1875,
      "completions/min_length": 261.0,
      "epoch": 9.129380053908356,
      "grad_norm": 3.236169155394427,
      "kl": 2.71875,
      "learning_rate": 1.8956202982353286e-08,
      "loss": 0.24562445282936096,
      "memory(GiB)": 75.26,
      "reward": 1.8995555639266968,
      "reward_std": 0.46369463205337524,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.1953125,
      "rewards/MultiModalAccuracyORM/std": 0.3851911425590515,
      "rewards/ReflectionBalance/mean": 0.8018994629383087,
      "rewards/ReflectionBalance/std": 0.257695309817791,
      "step": 1689,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 494.578125,
      "completions/min_length": 249.0,
      "epoch": 9.134770889487871,
      "grad_norm": 3.273063828896465,
      "kl": 3.4921875,
      "learning_rate": 1.8722929733576364e-08,
      "loss": 0.2724400758743286,
      "memory(GiB)": 75.26,
      "reward": 1.9201423525810242,
      "reward_std": 0.5882562398910522,
      "rewards/Format/mean": 0.8671875,
      "rewards/Format/std": 0.3389271944761276,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.43903933465480804,
      "rewards/ReflectionBalance/mean": 0.783423662185669,
      "rewards/ReflectionBalance/std": 0.28577500581741333,
      "step": 1690,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 521.3828125,
      "completions/min_length": 280.5,
      "epoch": 9.140161725067385,
      "grad_norm": 3.233340427006455,
      "kl": 2.8046875,
      "learning_rate": 1.849107331602956e-08,
      "loss": 0.2464522421360016,
      "memory(GiB)": 75.26,
      "reward": 1.8332242369651794,
      "reward_std": 0.49523693323135376,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.1171875,
      "rewards/MultiModalAccuracyORM/std": 0.3208130896091461,
      "rewards/ReflectionBalance/mean": 0.8136930167675018,
      "rewards/ReflectionBalance/std": 0.24096377193927765,
      "step": 1691,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 532.74609375,
      "completions/min_length": 242.5,
      "epoch": 9.1455525606469,
      "grad_norm": 3.948041311712029,
      "kl": 4.0390625,
      "learning_rate": 1.826063441227471e-08,
      "loss": 0.3053820729255676,
      "memory(GiB)": 75.26,
      "reward": 1.88884437084198,
      "reward_std": 0.5476096272468567,
      "rewards/Format/mean": 0.87109375,
      "rewards/Format/std": 0.330063596367836,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.424868568778038,
      "rewards/ReflectionBalance/mean": 0.7833756506443024,
      "rewards/ReflectionBalance/std": 0.28997887670993805,
      "step": 1692,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 451.79296875,
      "completions/min_length": 256.5,
      "epoch": 9.150943396226415,
      "grad_norm": 3.67728248311015,
      "kl": 2.8203125,
      "learning_rate": 1.8031613700700887e-08,
      "loss": 0.2548171281814575,
      "memory(GiB)": 75.26,
      "reward": 1.9167545437812805,
      "reward_std": 0.4516783356666565,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29434798657894135,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3946705758571625,
      "rewards/ReflectionBalance/mean": 0.8230045437812805,
      "rewards/ReflectionBalance/std": 0.21133694797754288,
      "step": 1693,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 478.015625,
      "completions/min_length": 273.5,
      "epoch": 9.15633423180593,
      "grad_norm": 3.726594618734515,
      "kl": 2.28515625,
      "learning_rate": 1.780401185552194e-08,
      "loss": 0.2103869765996933,
      "memory(GiB)": 75.26,
      "reward": 1.928433358669281,
      "reward_std": 0.4817596971988678,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27079907804727554,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.4002292603254318,
      "rewards/ReflectionBalance/mean": 0.8112459480762482,
      "rewards/ReflectionBalance/std": 0.23787030577659607,
      "step": 1694,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 495.83984375,
      "completions/min_length": 253.5,
      "epoch": 9.161725067385445,
      "grad_norm": 4.153940852297652,
      "kl": 3.1640625,
      "learning_rate": 1.757782954677467e-08,
      "loss": 0.23986831307411194,
      "memory(GiB)": 75.26,
      "reward": 1.8459798097610474,
      "reward_std": 0.447131872177124,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3520684093236923,
      "rewards/ReflectionBalance/mean": 0.7717610597610474,
      "rewards/ReflectionBalance/std": 0.326244056224823,
      "step": 1695,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 479.8203125,
      "completions/min_length": 245.5,
      "epoch": 9.167115902964959,
      "grad_norm": 3.979614067963485,
      "kl": 2.3515625,
      "learning_rate": 1.735306744031728e-08,
      "loss": 0.19011563062667847,
      "memory(GiB)": 75.26,
      "reward": 1.9924992322921753,
      "reward_std": 0.5039052367210388,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.43433645367622375,
      "rewards/ReflectionBalance/mean": 0.8284367024898529,
      "rewards/ReflectionBalance/std": 0.23828984797000885,
      "step": 1696,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 476.3203125,
      "completions/min_length": 256.5,
      "epoch": 9.172506738544474,
      "grad_norm": 3.6824628590666615,
      "kl": 2.2578125,
      "learning_rate": 1.7129726197826488e-08,
      "loss": 0.20937223732471466,
      "memory(GiB)": 75.26,
      "reward": 2.0390332341194153,
      "reward_std": 0.4618084728717804,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.4399145394563675,
      "rewards/ReflectionBalance/mean": 0.8241894543170929,
      "rewards/ReflectionBalance/std": 0.24282550811767578,
      "step": 1697,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 524.49609375,
      "completions/min_length": 278.0,
      "epoch": 9.177897574123989,
      "grad_norm": 3.1843212120318247,
      "kl": 2.2890625,
      "learning_rate": 1.6907806476796516e-08,
      "loss": 0.19976806640625,
      "memory(GiB)": 75.26,
      "reward": 1.897972047328949,
      "reward_std": 0.49680040776729584,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2801405265927315,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.37408973276615143,
      "rewards/ReflectionBalance/mean": 0.8120345175266266,
      "rewards/ReflectionBalance/std": 0.2691306471824646,
      "step": 1698,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 473.38671875,
      "completions/min_length": 218.0,
      "epoch": 9.183288409703504,
      "grad_norm": 3.6089161516413615,
      "kl": 2.10546875,
      "learning_rate": 1.6687308930536825e-08,
      "loss": 0.17808927595615387,
      "memory(GiB)": 75.26,
      "reward": 2.0114762783050537,
      "reward_std": 0.4768283814191818,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.4401901513338089,
      "rewards/ReflectionBalance/mean": 0.8200699985027313,
      "rewards/ReflectionBalance/std": 0.22812560945749283,
      "step": 1699,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 475.203125,
      "completions/min_length": 273.5,
      "epoch": 9.18867924528302,
      "grad_norm": 3.4340652741393813,
      "kl": 2.421875,
      "learning_rate": 1.6468234208169786e-08,
      "loss": 0.19809964299201965,
      "memory(GiB)": 75.26,
      "reward": 2.097714900970459,
      "reward_std": 0.4383852183818817,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.328125,
      "rewards/MultiModalAccuracyORM/std": 0.4653245508670807,
      "rewards/ReflectionBalance/mean": 0.8438085913658142,
      "rewards/ReflectionBalance/std": 0.2341993823647499,
      "step": 1700,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 509.84375,
      "completions/min_length": 189.5,
      "epoch": 9.194070080862534,
      "grad_norm": 4.09502570566877,
      "kl": 3.4375,
      "learning_rate": 1.6250582954629445e-08,
      "loss": 0.2772149443626404,
      "memory(GiB)": 75.26,
      "reward": 1.8319010734558105,
      "reward_std": 0.7047967314720154,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.322907418012619,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.38125884532928467,
      "rewards/ReflectionBalance/mean": 0.7733072936534882,
      "rewards/ReflectionBalance/std": 0.3541446179151535,
      "step": 1701,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 523.1875,
      "completions/min_length": 280.0,
      "epoch": 9.199460916442048,
      "grad_norm": 3.7362835676205592,
      "kl": 2.578125,
      "learning_rate": 1.603435581065915e-08,
      "loss": 0.23946310579776764,
      "memory(GiB)": 75.26,
      "reward": 1.9595296382904053,
      "reward_std": 0.5249509811401367,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2897924929857254,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.42273683845996857,
      "rewards/ReflectionBalance/mean": 0.8189046084880829,
      "rewards/ReflectionBalance/std": 0.23029303550720215,
      "step": 1702,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 549.4609375,
      "completions/min_length": 299.5,
      "epoch": 9.204851752021563,
      "grad_norm": 3.896974843994123,
      "kl": 2.1484375,
      "learning_rate": 1.581955341280977e-08,
      "loss": 0.178850919008255,
      "memory(GiB)": 75.26,
      "reward": 1.9043880701065063,
      "reward_std": 0.46717098355293274,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2801405265927315,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3792887181043625,
      "rewards/ReflectionBalance/mean": 0.818450540304184,
      "rewards/ReflectionBalance/std": 0.229511559009552,
      "step": 1703,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 507.85546875,
      "completions/min_length": 259.5,
      "epoch": 9.210242587601078,
      "grad_norm": 3.0489498843804315,
      "kl": 2.921875,
      "learning_rate": 1.560617639343803e-08,
      "loss": 0.25165820121765137,
      "memory(GiB)": 75.26,
      "reward": 1.8556135892868042,
      "reward_std": 0.47649119794368744,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2813730239868164,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.33513282239437103,
      "rewards/ReflectionBalance/mean": 0.8126448690891266,
      "rewards/ReflectionBalance/std": 0.25634046643972397,
      "step": 1704,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 493.3125,
      "completions/min_length": 261.0,
      "epoch": 9.215633423180593,
      "grad_norm": 3.8589906223276143,
      "kl": 2.29296875,
      "learning_rate": 1.539422538070434e-08,
      "loss": 0.16998490691184998,
      "memory(GiB)": 75.26,
      "reward": 1.939308226108551,
      "reward_std": 0.49452565610408783,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.2960064336657524,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4270860105752945,
      "rewards/ReflectionBalance/mean": 0.8025895357131958,
      "rewards/ReflectionBalance/std": 0.23447149991989136,
      "step": 1705,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 500.98828125,
      "completions/min_length": 265.0,
      "epoch": 9.221024258760108,
      "grad_norm": 3.6865307670673806,
      "kl": 2.0625,
      "learning_rate": 1.518370099857119e-08,
      "loss": 0.16213995218276978,
      "memory(GiB)": 75.26,
      "reward": 1.8813021183013916,
      "reward_std": 0.3971695303916931,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2621928080916405,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.33063435554504395,
      "rewards/ReflectionBalance/mean": 0.8227083384990692,
      "rewards/ReflectionBalance/std": 0.24007752537727356,
      "step": 1706,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1020.5,
      "completions/mean_length": 470.8359375,
      "completions/min_length": 267.0,
      "epoch": 9.226415094339623,
      "grad_norm": 3.5712007655177,
      "kl": 1.75,
      "learning_rate": 1.4974603866800995e-08,
      "loss": 0.13059791922569275,
      "memory(GiB)": 75.26,
      "reward": 2.0730568766593933,
      "reward_std": 0.4811861217021942,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.3359375,
      "rewards/MultiModalAccuracyORM/std": 0.46682190895080566,
      "rewards/ReflectionBalance/mean": 0.8035257160663605,
      "rewards/ReflectionBalance/std": 0.23680385202169418,
      "step": 1707,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.5,
      "completions/mean_length": 493.359375,
      "completions/min_length": 280.0,
      "epoch": 9.231805929919137,
      "grad_norm": 3.191775889213673,
      "kl": 2.25390625,
      "learning_rate": 1.476693460095485e-08,
      "loss": 0.19986768066883087,
      "memory(GiB)": 75.26,
      "reward": 1.9409464001655579,
      "reward_std": 0.4860949218273163,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.40095221996307373,
      "rewards/ReflectionBalance/mean": 0.8198527097702026,
      "rewards/ReflectionBalance/std": 0.27085959911346436,
      "step": 1708,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 499.234375,
      "completions/min_length": 274.5,
      "epoch": 9.237196765498652,
      "grad_norm": 3.6383391105268275,
      "kl": 2.9296875,
      "learning_rate": 1.4560693812390112e-08,
      "loss": 0.21158912777900696,
      "memory(GiB)": 75.26,
      "reward": 1.8411539793014526,
      "reward_std": 0.43589383363723755,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.3338889628648758,
      "rewards/ReflectionBalance/mean": 0.8099039494991302,
      "rewards/ReflectionBalance/std": 0.23510615527629852,
      "step": 1709,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 494.68359375,
      "completions/min_length": 276.0,
      "epoch": 9.242587601078167,
      "grad_norm": 4.428157505629073,
      "kl": 2.43359375,
      "learning_rate": 1.4355882108258833e-08,
      "loss": 0.18958087265491486,
      "memory(GiB)": 75.26,
      "reward": 1.922900378704071,
      "reward_std": 0.46633338928222656,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38864077627658844,
      "rewards/ReflectionBalance/mean": 0.821337878704071,
      "rewards/ReflectionBalance/std": 0.2632293850183487,
      "step": 1710,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 504.9921875,
      "completions/min_length": 265.5,
      "epoch": 9.247978436657682,
      "grad_norm": 3.1504708018013288,
      "kl": 2.875,
      "learning_rate": 1.4152500091506091e-08,
      "loss": 0.2392755150794983,
      "memory(GiB)": 75.26,
      "reward": 1.8488468527793884,
      "reward_std": 0.4817833751440048,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3410066068172455,
      "rewards/ReflectionBalance/mean": 0.801971822977066,
      "rewards/ReflectionBalance/std": 0.27689122408628464,
      "step": 1711,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 472.31640625,
      "completions/min_length": 251.0,
      "epoch": 9.253369272237197,
      "grad_norm": 3.543365576143435,
      "kl": 1.73828125,
      "learning_rate": 1.3950548360867953e-08,
      "loss": 0.14882904291152954,
      "memory(GiB)": 75.26,
      "reward": 2.117882490158081,
      "reward_std": 0.43749991059303284,
      "rewards/Format/mean": 0.96484375,
      "rewards/Format/std": 0.17638477310538292,
      "rewards/MultiModalAccuracyORM/mean": 0.32421875,
      "rewards/MultiModalAccuracyORM/std": 0.4676540791988373,
      "rewards/ReflectionBalance/mean": 0.828819990158081,
      "rewards/ReflectionBalance/std": 0.23261777311563492,
      "step": 1712,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 478.84375,
      "completions/min_length": 262.5,
      "epoch": 9.25876010781671,
      "grad_norm": 3.207132660557291,
      "kl": 3.2109375,
      "learning_rate": 1.3750027510870177e-08,
      "loss": 0.2501416802406311,
      "memory(GiB)": 75.26,
      "reward": 1.8855451941490173,
      "reward_std": 0.48115743696689606,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39132989943027496,
      "rewards/ReflectionBalance/mean": 0.7878890037536621,
      "rewards/ReflectionBalance/std": 0.29002758860588074,
      "step": 1713,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 530.18359375,
      "completions/min_length": 290.0,
      "epoch": 9.264150943396226,
      "grad_norm": 3.0537824271547684,
      "kl": 2.984375,
      "learning_rate": 1.355093813182584e-08,
      "loss": 0.26044172048568726,
      "memory(GiB)": 75.26,
      "reward": 1.8226529955863953,
      "reward_std": 0.5127219557762146,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3229459524154663,
      "rewards/ReflectionBalance/mean": 0.8109342455863953,
      "rewards/ReflectionBalance/std": 0.2640842869877815,
      "step": 1714,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 490.7578125,
      "completions/min_length": 206.0,
      "epoch": 9.269541778975741,
      "grad_norm": 3.679131037585627,
      "kl": 2.90625,
      "learning_rate": 1.3353280809834e-08,
      "loss": 0.23432572185993195,
      "memory(GiB)": 75.26,
      "reward": 1.9313794374465942,
      "reward_std": 0.5848394930362701,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4312909245491028,
      "rewards/ReflectionBalance/mean": 0.7829419076442719,
      "rewards/ReflectionBalance/std": 0.2606634050607681,
      "step": 1715,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 499.2109375,
      "completions/min_length": 238.5,
      "epoch": 9.274932614555256,
      "grad_norm": 5.336629731032516,
      "kl": 2.7265625,
      "learning_rate": 1.3157056126778031e-08,
      "loss": 0.20678211748600006,
      "memory(GiB)": 75.26,
      "reward": 1.8735898733139038,
      "reward_std": 0.5324686765670776,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3240884244441986,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38704802095890045,
      "rewards/ReflectionBalance/mean": 0.8149960935115814,
      "rewards/ReflectionBalance/std": 0.24689758569002151,
      "step": 1716,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 476.9765625,
      "completions/min_length": 161.0,
      "epoch": 9.280323450134771,
      "grad_norm": 3.637331177422512,
      "kl": 2.0625,
      "learning_rate": 1.2962264660323464e-08,
      "loss": 0.15743473172187805,
      "memory(GiB)": 75.26,
      "reward": 2.0886881351470947,
      "reward_std": 0.4891507923603058,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.328125,
      "rewards/MultiModalAccuracyORM/std": 0.46660420298576355,
      "rewards/ReflectionBalance/mean": 0.8386881351470947,
      "rewards/ReflectionBalance/std": 0.1854042261838913,
      "step": 1717,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 490.84765625,
      "completions/min_length": 256.0,
      "epoch": 9.285714285714286,
      "grad_norm": 3.1219778707769206,
      "kl": 2.6328125,
      "learning_rate": 1.2768906983916916e-08,
      "loss": 0.19125300645828247,
      "memory(GiB)": 75.26,
      "reward": 1.910545289516449,
      "reward_std": 0.47925299406051636,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2897924929857254,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.39416566491127014,
      "rewards/ReflectionBalance/mean": 0.8128890097141266,
      "rewards/ReflectionBalance/std": 0.26846108585596085,
      "step": 1718,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 547.8359375,
      "completions/min_length": 281.5,
      "epoch": 9.2911051212938,
      "grad_norm": 4.602155508515284,
      "kl": 4.25,
      "learning_rate": 1.2576983666783835e-08,
      "loss": 0.32782620191574097,
      "memory(GiB)": 75.26,
      "reward": 1.778033196926117,
      "reward_std": 0.5803587734699249,
      "rewards/Format/mean": 0.8515625,
      "rewards/Format/std": 0.35259076952934265,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3634065240621567,
      "rewards/ReflectionBalance/mean": 0.7663144767284393,
      "rewards/ReflectionBalance/std": 0.34723278880119324,
      "step": 1719,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 499.08203125,
      "completions/min_length": 239.5,
      "epoch": 9.296495956873315,
      "grad_norm": 3.287848147060019,
      "kl": 3.078125,
      "learning_rate": 1.23864952739271e-08,
      "loss": 0.25564369559288025,
      "memory(GiB)": 75.26,
      "reward": 1.9396883249282837,
      "reward_std": 0.5090432167053223,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.3986760377883911,
      "rewards/ReflectionBalance/mean": 0.8185947239398956,
      "rewards/ReflectionBalance/std": 0.2251286432147026,
      "step": 1720,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.5,
      "completions/mean_length": 506.39453125,
      "completions/min_length": 171.0,
      "epoch": 9.30188679245283,
      "grad_norm": 4.186108262476558,
      "kl": 3.71875,
      "learning_rate": 1.2197442366125465e-08,
      "loss": 0.259072482585907,
      "memory(GiB)": 75.26,
      "reward": 1.7896207571029663,
      "reward_std": 0.5221225172281265,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.26704202964901924,
      "rewards/ReflectionBalance/mean": 0.7818082571029663,
      "rewards/ReflectionBalance/std": 0.3006039708852768,
      "step": 1721,
      "train_speed(iter/s)": 0.007123
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 495.85546875,
      "completions/min_length": 282.5,
      "epoch": 9.307277628032345,
      "grad_norm": 2.9366767161675584,
      "kl": 3.1171875,
      "learning_rate": 1.2009825499931626e-08,
      "loss": 0.25581589341163635,
      "memory(GiB)": 75.26,
      "reward": 1.946835994720459,
      "reward_std": 0.5632698237895966,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.43581391870975494,
      "rewards/ReflectionBalance/mean": 0.7944921851158142,
      "rewards/ReflectionBalance/std": 0.2750280946493149,
      "step": 1722,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 476.84375,
      "completions/min_length": 248.5,
      "epoch": 9.31266846361186,
      "grad_norm": 4.062857976640973,
      "kl": 2.9453125,
      "learning_rate": 1.1823645227670654e-08,
      "loss": 0.22418025135993958,
      "memory(GiB)": 75.26,
      "reward": 1.8737727999687195,
      "reward_std": 0.5287550389766693,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3068038523197174,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38784652948379517,
      "rewards/ReflectionBalance/mean": 0.7995540201663971,
      "rewards/ReflectionBalance/std": 0.2440415546298027,
      "step": 1723,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 491.03125,
      "completions/min_length": 240.0,
      "epoch": 9.318059299191376,
      "grad_norm": 3.450100942646596,
      "kl": 2.5546875,
      "learning_rate": 1.1638902097438618e-08,
      "loss": 0.21342360973358154,
      "memory(GiB)": 75.26,
      "reward": 1.9266275763511658,
      "reward_std": 0.486745148897171,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.40016549825668335,
      "rewards/ReflectionBalance/mean": 0.8094401061534882,
      "rewards/ReflectionBalance/std": 0.2380911409854889,
      "step": 1724,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 911.5,
      "completions/mean_length": 502.91796875,
      "completions/min_length": 257.5,
      "epoch": 9.323450134770889,
      "grad_norm": 3.494171420969686,
      "kl": 2.0703125,
      "learning_rate": 1.1455596653100642e-08,
      "loss": 0.16110962629318237,
      "memory(GiB)": 75.26,
      "reward": 1.8983349204063416,
      "reward_std": 0.48475922644138336,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.27558503299951553,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3636700212955475,
      "rewards/ReflectionBalance/mean": 0.8123974800109863,
      "rewards/ReflectionBalance/std": 0.23189112544059753,
      "step": 1725,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 507.8828125,
      "completions/min_length": 258.5,
      "epoch": 9.328840970350404,
      "grad_norm": 3.0196680657061514,
      "kl": 1.97265625,
      "learning_rate": 1.1273729434289558e-08,
      "loss": 0.15238842368125916,
      "memory(GiB)": 75.26,
      "reward": 1.8694466352462769,
      "reward_std": 0.4056870639324188,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.3206050544977188,
      "rewards/ReflectionBalance/mean": 0.8225716054439545,
      "rewards/ReflectionBalance/std": 0.22679510712623596,
      "step": 1726,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 508.7265625,
      "completions/min_length": 260.5,
      "epoch": 9.33423180592992,
      "grad_norm": 3.9583891189053233,
      "kl": 3.0390625,
      "learning_rate": 1.1093300976404042e-08,
      "loss": 0.23219186067581177,
      "memory(GiB)": 75.26,
      "reward": 1.9278027415275574,
      "reward_std": 0.5443556308746338,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.422762006521225,
      "rewards/ReflectionBalance/mean": 0.8067089915275574,
      "rewards/ReflectionBalance/std": 0.2610793933272362,
      "step": 1727,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 469.52734375,
      "completions/min_length": 261.5,
      "epoch": 9.339622641509434,
      "grad_norm": 3.084013144862746,
      "kl": 2.39453125,
      "learning_rate": 1.0914311810607424e-08,
      "loss": 0.1937524527311325,
      "memory(GiB)": 75.26,
      "reward": 1.9737792015075684,
      "reward_std": 0.42232072353363037,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.2379390001296997,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.405468225479126,
      "rewards/ReflectionBalance/mean": 0.8253417909145355,
      "rewards/ReflectionBalance/std": 0.22613365948200226,
      "step": 1728,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 488.5390625,
      "completions/min_length": 226.5,
      "epoch": 9.34501347708895,
      "grad_norm": 3.2724567854858577,
      "kl": 2.39453125,
      "learning_rate": 1.0736762463825656e-08,
      "loss": 0.2069237232208252,
      "memory(GiB)": 75.26,
      "reward": 1.957314372062683,
      "reward_std": 0.544323742389679,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3191515803337097,
      "rewards/MultiModalAccuracyORM/mean": 0.265625,
      "rewards/MultiModalAccuracyORM/std": 0.4401783347129822,
      "rewards/ReflectionBalance/mean": 0.8088769614696503,
      "rewards/ReflectionBalance/std": 0.23353544622659683,
      "step": 1729,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 528.12890625,
      "completions/min_length": 284.5,
      "epoch": 9.350404312668463,
      "grad_norm": 3.6108768049115105,
      "kl": 3.265625,
      "learning_rate": 1.0560653458746183e-08,
      "loss": 0.2762284576892853,
      "memory(GiB)": 75.26,
      "reward": 1.8412858247756958,
      "reward_std": 0.5964239239692688,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32746291160583496,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3682406395673752,
      "rewards/ReflectionBalance/mean": 0.7748795449733734,
      "rewards/ReflectionBalance/std": 0.3089938163757324,
      "step": 1730,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 485.9609375,
      "completions/min_length": 295.5,
      "epoch": 9.355795148247978,
      "grad_norm": 3.1284868701805983,
      "kl": 2.15625,
      "learning_rate": 1.0385985313816292e-08,
      "loss": 0.17993852496147156,
      "memory(GiB)": 75.26,
      "reward": 1.8815893530845642,
      "reward_std": 0.41814015805721283,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2694226801395416,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.356930136680603,
      "rewards/ReflectionBalance/mean": 0.8112768530845642,
      "rewards/ReflectionBalance/std": 0.24784813821315765,
      "step": 1731,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 512.671875,
      "completions/min_length": 256.5,
      "epoch": 9.361185983827493,
      "grad_norm": 2.875153208207539,
      "kl": 1.99609375,
      "learning_rate": 1.0212758543241218e-08,
      "loss": 0.1779579520225525,
      "memory(GiB)": 75.26,
      "reward": 1.9733480215072632,
      "reward_std": 0.46689988672733307,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23443959653377533,
      "rewards/MultiModalAccuracyORM/mean": 0.20703125,
      "rewards/MultiModalAccuracyORM/std": 0.4018339514732361,
      "rewards/ReflectionBalance/mean": 0.8249104917049408,
      "rewards/ReflectionBalance/std": 0.2489563226699829,
      "step": 1732,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 501.453125,
      "completions/min_length": 269.0,
      "epoch": 9.366576819407008,
      "grad_norm": 3.5088168814683853,
      "kl": 2.453125,
      "learning_rate": 1.004097365698331e-08,
      "loss": 0.19663439691066742,
      "memory(GiB)": 75.26,
      "reward": 1.8811962604522705,
      "reward_std": 0.427192747592926,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3050640821456909,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3842122554779053,
      "rewards/ReflectionBalance/mean": 0.8069775700569153,
      "rewards/ReflectionBalance/std": 0.25663280487060547,
      "step": 1733,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 487.3984375,
      "completions/min_length": 273.0,
      "epoch": 9.371967654986523,
      "grad_norm": 3.809117280078344,
      "kl": 2.3125,
      "learning_rate": 9.870631160759868e-09,
      "loss": 0.181738942861557,
      "memory(GiB)": 75.26,
      "reward": 1.9253954887390137,
      "reward_std": 0.41476261615753174,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.1640625,
      "rewards/MultiModalAccuracyORM/std": 0.3166762590408325,
      "rewards/ReflectionBalance/mean": 0.8316455185413361,
      "rewards/ReflectionBalance/std": 0.23499468713998795,
      "step": 1734,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 508.08984375,
      "completions/min_length": 158.5,
      "epoch": 9.377358490566039,
      "grad_norm": 2.797683525693373,
      "kl": 2.8046875,
      "learning_rate": 9.701731556042036e-09,
      "loss": 0.21183812618255615,
      "memory(GiB)": 75.26,
      "reward": 1.9455289840698242,
      "reward_std": 0.5843646228313446,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31333550810813904,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.43674568831920624,
      "rewards/ReflectionBalance/mean": 0.8009977340698242,
      "rewards/ReflectionBalance/std": 0.28643907606601715,
      "step": 1735,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 499.51953125,
      "completions/min_length": 253.0,
      "epoch": 9.382749326145552,
      "grad_norm": 3.23153665422673,
      "kl": 2.953125,
      "learning_rate": 9.534275340053355e-09,
      "loss": 0.26348286867141724,
      "memory(GiB)": 75.26,
      "reward": 1.9177539348602295,
      "reward_std": 0.5956528186798096,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31763872504234314,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.3935956656932831,
      "rewards/ReflectionBalance/mean": 0.7966601550579071,
      "rewards/ReflectionBalance/std": 0.2981445789337158,
      "step": 1736,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 509.3046875,
      "completions/min_length": 308.5,
      "epoch": 9.388140161725067,
      "grad_norm": 4.008978696496927,
      "kl": 2.375,
      "learning_rate": 9.36826300576793e-09,
      "loss": 0.16771449148654938,
      "memory(GiB)": 75.26,
      "reward": 1.9707112908363342,
      "reward_std": 0.46193547546863556,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4141528308391571,
      "rewards/ReflectionBalance/mean": 0.8183675110340118,
      "rewards/ReflectionBalance/std": 0.25830787420272827,
      "step": 1737,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 492.28125,
      "completions/min_length": 233.5,
      "epoch": 9.393530997304582,
      "grad_norm": 3.3840971012040595,
      "kl": 3.2734375,
      "learning_rate": 9.203695041909431e-09,
      "loss": 0.25607454776763916,
      "memory(GiB)": 75.26,
      "reward": 1.8233105540275574,
      "reward_std": 0.5092452168464661,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.3645188808441162,
      "rewards/ReflectionBalance/mean": 0.7803418040275574,
      "rewards/ReflectionBalance/std": 0.3113171011209488,
      "step": 1738,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 471.14453125,
      "completions/min_length": 267.5,
      "epoch": 9.398921832884097,
      "grad_norm": 4.494112327051385,
      "kl": 1.63671875,
      "learning_rate": 9.040571932949326e-09,
      "loss": 0.16614513099193573,
      "memory(GiB)": 75.26,
      "reward": 2.110633969306946,
      "reward_std": 0.4591474086046219,
      "rewards/Format/mean": 0.96875,
      "rewards/Format/std": 0.17319392412900925,
      "rewards/MultiModalAccuracyORM/mean": 0.3125,
      "rewards/MultiModalAccuracyORM/std": 0.4650258719921112,
      "rewards/ReflectionBalance/mean": 0.8293839693069458,
      "rewards/ReflectionBalance/std": 0.2222408652305603,
      "step": 1739,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 968.5,
      "completions/mean_length": 491.23828125,
      "completions/min_length": 264.0,
      "epoch": 9.404312668463612,
      "grad_norm": 2.8304721979167664,
      "kl": 2.21484375,
      "learning_rate": 8.878894159105644e-09,
      "loss": 0.17710380256175995,
      "memory(GiB)": 75.26,
      "reward": 1.974916934967041,
      "reward_std": 0.5344254225492477,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.28448406606912613,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.43412669003009796,
      "rewards/ReflectionBalance/mean": 0.8147607445716858,
      "rewards/ReflectionBalance/std": 0.24708855897188187,
      "step": 1740,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1042.0,
      "completions/mean_length": 490.48046875,
      "completions/min_length": 215.0,
      "epoch": 9.409703504043126,
      "grad_norm": 3.4919333641370605,
      "kl": 2.0234375,
      "learning_rate": 8.718662196341542e-09,
      "loss": 0.13849464058876038,
      "memory(GiB)": 75.26,
      "reward": 2.0019742250442505,
      "reward_std": 0.41256022453308105,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2524137645959854,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4312489330768585,
      "rewards/ReflectionBalance/mean": 0.8261930346488953,
      "rewards/ReflectionBalance/std": 0.23138156533241272,
      "step": 1741,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 463.67578125,
      "completions/min_length": 180.0,
      "epoch": 9.415094339622641,
      "grad_norm": 30.11617402875464,
      "kl": 1.61328125,
      "learning_rate": 8.559876516363696e-09,
      "loss": 0.1216234341263771,
      "memory(GiB)": 75.26,
      "reward": 2.0286149978637695,
      "reward_std": 0.4059555381536484,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.2356376200914383,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.43581391870975494,
      "rewards/ReflectionBalance/mean": 0.8333024084568024,
      "rewards/ReflectionBalance/std": 0.211410753428936,
      "step": 1742,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 479.48828125,
      "completions/min_length": 271.5,
      "epoch": 9.420485175202156,
      "grad_norm": 3.0206130184077056,
      "kl": 2.62109375,
      "learning_rate": 8.402537586621183e-09,
      "loss": 0.2277406007051468,
      "memory(GiB)": 75.26,
      "reward": 1.9359057545661926,
      "reward_std": 0.4337502419948578,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.23675358667969704,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.34576793015003204,
      "rewards/ReflectionBalance/mean": 0.8148120045661926,
      "rewards/ReflectionBalance/std": 0.23882964253425598,
      "step": 1743,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 505.390625,
      "completions/min_length": 283.5,
      "epoch": 9.425876010781671,
      "grad_norm": 3.9708624192444932,
      "kl": 2.7734375,
      "learning_rate": 8.24664587030388e-09,
      "loss": 0.19457577168941498,
      "memory(GiB)": 75.26,
      "reward": 1.9451366662979126,
      "reward_std": 0.4511909484863281,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41635647416114807,
      "rewards/ReflectionBalance/mean": 0.8045117259025574,
      "rewards/ReflectionBalance/std": 0.24266328662633896,
      "step": 1744,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 474.46875,
      "completions/min_length": 260.5,
      "epoch": 9.431266846361186,
      "grad_norm": 3.3723405885736932,
      "kl": 2.3515625,
      "learning_rate": 8.092201826341461e-09,
      "loss": 0.20925024151802063,
      "memory(GiB)": 75.26,
      "reward": 2.035813570022583,
      "reward_std": 0.45657308399677277,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.4473883658647537,
      "rewards/ReflectionBalance/mean": 0.8248763084411621,
      "rewards/ReflectionBalance/std": 0.2274961844086647,
      "step": 1745,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 488.7109375,
      "completions/min_length": 230.0,
      "epoch": 9.436657681940702,
      "grad_norm": 3.350872505358001,
      "kl": 2.3046875,
      "learning_rate": 7.939205909401558e-09,
      "loss": 0.18192048370838165,
      "memory(GiB)": 75.26,
      "reward": 1.9150308966636658,
      "reward_std": 0.4921368956565857,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3787541687488556,
      "rewards/ReflectionBalance/mean": 0.8290933966636658,
      "rewards/ReflectionBalance/std": 0.2370578572154045,
      "step": 1746,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 490.8359375,
      "completions/min_length": 275.5,
      "epoch": 9.442048517520215,
      "grad_norm": 3.6793162991763384,
      "kl": 2.2109375,
      "learning_rate": 7.787658569888722e-09,
      "loss": 0.1660769283771515,
      "memory(GiB)": 75.26,
      "reward": 2.001064419746399,
      "reward_std": 0.5110474228858948,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.26171875,
      "rewards/MultiModalAccuracyORM/std": 0.4410945028066635,
      "rewards/ReflectionBalance/mean": 0.8135644197463989,
      "rewards/ReflectionBalance/std": 0.24340634047985077,
      "step": 1747,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 482.9609375,
      "completions/min_length": 267.0,
      "epoch": 9.44743935309973,
      "grad_norm": 3.2916722117840287,
      "kl": 2.84375,
      "learning_rate": 7.637560253943132e-09,
      "loss": 0.2525032162666321,
      "memory(GiB)": 75.26,
      "reward": 1.939415156841278,
      "reward_std": 0.5212972164154053,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.23046875,
      "rewards/MultiModalAccuracyORM/std": 0.4225567579269409,
      "rewards/ReflectionBalance/mean": 0.8066025972366333,
      "rewards/ReflectionBalance/std": 0.264851413667202,
      "step": 1748,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 502.6640625,
      "completions/min_length": 173.0,
      "epoch": 9.452830188679245,
      "grad_norm": 5.4198673891780444,
      "kl": 2.3046875,
      "learning_rate": 7.488911403439046e-09,
      "loss": 0.17933548986911774,
      "memory(GiB)": 75.26,
      "reward": 1.8791096806526184,
      "reward_std": 0.47160741686820984,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2829600051045418,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.38125884532928467,
      "rewards/ReflectionBalance/mean": 0.7931722402572632,
      "rewards/ReflectionBalance/std": 0.2806062400341034,
      "step": 1749,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1032.5,
      "completions/mean_length": 500.80859375,
      "completions/min_length": 287.5,
      "epoch": 9.45822102425876,
      "grad_norm": 3.684477622952376,
      "kl": 2.6328125,
      "learning_rate": 7.341712455983695e-09,
      "loss": 0.21342343091964722,
      "memory(GiB)": 75.26,
      "reward": 1.8284797668457031,
      "reward_std": 0.4484306275844574,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.11328125,
      "rewards/MultiModalAccuracyORM/std": 0.3166656941175461,
      "rewards/ReflectionBalance/mean": 0.8089486062526703,
      "rewards/ReflectionBalance/std": 0.23871012032032013,
      "step": 1750,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 510.71484375,
      "completions/min_length": 264.5,
      "epoch": 9.463611859838275,
      "grad_norm": 4.008298871216488,
      "kl": 2.30078125,
      "learning_rate": 7.195963844916053e-09,
      "loss": 0.18916207551956177,
      "memory(GiB)": 75.26,
      "reward": 1.9247721433639526,
      "reward_std": 0.5232656300067902,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3050640821456909,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.42014752328395844,
      "rewards/ReflectionBalance/mean": 0.8036783933639526,
      "rewards/ReflectionBalance/std": 0.2412608489394188,
      "step": 1751,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1021.0,
      "completions/mean_length": 474.83203125,
      "completions/min_length": 217.5,
      "epoch": 9.46900269541779,
      "grad_norm": 4.569882962202022,
      "kl": 2.4765625,
      "learning_rate": 7.051665999305234e-09,
      "loss": 0.16434770822525024,
      "memory(GiB)": 75.26,
      "reward": 1.8429652452468872,
      "reward_std": 0.5280993282794952,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.3562464565038681,
      "rewards/ReflectionBalance/mean": 0.7843714356422424,
      "rewards/ReflectionBalance/std": 0.318198561668396,
      "step": 1752,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 496.70703125,
      "completions/min_length": 227.5,
      "epoch": 9.474393530997304,
      "grad_norm": 2.925411480009263,
      "kl": 3.03125,
      "learning_rate": 6.908819343949656e-09,
      "loss": 0.24252654612064362,
      "memory(GiB)": 75.26,
      "reward": 1.8767187595367432,
      "reward_std": 0.5614569187164307,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.3846261203289032,
      "rewards/ReflectionBalance/mean": 0.7907812297344208,
      "rewards/ReflectionBalance/std": 0.2826945334672928,
      "step": 1753,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 483.28125,
      "completions/min_length": 213.5,
      "epoch": 9.479784366576819,
      "grad_norm": 3.028936808198931,
      "kl": 2.6015625,
      "learning_rate": 6.76742429937549e-09,
      "loss": 0.19625678658485413,
      "memory(GiB)": 75.26,
      "reward": 1.9136927723884583,
      "reward_std": 0.4882950335741043,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4089515954256058,
      "rewards/ReflectionBalance/mean": 0.7886927127838135,
      "rewards/ReflectionBalance/std": 0.28174228966236115,
      "step": 1754,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 493.26171875,
      "completions/min_length": 249.5,
      "epoch": 9.485175202156334,
      "grad_norm": 3.294899107059203,
      "kl": 2.0390625,
      "learning_rate": 6.62748128183549e-09,
      "loss": 0.1637364774942398,
      "memory(GiB)": 75.26,
      "reward": 1.9473013877868652,
      "reward_std": 0.46445903182029724,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.36347243934869766,
      "rewards/ReflectionBalance/mean": 0.8105826675891876,
      "rewards/ReflectionBalance/std": 0.2548331320285797,
      "step": 1755,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 500.6875,
      "completions/min_length": 237.0,
      "epoch": 9.49056603773585,
      "grad_norm": 3.4935716929416967,
      "kl": 2.4765625,
      "learning_rate": 6.488990703307717e-09,
      "loss": 0.19113147258758545,
      "memory(GiB)": 75.26,
      "reward": 1.8138639330863953,
      "reward_std": 0.43381278216838837,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.3077666610479355,
      "rewards/ReflectionBalance/mean": 0.7982389330863953,
      "rewards/ReflectionBalance/std": 0.2824038118124008,
      "step": 1756,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 514.375,
      "completions/min_length": 167.5,
      "epoch": 9.495956873315365,
      "grad_norm": 3.743607419697809,
      "kl": 2.609375,
      "learning_rate": 6.351952971494657e-09,
      "loss": 0.2126784324645996,
      "memory(GiB)": 75.26,
      "reward": 1.9713232517242432,
      "reward_std": 0.49880895018577576,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4324023276567459,
      "rewards/ReflectionBalance/mean": 0.8228857517242432,
      "rewards/ReflectionBalance/std": 0.24630098789930344,
      "step": 1757,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 500.95703125,
      "completions/min_length": 285.0,
      "epoch": 9.501347708894878,
      "grad_norm": 49.99206930303763,
      "kl": 3.0234375,
      "learning_rate": 6.2163684898214334e-09,
      "loss": 0.2050076276063919,
      "memory(GiB)": 75.26,
      "reward": 1.94443678855896,
      "reward_std": 0.392897292971611,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2488820105791092,
      "rewards/MultiModalAccuracyORM/mean": 0.17578125,
      "rewards/MultiModalAccuracyORM/std": 0.3778642416000366,
      "rewards/ReflectionBalance/mean": 0.8389680981636047,
      "rewards/ReflectionBalance/std": 0.1970771849155426,
      "step": 1758,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 946.0,
      "completions/mean_length": 522.75,
      "completions/min_length": 290.5,
      "epoch": 9.506738544474393,
      "grad_norm": 3.0042475934571566,
      "kl": 2.19921875,
      "learning_rate": 6.082237657435152e-09,
      "loss": 0.17560794949531555,
      "memory(GiB)": 75.26,
      "reward": 1.9670572876930237,
      "reward_std": 0.42485859990119934,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2419506385922432,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.40794737637043,
      "rewards/ReflectionBalance/mean": 0.8225260674953461,
      "rewards/ReflectionBalance/std": 0.21108602732419968,
      "step": 1759,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 479.26171875,
      "completions/min_length": 274.0,
      "epoch": 9.512129380053908,
      "grad_norm": 3.92306102142445,
      "kl": 1.86328125,
      "learning_rate": 5.949560869203562e-09,
      "loss": 0.15843963623046875,
      "memory(GiB)": 75.26,
      "reward": 1.9993432760238647,
      "reward_std": 0.40884849429130554,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41857391595840454,
      "rewards/ReflectionBalance/mean": 0.8352807462215424,
      "rewards/ReflectionBalance/std": 0.20465848594903946,
      "step": 1760,
      "train_speed(iter/s)": 0.007124
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 503.609375,
      "completions/min_length": 301.5,
      "epoch": 9.517520215633423,
      "grad_norm": 4.69184065756046,
      "kl": 2.859375,
      "learning_rate": 5.8183385157137275e-09,
      "loss": 0.2240031212568283,
      "memory(GiB)": 75.26,
      "reward": 1.8677507042884827,
      "reward_std": 0.49232713878154755,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3029807060956955,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.36627575755119324,
      "rewards/ReflectionBalance/mean": 0.8091568946838379,
      "rewards/ReflectionBalance/std": 0.2749854028224945,
      "step": 1761,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 485.15234375,
      "completions/min_length": 267.0,
      "epoch": 9.522911051212938,
      "grad_norm": 3.318280233460712,
      "kl": 2.703125,
      "learning_rate": 5.688570983270968e-09,
      "loss": 0.22606441378593445,
      "memory(GiB)": 75.26,
      "reward": 1.9706069827079773,
      "reward_std": 0.45450371503829956,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.27558503299951553,
      "rewards/MultiModalAccuracyORM/mean": 0.2421875,
      "rewards/MultiModalAccuracyORM/std": 0.42764075100421906,
      "rewards/ReflectionBalance/mean": 0.8143571019172668,
      "rewards/ReflectionBalance/std": 0.22900864481925964,
      "step": 1762,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 933.0,
      "completions/mean_length": 464.84375,
      "completions/min_length": 249.5,
      "epoch": 9.528301886792454,
      "grad_norm": 2.9222438030847413,
      "kl": 1.81640625,
      "learning_rate": 5.56025865389792e-09,
      "loss": 0.14702866971492767,
      "memory(GiB)": 75.26,
      "reward": 2.0809383392333984,
      "reward_std": 0.47856560349464417,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.3203125,
      "rewards/MultiModalAccuracyORM/std": 0.4647130221128464,
      "rewards/ReflectionBalance/mean": 0.823125809431076,
      "rewards/ReflectionBalance/std": 0.20870095491409302,
      "step": 1763,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 505.046875,
      "completions/min_length": 246.0,
      "epoch": 9.533692722371967,
      "grad_norm": 2.770750598737762,
      "kl": 3.03125,
      "learning_rate": 5.43340190533309e-09,
      "loss": 0.26587554812431335,
      "memory(GiB)": 75.26,
      "reward": 1.9126920104026794,
      "reward_std": 0.4844963550567627,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.4037879854440689,
      "rewards/ReflectionBalance/mean": 0.8150357604026794,
      "rewards/ReflectionBalance/std": 0.23476170748472214,
      "step": 1764,
      "train_speed(iter/s)": 0.007125
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 468.2890625,
      "completions/min_length": 254.0,
      "epoch": 9.539083557951482,
      "grad_norm": 3.1878185520455804,
      "kl": 2.37890625,
      "learning_rate": 5.308001111030025e-09,
      "loss": 0.19229698181152344,
      "memory(GiB)": 75.26,
      "reward": 2.1082812547683716,
      "reward_std": 0.5291336476802826,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2657608464360237,
      "rewards/MultiModalAccuracyORM/mean": 0.3671875,
      "rewards/MultiModalAccuracyORM/std": 0.483863890171051,
      "rewards/ReflectionBalance/mean": 0.8192187547683716,
      "rewards/ReflectionBalance/std": 0.2502596899867058,
      "step": 1765,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 487.04296875,
      "completions/min_length": 160.0,
      "epoch": 9.544474393530997,
      "grad_norm": 3.3811921873566337,
      "kl": 2.75,
      "learning_rate": 5.184056640156031e-09,
      "loss": 0.21055030822753906,
      "memory(GiB)": 75.26,
      "reward": 1.8474072217941284,
      "reward_std": 0.5226948410272598,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.30740001052618027,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3634065240621567,
      "rewards/ReflectionBalance/mean": 0.8005322515964508,
      "rewards/ReflectionBalance/std": 0.25574713200330734,
      "step": 1766,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 496.80078125,
      "completions/min_length": 282.0,
      "epoch": 9.549865229110512,
      "grad_norm": 5.121765954178305,
      "kl": 2.46875,
      "learning_rate": 5.0615688575910695e-09,
      "loss": 0.22628769278526306,
      "memory(GiB)": 75.26,
      "reward": 1.9711555242538452,
      "reward_std": 0.4465321898460388,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.42014752328395844,
      "rewards/ReflectionBalance/mean": 0.8344368636608124,
      "rewards/ReflectionBalance/std": 0.21337196975946426,
      "step": 1767,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 982.5,
      "completions/mean_length": 476.98046875,
      "completions/min_length": 240.5,
      "epoch": 9.555256064690028,
      "grad_norm": 3.1149833225645875,
      "kl": 2.15234375,
      "learning_rate": 4.940538123927029e-09,
      "loss": 0.16241030395030975,
      "memory(GiB)": 75.26,
      "reward": 1.9437353014945984,
      "reward_std": 0.4696992039680481,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3032590448856354,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.424868568778038,
      "rewards/ReflectionBalance/mean": 0.8109228610992432,
      "rewards/ReflectionBalance/std": 0.2454986348748207,
      "step": 1768,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 500.94140625,
      "completions/min_length": 114.0,
      "epoch": 9.560646900269543,
      "grad_norm": 4.168692765282104,
      "kl": 2.609375,
      "learning_rate": 4.8209647954659536e-09,
      "loss": 0.21933194994926453,
      "memory(GiB)": 75.26,
      "reward": 1.8640152215957642,
      "reward_std": 0.5767614841461182,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.2999713644385338,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3949224203824997,
      "rewards/ReflectionBalance/mean": 0.7780777812004089,
      "rewards/ReflectionBalance/std": 0.30845117568969727,
      "step": 1769,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 501.70703125,
      "completions/min_length": 294.0,
      "epoch": 9.566037735849056,
      "grad_norm": 3.304916838021009,
      "kl": 2.25,
      "learning_rate": 4.7028492242198185e-09,
      "loss": 0.19768351316452026,
      "memory(GiB)": 75.26,
      "reward": 1.9045361280441284,
      "reward_std": 0.4984297901391983,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.3842122554779053,
      "rewards/ReflectionBalance/mean": 0.802973598241806,
      "rewards/ReflectionBalance/std": 0.2524620294570923,
      "step": 1770,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 505.23828125,
      "completions/min_length": 270.5,
      "epoch": 9.571428571428571,
      "grad_norm": 3.2895217559534125,
      "kl": 3.03125,
      "learning_rate": 4.58619175790903e-09,
      "loss": 0.2561578154563904,
      "memory(GiB)": 75.26,
      "reward": 1.9553434252738953,
      "reward_std": 0.48547619581222534,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2781740501523018,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4120787978172302,
      "rewards/ReflectionBalance/mean": 0.8264371752738953,
      "rewards/ReflectionBalance/std": 0.22093340009450912,
      "step": 1771,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 492.5859375,
      "completions/min_length": 267.5,
      "epoch": 9.576819407008086,
      "grad_norm": 3.285132556092817,
      "kl": 2.15625,
      "learning_rate": 4.470992739961266e-09,
      "loss": 0.18091543018817902,
      "memory(GiB)": 75.26,
      "reward": 2.0377570390701294,
      "reward_std": 0.4954496920108795,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24081215262413025,
      "rewards/MultiModalAccuracyORM/mean": 0.27734375,
      "rewards/MultiModalAccuracyORM/std": 0.4468510299921036,
      "rewards/ReflectionBalance/mean": 0.822913408279419,
      "rewards/ReflectionBalance/std": 0.20445825904607773,
      "step": 1772,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1054.0,
      "completions/mean_length": 480.015625,
      "completions/min_length": 254.5,
      "epoch": 9.582210242587601,
      "grad_norm": 3.0725816187720216,
      "kl": 2.2890625,
      "learning_rate": 4.3572525095109665e-09,
      "loss": 0.18701937794685364,
      "memory(GiB)": 75.26,
      "reward": 1.954370141029358,
      "reward_std": 0.4285445660352707,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24984508007764816,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.4022025018930435,
      "rewards/ReflectionBalance/mean": 0.8098388612270355,
      "rewards/ReflectionBalance/std": 0.25750134885311127,
      "step": 1773,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 502.44140625,
      "completions/min_length": 270.0,
      "epoch": 9.587601078167117,
      "grad_norm": 4.366262867768473,
      "kl": 1.6015625,
      "learning_rate": 4.244971401397845e-09,
      "loss": 0.15186896920204163,
      "memory(GiB)": 75.26,
      "reward": 1.8544132709503174,
      "reward_std": 0.40091341733932495,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.26781924813985825,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.3077666610479355,
      "rewards/ReflectionBalance/mean": 0.8270694613456726,
      "rewards/ReflectionBalance/std": 0.19463791698217392,
      "step": 1774,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1039.0,
      "completions/mean_length": 509.92578125,
      "completions/min_length": 282.5,
      "epoch": 9.59299191374663,
      "grad_norm": 3.0415032195256995,
      "kl": 3.1328125,
      "learning_rate": 4.134149746166215e-09,
      "loss": 0.2500203251838684,
      "memory(GiB)": 75.26,
      "reward": 1.9212386012077332,
      "reward_std": 0.6355981230735779,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.30921147763729095,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4324023276567459,
      "rewards/ReflectionBalance/mean": 0.7845198512077332,
      "rewards/ReflectionBalance/std": 0.29491426050662994,
      "step": 1775,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 475.6875,
      "completions/min_length": 266.0,
      "epoch": 9.598382749326145,
      "grad_norm": 2.75814655626698,
      "kl": 1.671875,
      "learning_rate": 4.024787870063773e-09,
      "loss": 0.14590351283550262,
      "memory(GiB)": 75.26,
      "reward": 2.0050243735313416,
      "reward_std": 0.48870939016342163,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2566775679588318,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.4329945743083954,
      "rewards/ReflectionBalance/mean": 0.8214306831359863,
      "rewards/ReflectionBalance/std": 0.2505883499979973,
      "step": 1776,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 498.7734375,
      "completions/min_length": 167.0,
      "epoch": 9.60377358490566,
      "grad_norm": 4.118129030995363,
      "kl": 1.8515625,
      "learning_rate": 3.916886095041039e-09,
      "loss": 0.1423066109418869,
      "memory(GiB)": 75.26,
      "reward": 1.844969928264618,
      "reward_std": 0.4236271381378174,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2524137645959854,
      "rewards/MultiModalAccuracyORM/mean": 0.1015625,
      "rewards/MultiModalAccuracyORM/std": 0.3021402209997177,
      "rewards/ReflectionBalance/mean": 0.8137198984622955,
      "rewards/ReflectionBalance/std": 0.22687390446662903,
      "step": 1777,
      "train_speed(iter/s)": 0.007126
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 516.1015625,
      "completions/min_length": 208.5,
      "epoch": 9.609164420485175,
      "grad_norm": 92.56430791242745,
      "kl": 3.2578125,
      "learning_rate": 3.8104447387498624e-09,
      "loss": 0.2593899667263031,
      "memory(GiB)": 75.26,
      "reward": 1.9506136178970337,
      "reward_std": 0.582008570432663,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.22265625,
      "rewards/MultiModalAccuracyORM/std": 0.41742460429668427,
      "rewards/ReflectionBalance/mean": 0.8217073678970337,
      "rewards/ReflectionBalance/std": 0.2415064200758934,
      "step": 1778,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 495.01171875,
      "completions/min_length": 217.5,
      "epoch": 9.61455525606469,
      "grad_norm": 3.919578409234818,
      "kl": 2.140625,
      "learning_rate": 3.705464114542867e-09,
      "loss": 0.15945671498775482,
      "memory(GiB)": 75.26,
      "reward": 2.0271207690238953,
      "reward_std": 0.4770311415195465,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.2734375,
      "rewards/MultiModalAccuracyORM/std": 0.43866652250289917,
      "rewards/ReflectionBalance/mean": 0.8318082690238953,
      "rewards/ReflectionBalance/std": 0.22565432637929916,
      "step": 1779,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 481.92578125,
      "completions/min_length": 256.5,
      "epoch": 9.619946091644206,
      "grad_norm": 3.1853979500915317,
      "kl": 2.26171875,
      "learning_rate": 3.6019445314725004e-09,
      "loss": 0.15284030139446259,
      "memory(GiB)": 75.26,
      "reward": 2.008209526538849,
      "reward_std": 0.4724028557538986,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.265625,
      "rewards/MultiModalAccuracyORM/std": 0.44197413325309753,
      "rewards/ReflectionBalance/mean": 0.8246158957481384,
      "rewards/ReflectionBalance/std": 0.22706284373998642,
      "step": 1780,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 499.15234375,
      "completions/min_length": 269.0,
      "epoch": 9.625336927223719,
      "grad_norm": 3.137425045479942,
      "kl": 2.5546875,
      "learning_rate": 3.499886294289933e-09,
      "loss": 0.2120659053325653,
      "memory(GiB)": 75.26,
      "reward": 1.884599506855011,
      "reward_std": 0.48840832710266113,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.3021402209997177,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3723055571317673,
      "rewards/ReflectionBalance/mean": 0.8181934058666229,
      "rewards/ReflectionBalance/std": 0.25989777594804764,
      "step": 1781,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 524.69921875,
      "completions/min_length": 249.0,
      "epoch": 9.630727762803234,
      "grad_norm": 3.453799851825957,
      "kl": 3.3515625,
      "learning_rate": 3.3992897034444413e-09,
      "loss": 0.26913225650787354,
      "memory(GiB)": 75.26,
      "reward": 1.8707128763198853,
      "reward_std": 0.5793968439102173,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.19140625,
      "rewards/MultiModalAccuracyORM/std": 0.3946705758571625,
      "rewards/ReflectionBalance/mean": 0.78477543592453,
      "rewards/ReflectionBalance/std": 0.27376748621463776,
      "step": 1782,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 539.9375,
      "completions/min_length": 249.5,
      "epoch": 9.63611859838275,
      "grad_norm": 2.7801982224710886,
      "kl": 2.671875,
      "learning_rate": 3.300155055082077e-09,
      "loss": 0.2081446647644043,
      "memory(GiB)": 75.26,
      "reward": 1.798913598060608,
      "reward_std": 0.5174748599529266,
      "rewards/Format/mean": 0.88671875,
      "rewards/Format/std": 0.31812146306037903,
      "rewards/MultiModalAccuracyORM/mean": 0.10546875,
      "rewards/MultiModalAccuracyORM/std": 0.3082972764968872,
      "rewards/ReflectionBalance/mean": 0.8067260980606079,
      "rewards/ReflectionBalance/std": 0.253323495388031,
      "step": 1783,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 474.2578125,
      "completions/min_length": 257.0,
      "epoch": 9.641509433962264,
      "grad_norm": 3.517714117776825,
      "kl": 2.26953125,
      "learning_rate": 3.202482641045445e-09,
      "loss": 0.19757619500160217,
      "memory(GiB)": 75.26,
      "reward": 1.8822868466377258,
      "reward_std": 0.4210052341222763,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.260444276034832,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.332206666469574,
      "rewards/ReflectionBalance/mean": 0.8275992572307587,
      "rewards/ReflectionBalance/std": 0.22601526975631714,
      "step": 1784,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 479.30078125,
      "completions/min_length": 278.5,
      "epoch": 9.64690026954178,
      "grad_norm": 3.818128294833475,
      "kl": 2.7265625,
      "learning_rate": 3.106272748872152e-09,
      "loss": 0.23575711250305176,
      "memory(GiB)": 75.26,
      "reward": 1.9361083507537842,
      "reward_std": 0.5312432050704956,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27079907804727554,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4116354286670685,
      "rewards/ReflectionBalance/mean": 0.8032958805561066,
      "rewards/ReflectionBalance/std": 0.2410518378019333,
      "step": 1785,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 951.0,
      "completions/mean_length": 477.953125,
      "completions/min_length": 66.5,
      "epoch": 9.652291105121293,
      "grad_norm": 3.1006385599039614,
      "kl": 2.1328125,
      "learning_rate": 3.011525661794578e-09,
      "loss": 0.1444101333618164,
      "memory(GiB)": 75.26,
      "reward": 1.927270531654358,
      "reward_std": 0.5226421058177948,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2961650490760803,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.40018807351589203,
      "rewards/ReflectionBalance/mean": 0.8139892518520355,
      "rewards/ReflectionBalance/std": 0.22436698526144028,
      "step": 1786,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 517.1484375,
      "completions/min_length": 247.0,
      "epoch": 9.657681940700808,
      "grad_norm": 4.013933114011643,
      "kl": 2.5625,
      "learning_rate": 2.9182416587384408e-09,
      "loss": 0.22249484062194824,
      "memory(GiB)": 75.26,
      "reward": 1.9522883892059326,
      "reward_std": 0.4644419699907303,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29794247448444366,
      "rewards/MultiModalAccuracyORM/mean": 0.25390625,
      "rewards/MultiModalAccuracyORM/std": 0.4316662400960922,
      "rewards/ReflectionBalance/mean": 0.796038419008255,
      "rewards/ReflectionBalance/std": 0.27769213914871216,
      "step": 1787,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.5,
      "completions/mean_length": 489.89453125,
      "completions/min_length": 249.0,
      "epoch": 9.663072776280323,
      "grad_norm": 4.390345541865228,
      "kl": 3.1640625,
      "learning_rate": 2.826421014322622e-09,
      "loss": 0.2236167937517166,
      "memory(GiB)": 75.26,
      "reward": 1.8915998935699463,
      "reward_std": 0.4580841362476349,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.3809676319360733,
      "rewards/ReflectionBalance/mean": 0.7978498935699463,
      "rewards/ReflectionBalance/std": 0.26959486305713654,
      "step": 1788,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.5,
      "completions/mean_length": 473.0,
      "completions/min_length": 259.0,
      "epoch": 9.668463611859838,
      "grad_norm": 3.2909352330696855,
      "kl": 1.984375,
      "learning_rate": 2.736063998857674e-09,
      "loss": 0.1452387571334839,
      "memory(GiB)": 75.26,
      "reward": 1.8721873760223389,
      "reward_std": 0.4153848737478256,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24081215262413025,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.34826864302158356,
      "rewards/ReflectionBalance/mean": 0.7940625250339508,
      "rewards/ReflectionBalance/std": 0.27527859061956406,
      "step": 1789,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 479.234375,
      "completions/min_length": 271.0,
      "epoch": 9.673854447439354,
      "grad_norm": 3.0599864602235582,
      "kl": 2.4765625,
      "learning_rate": 2.647170878345706e-09,
      "loss": 0.20297858119010925,
      "memory(GiB)": 75.26,
      "reward": 1.9476888179779053,
      "reward_std": 0.5513885021209717,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.28863146156072617,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4270860105752945,
      "rewards/ReflectionBalance/mean": 0.8070638179779053,
      "rewards/ReflectionBalance/std": 0.2685745060443878,
      "step": 1790,
      "train_speed(iter/s)": 0.007127
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 489.45703125,
      "completions/min_length": 286.0,
      "epoch": 9.679245283018869,
      "grad_norm": 4.282335973579526,
      "kl": 2.6015625,
      "learning_rate": 2.559741914479052e-09,
      "loss": 0.2013629972934723,
      "memory(GiB)": 75.26,
      "reward": 1.9678678512573242,
      "reward_std": 0.5329846441745758,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27465173602104187,
      "rewards/MultiModalAccuracyORM/mean": 0.24609375,
      "rewards/MultiModalAccuracyORM/std": 0.4322105348110199,
      "rewards/ReflectionBalance/mean": 0.8038053214550018,
      "rewards/ReflectionBalance/std": 0.256216898560524,
      "step": 1791,
      "train_speed(iter/s)": 0.007128
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 525.90625,
      "completions/min_length": 102.5,
      "epoch": 9.684636118598382,
      "grad_norm": 4.572127324084438,
      "kl": 3.21875,
      "learning_rate": 2.473777364639773e-09,
      "loss": 0.23937857151031494,
      "memory(GiB)": 75.26,
      "reward": 1.7574771642684937,
      "reward_std": 0.6065689623355865,
      "rewards/Format/mean": 0.84765625,
      "rewards/Format/std": 0.36039385199546814,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.3488178104162216,
      "rewards/ReflectionBalance/mean": 0.7691959738731384,
      "rewards/ReflectionBalance/std": 0.3184817135334015,
      "step": 1792,
      "train_speed(iter/s)": 0.007128
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 503.4609375,
      "completions/min_length": 260.0,
      "epoch": 9.690026954177897,
      "grad_norm": 3.849807635189776,
      "kl": 3.7109375,
      "learning_rate": 2.3892774818988214e-09,
      "loss": 0.3061811327934265,
      "memory(GiB)": 75.26,
      "reward": 1.917848289012909,
      "reward_std": 0.5433762073516846,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.4346197694540024,
      "rewards/ReflectionBalance/mean": 0.7733170390129089,
      "rewards/ReflectionBalance/std": 0.31763194501399994,
      "step": 1793,
      "train_speed(iter/s)": 0.007128
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 470.734375,
      "completions/min_length": 275.5,
      "epoch": 9.695417789757412,
      "grad_norm": 3.1045426876419695,
      "kl": 2.33984375,
      "learning_rate": 2.306242515015433e-09,
      "loss": 0.18790104985237122,
      "memory(GiB)": 75.26,
      "reward": 1.9993375539779663,
      "reward_std": 0.4491486996412277,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.27558503299951553,
      "rewards/MultiModalAccuracyORM/mean": 0.26953125,
      "rewards/MultiModalAccuracyORM/std": 0.4313432276248932,
      "rewards/ReflectionBalance/mean": 0.8157438039779663,
      "rewards/ReflectionBalance/std": 0.2205875664949417,
      "step": 1794,
      "train_speed(iter/s)": 0.007128
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 506.3125,
      "completions/min_length": 258.5,
      "epoch": 9.700808625336927,
      "grad_norm": 3.461855379984843,
      "kl": 2.1796875,
      "learning_rate": 2.2246727084360705e-09,
      "loss": 0.16965800523757935,
      "memory(GiB)": 75.26,
      "reward": 1.9529785513877869,
      "reward_std": 0.5012324452400208,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2850065380334854,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.4276812821626663,
      "rewards/ReflectionBalance/mean": 0.8045410215854645,
      "rewards/ReflectionBalance/std": 0.27180860191583633,
      "step": 1795,
      "train_speed(iter/s)": 0.007128
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 472.19140625,
      "completions/min_length": 263.0,
      "epoch": 9.706199460916443,
      "grad_norm": 3.3868388459811474,
      "kl": 2.3984375,
      "learning_rate": 2.144568302294092e-09,
      "loss": 0.20477712154388428,
      "memory(GiB)": 75.26,
      "reward": 1.9936963319778442,
      "reward_std": 0.421488419175148,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.41856054961681366,
      "rewards/ReflectionBalance/mean": 0.8257275223731995,
      "rewards/ReflectionBalance/std": 0.19376686215400696,
      "step": 1796,
      "train_speed(iter/s)": 0.007128
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 503.875,
      "completions/min_length": 260.5,
      "epoch": 9.711590296495956,
      "grad_norm": 4.2663908968844995,
      "kl": 3.0703125,
      "learning_rate": 2.065929532408861e-09,
      "loss": 0.23972517251968384,
      "memory(GiB)": 75.26,
      "reward": 1.873668611049652,
      "reward_std": 0.5195892155170441,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.29434798657894135,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.37352070212364197,
      "rewards/ReflectionBalance/mean": 0.7994498610496521,
      "rewards/ReflectionBalance/std": 0.2884756922721863,
      "step": 1797,
      "train_speed(iter/s)": 0.007128
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 486.37109375,
      "completions/min_length": 257.0,
      "epoch": 9.716981132075471,
      "grad_norm": 3.074952604050588,
      "kl": 2.02734375,
      "learning_rate": 1.988756630284916e-09,
      "loss": 0.17334578931331635,
      "memory(GiB)": 75.26,
      "reward": 1.9079036116600037,
      "reward_std": 0.38000665605068207,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2255912721157074,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.3541714698076248,
      "rewards/ReflectionBalance/mean": 0.8141536712646484,
      "rewards/ReflectionBalance/std": 0.2071049064397812,
      "step": 1798,
      "train_speed(iter/s)": 0.007128
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 505.984375,
      "completions/min_length": 203.5,
      "epoch": 9.722371967654986,
      "grad_norm": 4.675590047362404,
      "kl": 3.5390625,
      "learning_rate": 1.9130498231114677e-09,
      "loss": 0.2693076729774475,
      "memory(GiB)": 75.26,
      "reward": 1.7872832417488098,
      "reward_std": 0.5394104719161987,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3318064510822296,
      "rewards/MultiModalAccuracyORM/mean": 0.125,
      "rewards/MultiModalAccuracyORM/std": 0.3265802264213562,
      "rewards/ReflectionBalance/mean": 0.7872832119464874,
      "rewards/ReflectionBalance/std": 0.29266612231731415,
      "step": 1799,
      "train_speed(iter/s)": 0.007128
    },
    {
      "epoch": 9.727762803234501,
      "grad_norm": 4.122800574131436,
      "learning_rate": 1.8388093337618482e-09,
      "loss": 0.21843190491199493,
      "memory(GiB)": 75.26,
      "step": 1800,
      "train_speed(iter/s)": 0.007128
    },
    {
      "epoch": 9.727762803234501,
      "eval_clip_ratio": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 1025.75,
      "eval_completions/mean_length": 517.234375,
      "eval_completions/min_length": 262.75,
      "eval_kl": 2.45703125,
      "eval_loss": 0.19832026958465576,
      "eval_reward": 1.8730216324329376,
      "eval_reward_std": 0.45617765933275223,
      "eval_rewards/Format/mean": 0.919921875,
      "eval_rewards/Format/std": 0.271080557256937,
      "eval_rewards/MultiModalAccuracyORM/mean": 0.138671875,
      "eval_rewards/MultiModalAccuracyORM/std": 0.3297058641910553,
      "eval_rewards/ReflectionBalance/mean": 0.8144278824329376,
      "eval_rewards/ReflectionBalance/std": 0.2546803541481495,
      "eval_runtime": 248.4153,
      "eval_samples_per_second": 0.238,
      "eval_steps_per_second": 0.004,
      "step": 1800
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1044.5,
      "completions/mean_length": 485.345703125,
      "completions/min_length": 257.25,
      "epoch": 9.733153638814017,
      "grad_norm": 4.250414559338243,
      "kl": 2.69921875,
      "learning_rate": 1.7660353807925076e-09,
      "loss": 0.220834881067276,
      "memory(GiB)": 75.26,
      "reward": 1.9453678131103516,
      "reward_std": 0.5053767114877701,
      "rewards/Format/mean": 0.908203125,
      "rewards/Format/std": 0.28574176505208015,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4138292744755745,
      "rewards/ReflectionBalance/mean": 0.802789717912674,
      "rewards/ReflectionBalance/std": 0.26020174846053123,
      "step": 1801,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 493.75390625,
      "completions/min_length": 249.0,
      "epoch": 9.738544474393532,
      "grad_norm": 3.2159691394567758,
      "kl": 2.828125,
      "learning_rate": 1.6947281784426838e-09,
      "loss": 0.24810966849327087,
      "memory(GiB)": 75.26,
      "reward": 1.8523104190826416,
      "reward_std": 0.4564610719680786,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.2690252959728241,
      "rewards/MultiModalAccuracyORM/mean": 0.1328125,
      "rewards/MultiModalAccuracyORM/std": 0.3405093401670456,
      "rewards/ReflectionBalance/mean": 0.7976228892803192,
      "rewards/ReflectionBalance/std": 0.2586381658911705,
      "step": 1802,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 519.0234375,
      "completions/min_length": 168.5,
      "epoch": 9.743935309973045,
      "grad_norm": 4.817176157237968,
      "kl": 3.4609375,
      "learning_rate": 1.6248879366335677e-09,
      "loss": 0.23316505551338196,
      "memory(GiB)": 75.26,
      "reward": 1.8435611724853516,
      "reward_std": 0.46647006273269653,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3247780203819275,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3734995424747467,
      "rewards/ReflectionBalance/mean": 0.7966862320899963,
      "rewards/ReflectionBalance/std": 0.2965964078903198,
      "step": 1803,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 456.7265625,
      "completions/min_length": 196.0,
      "epoch": 9.74932614555256,
      "grad_norm": 3.263141315669964,
      "kl": 2.53125,
      "learning_rate": 1.5565148609677502e-09,
      "loss": 0.20351113379001617,
      "memory(GiB)": 75.26,
      "reward": 1.9928303956985474,
      "reward_std": 0.6007061898708344,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.3046875,
      "rewards/MultiModalAccuracyORM/std": 0.448483869433403,
      "rewards/ReflectionBalance/mean": 0.7975178956985474,
      "rewards/ReflectionBalance/std": 0.27379944920539856,
      "step": 1804,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.5,
      "completions/mean_length": 503.23828125,
      "completions/min_length": 180.5,
      "epoch": 9.754716981132075,
      "grad_norm": 3.443165074336393,
      "kl": 2.578125,
      "learning_rate": 1.489609152728777e-09,
      "loss": 0.19937491416931152,
      "memory(GiB)": 75.26,
      "reward": 1.9667723774909973,
      "reward_std": 0.4414631575345993,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.4145888388156891,
      "rewards/ReflectionBalance/mean": 0.8183349370956421,
      "rewards/ReflectionBalance/std": 0.2567252963781357,
      "step": 1805,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 495.57421875,
      "completions/min_length": 212.0,
      "epoch": 9.76010781671159,
      "grad_norm": 4.391812374114042,
      "kl": 2.890625,
      "learning_rate": 1.4241710088802595e-09,
      "loss": 0.2209075540304184,
      "memory(GiB)": 75.26,
      "reward": 1.8504304885864258,
      "reward_std": 0.4827694147825241,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3077666610479355,
      "rewards/MultiModalAccuracyORM/mean": 0.1484375,
      "rewards/MultiModalAccuracyORM/std": 0.34826576709747314,
      "rewards/ReflectionBalance/mean": 0.8074617683887482,
      "rewards/ReflectionBalance/std": 0.26566915214061737,
      "step": 1806,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 500.0078125,
      "completions/min_length": 222.5,
      "epoch": 9.765498652291106,
      "grad_norm": 3.0351433470790994,
      "kl": 2.20703125,
      "learning_rate": 1.3602006220655438e-09,
      "loss": 0.1589004397392273,
      "memory(GiB)": 75.26,
      "reward": 1.9202563762664795,
      "reward_std": 0.49428027868270874,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.24400590360164642,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.4007115960121155,
      "rewards/ReflectionBalance/mean": 0.7913501262664795,
      "rewards/ReflectionBalance/std": 0.2694598436355591,
      "step": 1807,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 482.78515625,
      "completions/min_length": 146.5,
      "epoch": 9.77088948787062,
      "grad_norm": 12.736729227713957,
      "kl": 2.3671875,
      "learning_rate": 1.2976981806069321e-09,
      "loss": 0.1603279411792755,
      "memory(GiB)": 75.26,
      "reward": 1.9422136545181274,
      "reward_std": 0.5031185150146484,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.275397852063179,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4120787978172302,
      "rewards/ReflectionBalance/mean": 0.8094010353088379,
      "rewards/ReflectionBalance/std": 0.25919903814792633,
      "step": 1808,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 504.43359375,
      "completions/min_length": 245.5,
      "epoch": 9.776280323450134,
      "grad_norm": 3.545500322019305,
      "kl": 2.8984375,
      "learning_rate": 1.2366638685054054e-09,
      "loss": 0.23422425985336304,
      "memory(GiB)": 75.26,
      "reward": 1.878969669342041,
      "reward_std": 0.4800274968147278,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.368965208530426,
      "rewards/ReflectionBalance/mean": 0.7930322289466858,
      "rewards/ReflectionBalance/std": 0.2723100781440735,
      "step": 1809,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 521.18359375,
      "completions/min_length": 174.0,
      "epoch": 9.78167115902965,
      "grad_norm": 3.3279029412909433,
      "kl": 2.080078125,
      "learning_rate": 1.1770978654397911e-09,
      "loss": 0.16618937253952026,
      "memory(GiB)": 75.26,
      "reward": 1.9282349348068237,
      "reward_std": 0.4491225779056549,
      "rewards/Format/mean": 0.921875,
      "rewards/Format/std": 0.24629417806863785,
      "rewards/MultiModalAccuracyORM/mean": 0.1796875,
      "rewards/MultiModalAccuracyORM/std": 0.38325364887714386,
      "rewards/ReflectionBalance/mean": 0.826672375202179,
      "rewards/ReflectionBalance/std": 0.2327633947134018,
      "step": 1810,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 506.62890625,
      "completions/min_length": 279.0,
      "epoch": 9.787061994609164,
      "grad_norm": 3.489891581709637,
      "kl": 2.859375,
      "learning_rate": 1.1190003467663745e-09,
      "loss": 0.24316313862800598,
      "memory(GiB)": 75.26,
      "reward": 1.8522875308990479,
      "reward_std": 0.444546639919281,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.29137909412384033,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36387722194194794,
      "rewards/ReflectionBalance/mean": 0.7897875905036926,
      "rewards/ReflectionBalance/std": 0.28162239491939545,
      "step": 1811,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 500.88671875,
      "completions/min_length": 247.5,
      "epoch": 9.79245283018868,
      "grad_norm": 4.387144075544024,
      "kl": 2.05859375,
      "learning_rate": 1.0623714835184538e-09,
      "loss": 0.16288959980010986,
      "memory(GiB)": 75.26,
      "reward": 1.8619433641433716,
      "reward_std": 0.421856090426445,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27313581854104996,
      "rewards/MultiModalAccuracyORM/mean": 0.12890625,
      "rewards/MultiModalAccuracyORM/std": 0.332206666469574,
      "rewards/ReflectionBalance/mean": 0.8150683641433716,
      "rewards/ReflectionBalance/std": 0.28113237023353577,
      "step": 1812,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 498.97265625,
      "completions/min_length": 237.0,
      "epoch": 9.797843665768195,
      "grad_norm": 3.653500245054087,
      "kl": 2.75,
      "learning_rate": 1.0072114424056754e-09,
      "loss": 0.25029218196868896,
      "memory(GiB)": 75.26,
      "reward": 1.9882762432098389,
      "reward_std": 0.5599372982978821,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.3125,
      "rewards/MultiModalAccuracyORM/std": 0.4625517278909683,
      "rewards/ReflectionBalance/mean": 0.7851513624191284,
      "rewards/ReflectionBalance/std": 0.26944103837013245,
      "step": 1813,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 492.6953125,
      "completions/min_length": 242.0,
      "epoch": 9.80323450134771,
      "grad_norm": 31.190717487429872,
      "kl": 3.1015625,
      "learning_rate": 9.535203858136442e-10,
      "loss": 0.235042542219162,
      "memory(GiB)": 75.26,
      "reward": 1.9454598426818848,
      "reward_std": 0.5405156016349792,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.3130832314491272,
      "rewards/MultiModalAccuracyORM/mean": 0.2578125,
      "rewards/MultiModalAccuracyORM/std": 0.43905749917030334,
      "rewards/ReflectionBalance/mean": 0.79702228307724,
      "rewards/ReflectionBalance/std": 0.256645143032074,
      "step": 1814,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 483.0234375,
      "completions/min_length": 267.0,
      "epoch": 9.808625336927223,
      "grad_norm": 3.308726377843706,
      "kl": 2.05078125,
      "learning_rate": 9.012984718034244e-10,
      "loss": 0.16425710916519165,
      "memory(GiB)": 75.26,
      "reward": 1.9335075616836548,
      "reward_std": 0.4520374983549118,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2813730239868164,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39068229496479034,
      "rewards/ReflectionBalance/mean": 0.83194500207901,
      "rewards/ReflectionBalance/std": 0.2163073867559433,
      "step": 1815,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.0,
      "completions/mean_length": 517.625,
      "completions/min_length": 280.0,
      "epoch": 9.814016172506738,
      "grad_norm": 3.4321212686143547,
      "kl": 2.7109375,
      "learning_rate": 8.505458541110399e-10,
      "loss": 0.20276299118995667,
      "memory(GiB)": 75.26,
      "reward": 1.8483642935752869,
      "reward_std": 0.4927678108215332,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.3425614684820175,
      "rewards/ReflectionBalance/mean": 0.8014892637729645,
      "rewards/ReflectionBalance/std": 0.2714012414216995,
      "step": 1816,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 490.22265625,
      "completions/min_length": 221.0,
      "epoch": 9.819407008086253,
      "grad_norm": 3.431507278236129,
      "kl": 2.7421875,
      "learning_rate": 8.012626821471968e-10,
      "loss": 0.2243196964263916,
      "memory(GiB)": 75.26,
      "reward": 1.9273568391799927,
      "reward_std": 0.5657837092876434,
      "rewards/Format/mean": 0.8828125,
      "rewards/Format/std": 0.3169707804918289,
      "rewards/MultiModalAccuracyORM/mean": 0.25,
      "rewards/MultiModalAccuracyORM/std": 0.43433645367622375,
      "rewards/ReflectionBalance/mean": 0.7945442795753479,
      "rewards/ReflectionBalance/std": 0.2766578420996666,
      "step": 1817,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 505.5078125,
      "completions/min_length": 141.0,
      "epoch": 9.824797843665769,
      "grad_norm": 3.211121946915542,
      "kl": 2.890625,
      "learning_rate": 7.534491009964506e-10,
      "loss": 0.20742031931877136,
      "memory(GiB)": 75.26,
      "reward": 1.88938307762146,
      "reward_std": 0.45683981478214264,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3786114752292633,
      "rewards/ReflectionBalance/mean": 0.8112581372261047,
      "rewards/ReflectionBalance/std": 0.24148563295602798,
      "step": 1818,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 915.5,
      "completions/mean_length": 481.97265625,
      "completions/min_length": 282.0,
      "epoch": 9.830188679245284,
      "grad_norm": 3.3027202940488856,
      "kl": 1.98046875,
      "learning_rate": 7.07105251417206e-10,
      "loss": 0.16539117693901062,
      "memory(GiB)": 75.26,
      "reward": 1.8438460230827332,
      "reward_std": 0.4369371384382248,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2548178359866142,
      "rewards/MultiModalAccuracyORM/mean": 0.08203125,
      "rewards/MultiModalAccuracyORM/std": 0.257690891623497,
      "rewards/ReflectionBalance/mean": 0.8321273028850555,
      "rewards/ReflectionBalance/std": 0.22023066133260727,
      "step": 1819,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 477.87109375,
      "completions/min_length": 244.5,
      "epoch": 9.835579514824797,
      "grad_norm": 3.1464693568904387,
      "kl": 2.08984375,
      "learning_rate": 6.62231269841218e-10,
      "loss": 0.15618589520454407,
      "memory(GiB)": 75.26,
      "reward": 2.0397077798843384,
      "reward_std": 0.42629721760749817,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.28515625,
      "rewards/MultiModalAccuracyORM/std": 0.44853854179382324,
      "rewards/ReflectionBalance/mean": 0.8209578692913055,
      "rewards/ReflectionBalance/std": 0.23329973965883255,
      "step": 1820,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 471.3359375,
      "completions/min_length": 250.0,
      "epoch": 9.840970350404312,
      "grad_norm": 3.5710254552742047,
      "kl": 2.34375,
      "learning_rate": 6.188272883727586e-10,
      "loss": 0.17834961414337158,
      "memory(GiB)": 75.26,
      "reward": 2.02678382396698,
      "reward_std": 0.5382144153118134,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3082972764968872,
      "rewards/MultiModalAccuracyORM/mean": 0.32421875,
      "rewards/MultiModalAccuracyORM/std": 0.4694543182849884,
      "rewards/ReflectionBalance/mean": 0.80803382396698,
      "rewards/ReflectionBalance/std": 0.2818182334303856,
      "step": 1821,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 484.66015625,
      "completions/min_length": 250.5,
      "epoch": 9.846361185983827,
      "grad_norm": 4.043885498769119,
      "kl": 2.2421875,
      "learning_rate": 5.768934347889498e-10,
      "loss": 0.1558411419391632,
      "memory(GiB)": 75.26,
      "reward": 2.0577298402786255,
      "reward_std": 0.41964904963970184,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2630501240491867,
      "rewards/MultiModalAccuracyORM/mean": 0.30859375,
      "rewards/MultiModalAccuracyORM/std": 0.46136754751205444,
      "rewards/ReflectionBalance/mean": 0.8233548104763031,
      "rewards/ReflectionBalance/std": 0.22360607981681824,
      "step": 1822,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 486.1328125,
      "completions/min_length": 245.0,
      "epoch": 9.851752021563343,
      "grad_norm": 3.7503984953415666,
      "kl": 2.921875,
      "learning_rate": 5.364298325387651e-10,
      "loss": 0.1936855912208557,
      "memory(GiB)": 75.26,
      "reward": 1.9038346409797668,
      "reward_std": 0.5659390985965729,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.32612966001033783,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41857391595840454,
      "rewards/ReflectionBalance/mean": 0.7983658611774445,
      "rewards/ReflectionBalance/std": 0.271054171025753,
      "step": 1823,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 508.8359375,
      "completions/min_length": 252.0,
      "epoch": 9.857142857142858,
      "grad_norm": 3.709682912986476,
      "kl": 2.23046875,
      "learning_rate": 4.974366007429732e-10,
      "loss": 0.17170721292495728,
      "memory(GiB)": 75.26,
      "reward": 1.859217882156372,
      "reward_std": 0.45761674642562866,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2799683064222336,
      "rewards/MultiModalAccuracyORM/mean": 0.13671875,
      "rewards/MultiModalAccuracyORM/std": 0.33660952746868134,
      "rewards/ReflectionBalance/mean": 0.8084367215633392,
      "rewards/ReflectionBalance/std": 0.24579787999391556,
      "step": 1824,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 521.1875,
      "completions/min_length": 187.5,
      "epoch": 9.862533692722373,
      "grad_norm": 3.771379341257998,
      "kl": 3.890625,
      "learning_rate": 4.599138541937497e-10,
      "loss": 0.31322407722473145,
      "memory(GiB)": 75.26,
      "reward": 1.8712076544761658,
      "reward_std": 0.5734427720308304,
      "rewards/Format/mean": 0.875,
      "rewards/Format/std": 0.3265802264213562,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.39171716570854187,
      "rewards/ReflectionBalance/mean": 0.8087076544761658,
      "rewards/ReflectionBalance/std": 0.2545403838157654,
      "step": 1825,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 516.61328125,
      "completions/min_length": 291.5,
      "epoch": 9.867924528301886,
      "grad_norm": 3.263937610407323,
      "kl": 2.40625,
      "learning_rate": 4.2386170335423356e-10,
      "loss": 0.1954231709241867,
      "memory(GiB)": 75.26,
      "reward": 1.8342008590698242,
      "reward_std": 0.42790259420871735,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.140625,
      "rewards/MultiModalAccuracyORM/std": 0.344347283244133,
      "rewards/ReflectionBalance/mean": 0.8029508292675018,
      "rewards/ReflectionBalance/std": 0.2618003636598587,
      "step": 1826,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 504.31640625,
      "completions/min_length": 237.5,
      "epoch": 9.873315363881401,
      "grad_norm": 3.1889284570508765,
      "kl": 1.65625,
      "learning_rate": 3.8928025435835954e-10,
      "loss": 0.12995901703834534,
      "memory(GiB)": 75.26,
      "reward": 1.9113166332244873,
      "reward_std": 0.4666277766227722,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.2562176361680031,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3542858511209488,
      "rewards/ReflectionBalance/mean": 0.8136604428291321,
      "rewards/ReflectionBalance/std": 0.24380672723054886,
      "step": 1827,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 532.51953125,
      "completions/min_length": 176.0,
      "epoch": 9.878706199460916,
      "grad_norm": 2.9175838523695687,
      "kl": 2.0234375,
      "learning_rate": 3.5616960901047045e-10,
      "loss": 0.1793748438358307,
      "memory(GiB)": 75.26,
      "reward": 1.9678987264633179,
      "reward_std": 0.4288051128387451,
      "rewards/Format/mean": 0.92578125,
      "rewards/Format/std": 0.2577303349971771,
      "rewards/MultiModalAccuracyORM/mean": 0.2109375,
      "rewards/MultiModalAccuracyORM/std": 0.40395987033843994,
      "rewards/ReflectionBalance/mean": 0.8311800360679626,
      "rewards/ReflectionBalance/std": 0.21852627396583557,
      "step": 1828,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1023.0,
      "completions/mean_length": 494.7421875,
      "completions/min_length": 173.0,
      "epoch": 9.884097035040432,
      "grad_norm": 3.4258909916294575,
      "kl": 2.67578125,
      "learning_rate": 3.2452986478487264e-10,
      "loss": 0.21608313918113708,
      "memory(GiB)": 75.26,
      "reward": 1.8189566731452942,
      "reward_std": 0.471726730465889,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2875154986977577,
      "rewards/MultiModalAccuracyORM/mean": 0.08984375,
      "rewards/MultiModalAccuracyORM/std": 0.2829600051045418,
      "rewards/ReflectionBalance/mean": 0.8228629529476166,
      "rewards/ReflectionBalance/std": 0.24376384913921356,
      "step": 1829,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1028.5,
      "completions/mean_length": 496.13671875,
      "completions/min_length": 207.5,
      "epoch": 9.889487870619947,
      "grad_norm": 3.029398875991552,
      "kl": 3.0546875,
      "learning_rate": 2.943611148258918e-10,
      "loss": 0.2370772361755371,
      "memory(GiB)": 75.26,
      "reward": 1.9078710675239563,
      "reward_std": 0.4828376770019531,
      "rewards/Format/mean": 0.890625,
      "rewards/Format/std": 0.31232215464115143,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.41230008006095886,
      "rewards/ReflectionBalance/mean": 0.8024023175239563,
      "rewards/ReflectionBalance/std": 0.2665608823299408,
      "step": 1830,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.5,
      "completions/mean_length": 465.984375,
      "completions/min_length": 274.5,
      "epoch": 9.89487870619946,
      "grad_norm": 4.265450451272317,
      "kl": 1.39453125,
      "learning_rate": 2.656634479472064e-10,
      "loss": 0.14434927701950073,
      "memory(GiB)": 75.26,
      "reward": 2.050917863845825,
      "reward_std": 0.4826832562685013,
      "rewards/Format/mean": 0.9453125,
      "rewards/Format/std": 0.2276071086525917,
      "rewards/MultiModalAccuracyORM/mean": 0.265625,
      "rewards/MultiModalAccuracyORM/std": 0.4343355745077133,
      "rewards/ReflectionBalance/mean": 0.8399804532527924,
      "rewards/ReflectionBalance/std": 0.24254900962114334,
      "step": 1831,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.5,
      "completions/mean_length": 471.00390625,
      "completions/min_length": 100.5,
      "epoch": 9.900269541778975,
      "grad_norm": 2.898358531744884,
      "kl": 2.3984375,
      "learning_rate": 2.3843694863190377e-10,
      "loss": 0.16280344128608704,
      "memory(GiB)": 75.26,
      "reward": 1.8578092455863953,
      "reward_std": 0.5094453692436218,
      "rewards/Format/mean": 0.89453125,
      "rewards/Format/std": 0.3050640821456909,
      "rewards/MultiModalAccuracyORM/mean": 0.171875,
      "rewards/MultiModalAccuracyORM/std": 0.3774644732475281,
      "rewards/ReflectionBalance/mean": 0.7914029657840729,
      "rewards/ReflectionBalance/std": 0.26934046298265457,
      "step": 1832,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1027.0,
      "completions/mean_length": 505.46484375,
      "completions/min_length": 271.0,
      "epoch": 9.90566037735849,
      "grad_norm": 3.4671951545519284,
      "kl": 3.1953125,
      "learning_rate": 2.1268169703203554e-10,
      "loss": 0.2643199563026428,
      "memory(GiB)": 75.26,
      "reward": 1.9160611629486084,
      "reward_std": 0.6046275496482849,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.30072054266929626,
      "rewards/MultiModalAccuracyORM/mean": 0.23828125,
      "rewards/MultiModalAccuracyORM/std": 0.42748305201530457,
      "rewards/ReflectionBalance/mean": 0.7793424129486084,
      "rewards/ReflectionBalance/std": 0.2906063497066498,
      "step": 1833,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 488.46484375,
      "completions/min_length": 189.0,
      "epoch": 9.911051212938006,
      "grad_norm": 4.732364214202167,
      "kl": 2.109375,
      "learning_rate": 1.8839776896845127e-10,
      "loss": 0.1649421751499176,
      "memory(GiB)": 75.26,
      "reward": 1.9213964939117432,
      "reward_std": 0.39911749958992004,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27079907804727554,
      "rewards/MultiModalAccuracyORM/mean": 0.18359375,
      "rewards/MultiModalAccuracyORM/std": 0.38105446100234985,
      "rewards/ReflectionBalance/mean": 0.8198339939117432,
      "rewards/ReflectionBalance/std": 0.24657019972801208,
      "step": 1834,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.5,
      "completions/mean_length": 513.22265625,
      "completions/min_length": 290.5,
      "epoch": 9.91644204851752,
      "grad_norm": 3.178835892344946,
      "kl": 2.5625,
      "learning_rate": 1.65585235930632e-10,
      "loss": 0.20224934816360474,
      "memory(GiB)": 75.26,
      "reward": 1.8618611693382263,
      "reward_std": 0.49389560520648956,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.16015625,
      "rewards/MultiModalAccuracyORM/std": 0.3678421527147293,
      "rewards/ReflectionBalance/mean": 0.7993611395359039,
      "rewards/ReflectionBalance/std": 0.2631334960460663,
      "step": 1835,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.5,
      "completions/mean_length": 488.1015625,
      "completions/min_length": 268.0,
      "epoch": 9.921832884097036,
      "grad_norm": 3.4385920633879263,
      "kl": 2.921875,
      "learning_rate": 1.4424416507641257e-10,
      "loss": 0.22999262809753418,
      "memory(GiB)": 75.26,
      "reward": 1.9244986772537231,
      "reward_std": 0.45936471223831177,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.21484375,
      "rewards/MultiModalAccuracyORM/std": 0.4089515954256058,
      "rewards/ReflectionBalance/mean": 0.7994986772537231,
      "rewards/ReflectionBalance/std": 0.2585036903619766,
      "step": 1836,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1029.0,
      "completions/mean_length": 483.10546875,
      "completions/min_length": 235.0,
      "epoch": 9.92722371967655,
      "grad_norm": 4.353641600385665,
      "kl": 2.6640625,
      "learning_rate": 1.243746192316486e-10,
      "loss": 0.21052131056785583,
      "memory(GiB)": 75.26,
      "reward": 1.85494464635849,
      "reward_std": 0.5218815356492996,
      "rewards/Format/mean": 0.90625,
      "rewards/Format/std": 0.2923160344362259,
      "rewards/MultiModalAccuracyORM/mean": 0.14453125,
      "rewards/MultiModalAccuracyORM/std": 0.3526121824979782,
      "rewards/ReflectionBalance/mean": 0.80416339635849,
      "rewards/ReflectionBalance/std": 0.27397094666957855,
      "step": 1837,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 485.421875,
      "completions/min_length": 249.5,
      "epoch": 9.932614555256064,
      "grad_norm": 3.8793025835517243,
      "kl": 1.90234375,
      "learning_rate": 1.0597665689038304e-10,
      "loss": 0.17201349139213562,
      "memory(GiB)": 75.26,
      "reward": 1.900846242904663,
      "reward_std": 0.36628712713718414,
      "rewards/Format/mean": 0.94140625,
      "rewards/Format/std": 0.23196382820606232,
      "rewards/MultiModalAccuracyORM/mean": 0.12109375,
      "rewards/MultiModalAccuracyORM/std": 0.25683019682765007,
      "rewards/ReflectionBalance/mean": 0.8383463323116302,
      "rewards/ReflectionBalance/std": 0.18994846940040588,
      "step": 1838,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1022.0,
      "completions/mean_length": 500.8828125,
      "completions/min_length": 287.0,
      "epoch": 9.93800539083558,
      "grad_norm": 3.4923594578921455,
      "kl": 2.7421875,
      "learning_rate": 8.905033221429104e-11,
      "loss": 0.20933261513710022,
      "memory(GiB)": 75.26,
      "reward": 1.9469172954559326,
      "reward_std": 0.5051607489585876,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2863408625125885,
      "rewards/MultiModalAccuracyORM/mean": 0.2265625,
      "rewards/MultiModalAccuracyORM/std": 0.41983383893966675,
      "rewards/ReflectionBalance/mean": 0.810198575258255,
      "rewards/ReflectionBalance/std": 0.2461383193731308,
      "step": 1839,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 527.5859375,
      "completions/min_length": 281.5,
      "epoch": 9.943396226415095,
      "grad_norm": 4.171095495159822,
      "kl": 2.92578125,
      "learning_rate": 7.359569503279095e-11,
      "loss": 0.2073386013507843,
      "memory(GiB)": 75.26,
      "reward": 1.85813969373703,
      "reward_std": 0.5247599482536316,
      "rewards/Format/mean": 0.87890625,
      "rewards/Format/std": 0.3206050544977188,
      "rewards/MultiModalAccuracyORM/mean": 0.1875,
      "rewards/MultiModalAccuracyORM/std": 0.3897709399461746,
      "rewards/ReflectionBalance/mean": 0.7917334139347076,
      "rewards/ReflectionBalance/std": 0.25738850980997086,
      "step": 1840,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 509.98046875,
      "completions/min_length": 260.5,
      "epoch": 9.94878706199461,
      "grad_norm": 4.42201471687718,
      "kl": 2.59765625,
      "learning_rate": 5.961279084282234e-11,
      "loss": 0.19616827368736267,
      "memory(GiB)": 75.26,
      "reward": 1.9302091598510742,
      "reward_std": 0.4929715245962143,
      "rewards/Format/mean": 0.8984375,
      "rewards/Format/std": 0.2960064336657524,
      "rewards/MultiModalAccuracyORM/mean": 0.21875,
      "rewards/MultiModalAccuracyORM/std": 0.41502299904823303,
      "rewards/ReflectionBalance/mean": 0.8130216300487518,
      "rewards/ReflectionBalance/std": 0.236529640853405,
      "step": 1841,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 476.58984375,
      "completions/min_length": 260.5,
      "epoch": 9.954177897574123,
      "grad_norm": 3.101303901105015,
      "kl": 2.25,
      "learning_rate": 4.710166080856836e-11,
      "loss": 0.16794493794441223,
      "memory(GiB)": 75.26,
      "reward": 1.922021508216858,
      "reward_std": 0.4434638172388077,
      "rewards/Format/mean": 0.90234375,
      "rewards/Format/std": 0.2973542660474777,
      "rewards/MultiModalAccuracyORM/mean": 0.19921875,
      "rewards/MultiModalAccuracyORM/std": 0.39205317199230194,
      "rewards/ReflectionBalance/mean": 0.8204590082168579,
      "rewards/ReflectionBalance/std": 0.19776280224323273,
      "step": 1842,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1030.0,
      "completions/mean_length": 492.89453125,
      "completions/min_length": 298.0,
      "epoch": 9.959568733153638,
      "grad_norm": 2.963211868950635,
      "kl": 2.4609375,
      "learning_rate": 3.606234176156686e-11,
      "loss": 0.18633785843849182,
      "memory(GiB)": 75.26,
      "reward": 1.9448941946029663,
      "reward_std": 0.5017232596874237,
      "rewards/Format/mean": 0.91015625,
      "rewards/Format/std": 0.2869994640350342,
      "rewards/MultiModalAccuracyORM/mean": 0.234375,
      "rewards/MultiModalAccuracyORM/std": 0.4243630915880203,
      "rewards/ReflectionBalance/mean": 0.8003629446029663,
      "rewards/ReflectionBalance/std": 0.2515207678079605,
      "step": 1843,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1026.0,
      "completions/mean_length": 466.85546875,
      "completions/min_length": 135.0,
      "epoch": 9.964959568733153,
      "grad_norm": 2.8097613850709684,
      "kl": 1.8671875,
      "learning_rate": 2.6494866200543753e-11,
      "loss": 0.12971298396587372,
      "memory(GiB)": 75.26,
      "reward": 1.9688518047332764,
      "reward_std": 0.44952045381069183,
      "rewards/Format/mean": 0.9375,
      "rewards/Format/std": 0.24247010797262192,
      "rewards/MultiModalAccuracyORM/mean": 0.203125,
      "rewards/MultiModalAccuracyORM/std": 0.4037879854440689,
      "rewards/ReflectionBalance/mean": 0.8282266855239868,
      "rewards/ReflectionBalance/std": 0.23345203697681427,
      "step": 1844,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1025.0,
      "completions/mean_length": 493.796875,
      "completions/min_length": 197.5,
      "epoch": 9.970350404312669,
      "grad_norm": 166.18213604086856,
      "kl": 2.7421875,
      "learning_rate": 1.839926229102451e-11,
      "loss": 0.19986242055892944,
      "memory(GiB)": 75.26,
      "reward": 1.8872509598731995,
      "reward_std": 0.46079692244529724,
      "rewards/Format/mean": 0.9140625,
      "rewards/Format/std": 0.2810242921113968,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3642655462026596,
      "rewards/ReflectionBalance/mean": 0.8052197098731995,
      "rewards/ReflectionBalance/std": 0.2724013775587082,
      "step": 1845,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1024.0,
      "completions/mean_length": 472.61328125,
      "completions/min_length": 223.5,
      "epoch": 9.975741239892184,
      "grad_norm": 3.1951314295877484,
      "kl": 1.8046875,
      "learning_rate": 1.1775553865833732e-11,
      "loss": 0.13364873826503754,
      "memory(GiB)": 75.26,
      "reward": 2.057664394378662,
      "reward_std": 0.42827752232551575,
      "rewards/Format/mean": 0.9296875,
      "rewards/Format/std": 0.24400590360164642,
      "rewards/MultiModalAccuracyORM/mean": 0.29296875,
      "rewards/MultiModalAccuracyORM/std": 0.4494776427745819,
      "rewards/ReflectionBalance/mean": 0.8350081145763397,
      "rewards/ReflectionBalance/std": 0.21412844210863113,
      "step": 1846,
      "train_speed(iter/s)": 0.007119
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 929.0,
      "completions/mean_length": 503.86328125,
      "completions/min_length": 301.5,
      "epoch": 9.981132075471699,
      "grad_norm": 3.5651628339958865,
      "kl": 1.4765625,
      "learning_rate": 6.623760424318003e-12,
      "loss": 0.11782091110944748,
      "memory(GiB)": 75.26,
      "reward": 2.1088573932647705,
      "reward_std": 0.3560582995414734,
      "rewards/Format/mean": 0.95703125,
      "rewards/Format/std": 0.19744773209095,
      "rewards/MultiModalAccuracyORM/mean": 0.296875,
      "rewards/MultiModalAccuracyORM/std": 0.4585958868265152,
      "rewards/ReflectionBalance/mean": 0.8549511730670929,
      "rewards/ReflectionBalance/std": 0.1755513995885849,
      "step": 1847,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 931.0,
      "completions/mean_length": 491.17578125,
      "completions/min_length": 277.5,
      "epoch": 9.986522911051212,
      "grad_norm": 2.977772979484868,
      "kl": 1.8203125,
      "learning_rate": 2.9438971329565077e-12,
      "loss": 0.13274043798446655,
      "memory(GiB)": 75.26,
      "reward": 1.93207848072052,
      "reward_std": 0.41342656314373016,
      "rewards/Format/mean": 0.93359375,
      "rewards/Format/std": 0.24884266406297684,
      "rewards/MultiModalAccuracyORM/mean": 0.16796875,
      "rewards/MultiModalAccuracyORM/std": 0.3542858511209488,
      "rewards/ReflectionBalance/mean": 0.83051598072052,
      "rewards/ReflectionBalance/std": 0.2158743515610695,
      "step": 1848,
      "train_speed(iter/s)": 0.00712
    },
    {
      "clip_ratio": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1053.0,
      "completions/mean_length": 505.8203125,
      "completions/min_length": 266.0,
      "epoch": 9.991913746630727,
      "grad_norm": 29.58928281881917,
      "kl": 3.03125,
      "learning_rate": 7.359748249169406e-13,
      "loss": 0.21770888566970825,
      "memory(GiB)": 75.26,
      "reward": 1.8838125467300415,
      "reward_std": 0.4801875352859497,
      "rewards/Format/mean": 0.91796875,
      "rewards/Format/std": 0.27079907804727554,
      "rewards/MultiModalAccuracyORM/mean": 0.15625,
      "rewards/MultiModalAccuracyORM/std": 0.36387722194194794,
      "rewards/ReflectionBalance/mean": 0.8095939457416534,
      "rewards/ReflectionBalance/std": 0.2505914717912674,
      "step": 1849,
      "train_speed(iter/s)": 0.007121
    },
    {
      "epoch": 9.997304582210242,
      "grad_norm": 3.6624050852856405,
      "learning_rate": 0.0,
      "loss": 0.2296832650899887,
      "memory(GiB)": 75.26,
      "step": 1850,
      "train_speed(iter/s)": 0.00712
    },
    {
      "epoch": 9.997304582210242,
      "eval_clip_ratio": 0.0,
      "eval_completions/clipped_ratio": 0.0,
      "eval_completions/max_length": 1029.0,
      "eval_completions/mean_length": 527.783203125,
      "eval_completions/min_length": 283.5,
      "eval_kl": 2.359375,
      "eval_loss": 0.20960931479930878,
      "eval_reward": 1.8751505017280579,
      "eval_reward_std": 0.4248453676700592,
      "eval_rewards/Format/mean": 0.923828125,
      "eval_rewards/Format/std": 0.2651078440248966,
      "eval_rewards/MultiModalAccuracyORM/mean": 0.130859375,
      "eval_rewards/MultiModalAccuracyORM/std": 0.3280014581978321,
      "eval_rewards/ReflectionBalance/mean": 0.8204630464315414,
      "eval_rewards/ReflectionBalance/std": 0.23137042298913002,
      "eval_runtime": 248.347,
      "eval_samples_per_second": 0.238,
      "eval_steps_per_second": 0.004,
      "step": 1850
    }
  ],
  "logging_steps": 1,
  "max_steps": 1850,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 10,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 16,
  "trial_name": null,
  "trial_params": null
}